zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Multi-valued variants (#2386)

Browse Source 
Modifications:
- added support for multi-valued variants
- refactored code related to variants into variant.py
- added new generic features to AutotoolsPackage that leverage multi-valued variants
- modified openmpi to use new features
- added unit tests for the new semantics
This commit is contained in:
Massimiliano Culpo
2017-05-01 22:08:47 +02:00
committed by Todd Gamblin
parent 5d0d670b72
commit 9e4b0eb34a
22 changed files with 1959 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
lib/spack/llnl/util/lang.py
View File

@@ -266,10 +266,28 @@ def setter(name, value):
@key_ordering
class HashableMap(dict):
class HashableMap(collections.MutableMapping):
"""This is a hashable, comparable dictionary. Hash is performed on
a tuple of the values in the dictionary."""
def __init__(self):
self.dict = {}
def __getitem__(self, key):
return self.dict[key]
def __setitem__(self, key, value):
self.dict[key] = value
def __iter__(self):
return iter(self.dict)
def __len__(self):
return len(self.dict)
def __delitem__(self, key):
del self.dict[key]
def _cmp_key(self):
return tuple(sorted(self.values()))

16
lib/spack/llnl/util/tty/__init__.py
View File

@@ -22,22 +22,22 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import sys
import os
import textwrap
import fcntl
import termios
import os
import struct
import sys
import termios
import textwrap
import traceback
from six import StringIO
from six.moves import input
from llnl.util.tty.color import *
_debug = False
_debug = False
_verbose = False
_stacktrace = False
indent = " "
indent = " "
def is_verbose():
@@ -100,7 +100,7 @@ def msg(message, *args, **kwargs):
def info(message, *args, **kwargs):
format = kwargs.get('format', '*b')
stream = kwargs.get('stream', sys.stdout)
wrap = kwargs.get('wrap', False)
wrap = kwargs.get('wrap', False)
break_long_words = kwargs.get('break_long_words', False)
st_countback = kwargs.get('countback', 3)
@@ -218,7 +218,7 @@ def hline(label=None, **kwargs):
char (str): Char to draw the line with. Default '-'
max_width (int): Maximum width of the line. Default is 64 chars.
"""
char = kwargs.pop('char', '-')
char = kwargs.pop('char', '-')
max_width = kwargs.pop('max_width', 64)
if kwargs:
raise TypeError(

45
lib/spack/spack/build_systems/autotools.py
View File

@@ -289,6 +289,51 @@ def check(self):
self._if_make_target_execute('test')
self._if_make_target_execute('check')
def _activate_or_not(self, active, inactive, name, active_parameters=None):
spec = self.spec
args = []
# For each allowed value in the list of values
for value in self.variants[name].values:
# Check if the value is active in the current spec
condition = '{name}={value}'.format(name=name, value=value)
activated = condition in spec
# Search for an override in the package for this value
override_name = '{0}_or_{1}_{2}'.format(active, inactive, value)
line_generator = getattr(self, override_name, None)
# If not available use a sensible default
if line_generator is None:
def _default_generator(is_activated):
if is_activated:
line = '--{0}-{1}'.format(active, value)
if active_parameters is not None and active_parameters(value): # NOQA=ignore=E501
line += '={0}'.format(active_parameters(value))
return line
return '--{0}-{1}'.format(inactive, value)
line_generator = _default_generator
args.append(line_generator(activated))
return args
def with_or_without(self, name, active_parameters=None):
"""Inspects the multi-valued variant 'name' and returns the configure
arguments that activate / deactivate the selected feature.
:param str name: name of a valid multi-valued variant
:param callable active_parameters: if present accepts a single value
and returns the parameter to be used leading to an entry of the
type '--with-{name}={parameter}
"""
return self._activate_or_not(
'with', 'without', name, active_parameters
)
def enable_or_disable(self, name, active_parameters=None):
"""Inspects the multi-valued variant 'name' and returns the configure
arguments that activate / deactivate the selected feature.
"""
return self._activate_or_not(
'enable', 'disable', name, active_parameters
)
run_after('install')(PackageBase._run_default_install_time_test_callbacks)
def installcheck(self):

98
lib/spack/spack/cmd/info.py
View File

@@ -25,7 +25,7 @@
from __future__ import print_function
import textwrap
import itertools
from llnl.util.tty.colify import *
import spack
import spack.fetch_strategy as fs
@@ -49,6 +49,81 @@ def setup_parser(subparser):
'name', metavar="PACKAGE", help="name of package to get info for")
class VariantFormatter(object):
def __init__(self, variants, max_widths=(25, 20, 35)):
self.variants = variants
self.headers = ('Name [Default]', 'Allowed values', 'Description')
# Set max headers lengths
self.max_column_widths = max_widths
# Formats
fmt_name = '{0} [{1}]'
# Initialize column widths with the length of the
# corresponding headers, as they cannot be shorter
# than that
self.column_widths = [len(x) for x in self.headers]
# Update according to line lengths
for k, v in variants.items():
candidate_max_widths = (
len(fmt_name.format(k, self.default(v))), # Name [Default]
len(v.allowed_values), # Allowed values
len(v.description) # Description
)
self.column_widths = (
max(self.column_widths[0], candidate_max_widths[0]),
max(self.column_widths[1], candidate_max_widths[1]),
max(self.column_widths[2], candidate_max_widths[2])
)
# Reduce to at most the maximum allowed
self.column_widths = (
min(self.column_widths[0], self.max_column_widths[0]),
min(self.column_widths[1], self.max_column_widths[1]),
min(self.column_widths[2], self.max_column_widths[2])
)
# Compute the format
self.fmt = "%%-%ss%%-%ss%%s" % (
self.column_widths[0] + 4,
self.column_widths[1] + 4
)
def default(self, v):
s = 'on' if v.default is True else 'off'
if not isinstance(v.default, bool):
s = v.default
return s
@property
def lines(self):
if not self.variants:
yield " None"
else:
yield " " + self.fmt % self.headers
yield '\n'
for k, v in sorted(self.variants.items()):
name = textwrap.wrap(
'{0} [{1}]'.format(k, self.default(v)),
width=self.column_widths[0]
)
allowed = textwrap.wrap(
v.allowed_values,
width=self.column_widths[1]
)
description = textwrap.wrap(
v.description,
width=self.column_widths[2]
)
for t in itertools.izip_longest(
name, allowed, description, fillvalue=''
):
yield " " + self.fmt % t
yield '' # Trigger a new line
def print_text_info(pkg):
"""Print out a plain text description of a package."""
header = "{0}: ".format(pkg.build_system_class)
@@ -70,25 +145,10 @@ def print_text_info(pkg):
print()
print("Variants:")
if not pkg.variants:
print(" None")
else:
pad = padder(pkg.variants, 4)
maxv = max(len(v) for v in sorted(pkg.variants))
fmt = "%%-%ss%%-10s%%s" % (maxv + 4)
print(" " + fmt % ('Name', 'Default', 'Description'))
print()
for name in sorted(pkg.variants):
v = pkg.variants[name]
default = 'on' if v.default else 'off'
lines = textwrap.wrap(v.description)
lines[1:] = [" " + (" " * maxv) + l for l in lines[1:]]
desc = "\n".join(lines)
print(" " + fmt % (name, default, desc))
formatter = VariantFormatter(pkg.variants)
for line in formatter.lines:
print(line)
print()
print("Installation Phases:")

3
lib/spack/spack/cmd/spec.py
View File

@@ -69,7 +69,8 @@ def spec(parser, args):
for spec in spack.cmd.parse_specs(args.specs):
# With -y, just print YAML to output.
if args.yaml:
spec.concretize()
if spec.name in spack.repo:
spec.concretize()
print(spec.to_yaml())
continue

7
lib/spack/spack/concretize.py
View File

@@ -245,14 +245,15 @@ def concretize_variants(self, spec):
"""
changed = False
preferred_variants = PackagePrefs.preferred_variants(spec.name)
for name, variant in spec.package_class.variants.items():
pkg_cls = spec.package_class
for name, variant in pkg_cls.variants.items():
if name not in spec.variants:
changed = True
if name in preferred_variants:
spec.variants[name] = preferred_variants.get(name)
else:
spec.variants[name] = spack.spec.VariantSpec(
name, variant.default)
spec.variants[name] = variant.make_default()
return changed
def concretize_compiler(self, spec):

36
lib/spack/spack/directives.py
View File

@@ -368,9 +368,37 @@ def _execute(pkg):
@directive('variants')
def variant(name, default=False, description=""):
def variant(
name,
default=None,
description='',
values=(True, False),
multi=False,
validator=None
):
"""Define a variant for the package. Packager can specify a default
value (on or off) as well as a text description."""
value as well as a text description.
Args:
name (str): name of the variant
default (str or bool): default value for the variant, if not
specified otherwise the default will be False for a boolean
variant and 'nothing' for a multi-valued variant
description (str): description of the purpose of the variant
values (tuple or callable): either a tuple of strings containing the
allowed values, or a callable accepting one value and returning
True if it is valid
multi (bool): if False only one value per spec is allowed for
this variant
validator (callable): optional group validator to enforce additional
logic. It receives a tuple of values and should raise an instance
of SpackError if the group doesn't meet the additional constraints
"""
if default is None:
default = False if values == (True, False) else ''
default = default
description = str(description).strip()
def _execute(pkg):
@@ -379,7 +407,9 @@ def _execute(pkg):
msg = "Invalid variant name in {0}: '{1}'"
raise DirectiveError(directive, msg.format(pkg.name, name))
pkg.variants[name] = Variant(default, description)
pkg.variants[name] = Variant(
name, default, description, values, multi, validator
)
return _execute

15
lib/spack/spack/error.py
View File

@@ -100,3 +100,18 @@ def __init__(self, message, url):
"No network connection: " + str(message),
"URL was: " + str(url))
self.url = url
class SpecError(SpackError):
"""Superclass for all errors that occur while constructing specs."""
class UnsatisfiableSpecError(SpecError):
"""Raised when a spec conflicts with package constraints.
Provide the requirement that was violated when raising."""
def __init__(self, provided, required, constraint_type):
super(UnsatisfiableSpecError, self).__init__(
"%s does not satisfy %s" % (provided, required))
self.provided = provided
self.required = required
self.constraint_type = constraint_type

233
lib/spack/spack/spec.py
View File

@@ -121,12 +121,14 @@
from llnl.util.lang import *
from llnl.util.tty.color import *
from spack.build_environment import get_path_from_module, load_module
from spack.error import SpecError, UnsatisfiableSpecError
from spack.provider_index import ProviderIndex
from spack.util.crypto import prefix_bits
from spack.util.executable import Executable
from spack.util.prefix import Prefix
from spack.util.spack_yaml import syaml_dict
from spack.util.string import *
from spack.variant import *
from spack.version import *
from yaml.error import MarkedYAMLError
@@ -606,81 +608,6 @@ def __str__(self):
self.spec.name if self.spec else None)
@key_ordering
class VariantSpec(object):
"""Variants are named, build-time options for a package. Names depend
on the particular package being built, and each named variant can
be enabled or disabled.
"""
def __init__(self, name, value):
self.name = name
self.value = value
def _cmp_key(self):
return (self.name, self.value)
def copy(self):
return VariantSpec(self.name, self.value)
def __str__(self):
if type(self.value) == bool:
return '{0}{1}'.format('+' if self.value else '~', self.name)
else:
return ' {0}={1} '.format(self.name, self.value)
class VariantMap(HashableMap):
def __init__(self, spec):
super(VariantMap, self).__init__()
self.spec = spec
def satisfies(self, other, strict=False):
if strict or self.spec._concrete:
return all(k in self and self[k].value == other[k].value
for k in other)
else:
return all(self[k].value == other[k].value
for k in other if k in self)
def constrain(self, other):
"""Add all variants in other that aren't in self to self.
Raises an error if any common variants don't match.
Return whether the spec changed.
"""
if other.spec._concrete:
for k in self:
if k not in other:
raise UnsatisfiableVariantSpecError(self[k], '<absent>')
changed = False
for k in other:
if k in self:
if self[k].value != other[k].value:
raise UnsatisfiableVariantSpecError(self[k], other[k])
else:
self[k] = other[k].copy()
changed = True
return changed
@property
def concrete(self):
return self.spec._concrete or all(
v in self for v in self.spec.package_class.variants)
def copy(self):
clone = VariantMap(None)
for name, variant in self.items():
clone[name] = variant.copy()
return clone
def __str__(self):
sorted_keys = sorted(self.keys())
return ''.join(str(self[key]) for key in sorted_keys)
_valid_compiler_flags = [
'cflags', 'cxxflags', 'fflags', 'ldflags', 'ldlibs', 'cppflags']
@@ -1094,17 +1021,6 @@ def _add_version(self, version):
"""Called by the parser to add an allowable version."""
self.versions.add(version)
def _add_variant(self, name, value):
"""Called by the parser to add a variant."""
if name in self.variants:
raise DuplicateVariantError(
"Cannot specify variant '%s' twice" % name)
if isinstance(value, string_types) and value.upper() == 'TRUE':
value = True
elif isinstance(value, string_types) and value.upper() == 'FALSE':
value = False
self.variants[name] = VariantSpec(name, value)
def _add_flag(self, name, value):
"""Called by the parser to add a known flag.
Known flags currently include "arch"
@@ -1124,7 +1040,13 @@ def _add_flag(self, name, value):
assert(self.compiler_flags is not None)
self.compiler_flags[name] = value.split()
else:
self._add_variant(name, value)
# All other flags represent variants. 'foo=true' and 'foo=false'
# map to '+foo' and '~foo' respectively. As such they need a
# BoolValuedVariant instance.
if str(value).upper() == 'TRUE' or str(value).upper() == 'FALSE':
self.variants[name] = BoolValuedVariant(name, value)
else:
self.variants[name] = MultiValuedVariant(name, value)
def _set_architecture(self, **kwargs):
"""Called by the parser to set the architecture."""
@@ -1424,8 +1346,11 @@ def to_node_dict(self):
if self.namespace:
d['namespace'] = self.namespace
params = syaml_dict(sorted(
(name, v.value) for name, v in self.variants.items()))
params = syaml_dict(
sorted(
v.yaml_entry() for _, v in self.variants.items()
)
)
params.update(sorted(self.compiler_flags.items()))
if params:
d['parameters'] = params
@@ -1491,11 +1416,14 @@ def from_node_dict(node):
if name in _valid_compiler_flags:
spec.compiler_flags[name] = value
else:
spec.variants[name] = VariantSpec(name, value)
spec.variants[name] = MultiValuedVariant.from_node_dict(
name, value
)
elif 'variants' in node:
for name, value in node['variants'].items():
spec.variants[name] = VariantSpec(name, value)
spec.variants[name] = MultiValuedVariant.from_node_dict(
name, value
)
for name in FlagMap.valid_compiler_flags():
spec.compiler_flags[name] = []
@@ -2076,7 +2004,7 @@ def normalize(self, force=False):
self._mark_concrete(False)
# Ensure first that all packages & compilers in the DAG exist.
self.validate_names()
self.validate_or_raise()
# Get all the dependencies into one DependencyMap
spec_deps = self.flat_dependencies(copy=False, deptype_query=alldeps)
@@ -2110,11 +2038,13 @@ def normalized(self):
clone.normalize()
return clone
def validate_names(self):
"""This checks that names of packages and compilers in this spec are real.
If they're not, it will raise either UnknownPackageError or
UnsupportedCompilerError.
def validate_or_raise(self):
"""Checks that names and values in this spec are real. If they're not,
it will raise an appropriate exception.
"""
# FIXME: this function should be lazy, and collect all the errors
# FIXME: before raising the exceptions, instead of being greedy and
# FIXME: raise just the first one encountered
for spec in self.traverse():
# raise an UnknownPackageError if the spec's package isn't real.
if (not spec.virtual) and spec.name:
@@ -2125,16 +2055,44 @@ def validate_names(self):
if not compilers.supported(spec.compiler):
raise UnsupportedCompilerError(spec.compiler.name)
# Ensure that variants all exist.
for vname, variant in spec.variants.items():
if vname not in spec.package_class.variants:
raise UnknownVariantError(spec.name, vname)
# FIXME: Move the logic below into the variant.py module
# Ensure correctness of variants (if the spec is not virtual)
if not spec.virtual:
pkg_cls = spec.package_class
pkg_variants = pkg_cls.variants
not_existing = set(spec.variants) - set(pkg_variants)
if not_existing:
raise UnknownVariantError(spec.name, not_existing)
for name, v in [(x, y) for (x, y) in spec.variants.items()]:
# When parsing a spec every variant of the form
# 'foo=value' will be interpreted by default as a
# multi-valued variant. During validation of the
# variants we use the information in the package
# to turn any variant that needs it to a single-valued
# variant.
pkg_variant = pkg_variants[name]
pkg_variant.validate_or_raise(v, pkg_cls)
spec.variants.substitute(
pkg_variant.make_variant(v._original_value)
)
def constrain(self, other, deps=True):
"""Merge the constraints of other with self.
Returns True if the spec changed as a result, False if not.
"""
# If we are trying to constrain a concrete spec, either the spec
# already satisfies the constraint (and the method returns False)
# or it raises an exception
if self.concrete:
if self.satisfies(other):
return False
else:
raise UnsatisfiableSpecError(
self, other, 'constrain a concrete spec'
)
other = self._autospec(other)
if not (self.name == other.name or
@@ -2150,11 +2108,11 @@ def constrain(self, other, deps=True):
if not self.versions.overlaps(other.versions):
raise UnsatisfiableVersionSpecError(self.versions, other.versions)
for v in other.variants:
if (v in self.variants and
self.variants[v].value != other.variants[v].value):
raise UnsatisfiableVariantSpecError(self.variants[v],
other.variants[v])
for v in [x for x in other.variants if x in self.variants]:
if not self.variants[v].compatible(other.variants[v]):
raise UnsatisfiableVariantSpecError(
self.variants[v], other.variants[v]
)
# TODO: Check out the logic here
sarch, oarch = self.architecture, other.architecture
@@ -2328,6 +2286,30 @@ def satisfies(self, other, deps=True, strict=False, strict_deps=False):
elif strict and (other.compiler and not self.compiler):
return False
# If self is a concrete spec, and other is not virtual, then we need
# to substitute every multi-valued variant that needs it with a
# single-valued variant.
if self.concrete:
for name, v in [(x, y) for (x, y) in other.variants.items()]:
# When parsing a spec every variant of the form
# 'foo=value' will be interpreted by default as a
# multi-valued variant. During validation of the
# variants we use the information in the package
# to turn any variant that needs it to a single-valued
# variant.
pkg_cls = type(other.package)
try:
pkg_variant = other.package.variants[name]
pkg_variant.validate_or_raise(v, pkg_cls)
except (SpecError, KeyError):
# Catch the two things that could go wrong above:
# 1. name is not a valid variant (KeyError)
# 2. the variant is not validated (SpecError)
return False
other.variants.substitute(
pkg_variant.make_variant(v._original_value)
)
var_strict = strict
if (not self.name) or (not other.name):
var_strict = True
@@ -3118,10 +3100,12 @@ def spec(self, name):
added_version = True
elif self.accept(ON):
spec._add_variant(self.variant(), True)
name = self.variant()
spec.variants[name] = BoolValuedVariant(name, True)
elif self.accept(OFF):
spec._add_variant(self.variant(), False)
name = self.variant()
spec.variants[name] = BoolValuedVariant(name, False)
elif self.accept(PCT):
spec._set_compiler(self.compiler())
@@ -3275,10 +3259,6 @@ def base32_prefix_bits(hash_string, bits):
return prefix_bits(hash_bytes, bits)
class SpecError(spack.error.SpackError):
"""Superclass for all errors that occur while constructing specs."""
class SpecParseError(SpecError):
"""Wrapper for ParseError for when we're parsing specs."""
def __init__(self, parse_error):
@@ -3291,10 +3271,6 @@ class DuplicateDependencyError(SpecError):
"""Raised when the same dependency occurs in a spec twice."""
class DuplicateVariantError(SpecError):
"""Raised when the same variant occurs in a spec twice."""
class DuplicateCompilerSpecError(SpecError):
"""Raised when the same compiler occurs in a spec twice."""
@@ -3306,13 +3282,6 @@ def __init__(self, compiler_name):
"The '%s' compiler is not yet supported." % compiler_name)
class UnknownVariantError(SpecError):
"""Raised when the same variant occurs in a spec twice."""
def __init__(self, pkg, variant):
super(UnknownVariantError, self).__init__(
"Package %s has no variant %s!" % (pkg, variant))
class DuplicateArchitectureError(SpecError):
"""Raised when the same architecture occurs in a spec twice."""
@@ -3354,17 +3323,6 @@ def __init__(self, vpkg, providers):
self.providers = providers
class UnsatisfiableSpecError(SpecError):
"""Raised when a spec conflicts with package constraints.
Provide the requirement that was violated when raising."""
def __init__(self, provided, required, constraint_type):
super(UnsatisfiableSpecError, self).__init__(
"%s does not satisfy %s" % (provided, required))
self.provided = provided
self.required = required
self.constraint_type = constraint_type
class UnsatisfiableSpecNameError(UnsatisfiableSpecError):
"""Raised when two specs aren't even for the same package."""
def __init__(self, provided, required):
@@ -3386,13 +3344,6 @@ def __init__(self, provided, required):
provided, required, "compiler")
class UnsatisfiableVariantSpecError(UnsatisfiableSpecError):
"""Raised when a spec variant conflicts with package constraints."""
def __init__(self, provided, required):
super(UnsatisfiableVariantSpecError, self).__init__(
provided, required, "variant")
class UnsatisfiableCompilerFlagSpecError(UnsatisfiableSpecError):
"""Raised when a spec variant conflicts with package constraints."""
def __init__(self, provided, required):

30
lib/spack/spack/test/build_systems.py
View File

@@ -24,6 +24,8 @@
##############################################################################
import spack
import pytest
from spack.build_environment import get_std_cmake_args
from spack.spec import Spec
@@ -40,3 +42,31 @@ def test_cmake_std_args(config, builtin_mock):
s.concretize()
pkg = spack.repo.get(s)
assert get_std_cmake_args(pkg)
@pytest.mark.usefixtures('config', 'builtin_mock')
class TestAutotoolsPackage(object):
def test_with_or_without(self):
s = Spec('a')
s.concretize()
pkg = spack.repo.get(s)
# Called without parameters
l = pkg.with_or_without('foo')
assert '--with-bar' in l
assert '--without-baz' in l
assert '--no-fee' in l
def activate(value):
return 'something'
l = pkg.with_or_without('foo', active_parameters=activate)
assert '--with-bar=something' in l
assert '--without-baz' in l
assert '--no-fee' in l
l = pkg.enable_or_disable('foo')
assert '--enable-bar' in l
assert '--disable-baz' in l
assert '--disable-fee' in l

2
lib/spack/spack/test/concretize.py
View File

@@ -75,7 +75,7 @@ def check_concretize(abstract_spec):
# dag
'callpath', 'mpileaks', 'libelf',
# variant
'mpich+debug', 'mpich~debug', 'mpich debug=2', 'mpich',
'mpich+debug', 'mpich~debug', 'mpich debug=True', 'mpich',
# compiler flags
'mpich cppflags="-O3"',
# with virtual

2
lib/spack/spack/test/conftest.py
View File

@@ -55,7 +55,7 @@
@pytest.fixture(autouse=True)
def no_stdin_duplication(monkeypatch):
"""Duplicating stdin (or any other stream) returns an empty
cStringIO object.
StringIO object.
"""
monkeypatch.setattr(llnl.util.lang, 'duplicate_stream',
lambda x: StringIO())

141
lib/spack/spack/test/spec_semantics.py
View File

@@ -24,7 +24,9 @@
##############################################################################
import spack.architecture
import pytest
from spack.spec import *
from spack.variant import *
def check_satisfies(spec, anon_spec, concrete=False):
@@ -243,6 +245,128 @@ def test_satisfies_matching_variant(self):
check_satisfies('mpich~foo', 'mpich foo=FALSE')
check_satisfies('mpich foo=False', 'mpich~foo')
def test_satisfies_multi_value_variant(self):
# Check quoting
check_satisfies('multivalue_variant foo="bar,baz"',
'multivalue_variant foo="bar,baz"')
check_satisfies('multivalue_variant foo=bar,baz',
'multivalue_variant foo=bar,baz')
check_satisfies('multivalue_variant foo="bar,baz"',
'multivalue_variant foo=bar,baz')
# A more constrained spec satisfies a less constrained one
check_satisfies('multivalue_variant foo="bar,baz"',
'multivalue_variant foo="bar"')
check_satisfies('multivalue_variant foo="bar,baz"',
'multivalue_variant foo="baz"')
check_satisfies('multivalue_variant foo="bar,baz,barbaz"',
'multivalue_variant foo="bar,baz"')
check_satisfies('multivalue_variant foo="bar,baz"',
'foo="bar,baz"')
check_satisfies('multivalue_variant foo="bar,baz"',
'foo="bar"')
def test_satisfies_single_valued_variant(self):
"""Tests that the case reported in
https://github.com/LLNL/spack/pull/2386#issuecomment-282147639
is handled correctly.
"""
a = Spec('a foobar=bar')
a.concretize()
assert a.satisfies('foobar=bar')
def test_unsatisfiable_multi_value_variant(self):
# Semantics for a multi-valued variant is different
# Depending on whether the spec is concrete or not
a = Spec('multivalue_variant foo="bar"', concrete=True)
spec_str = 'multivalue_variant foo="bar,baz"'
b = Spec(spec_str)
assert not a.satisfies(b)
assert not a.satisfies(spec_str)
# A concrete spec cannot be constrained further
with pytest.raises(UnsatisfiableSpecError):
a.constrain(b)
a = Spec('multivalue_variant foo="bar"')
spec_str = 'multivalue_variant foo="bar,baz"'
b = Spec(spec_str)
assert not a.satisfies(b)
assert not a.satisfies(spec_str)
# An abstract spec can instead be constrained
assert a.constrain(b)
a = Spec('multivalue_variant foo="bar,baz"', concrete=True)
spec_str = 'multivalue_variant foo="bar,baz,quux"'
b = Spec(spec_str)
assert not a.satisfies(b)
assert not a.satisfies(spec_str)
# A concrete spec cannot be constrained further
with pytest.raises(UnsatisfiableSpecError):
a.constrain(b)
a = Spec('multivalue_variant foo="bar,baz"')
spec_str = 'multivalue_variant foo="bar,baz,quux"'
b = Spec(spec_str)
assert not a.satisfies(b)
assert not a.satisfies(spec_str)
# An abstract spec can instead be constrained
assert a.constrain(b)
# ...but will fail during concretization if there are
# values in the variant that are not allowed
with pytest.raises(InvalidVariantValueError):
a.concretize()
# This time we'll try to set a single-valued variant
a = Spec('multivalue_variant fee="bar"')
spec_str = 'multivalue_variant fee="baz"'
b = Spec(spec_str)
assert not a.satisfies(b)
assert not a.satisfies(spec_str)
# A variant cannot be parsed as single-valued until we try to
# concretize. This means that we can constrain the variant above
assert a.constrain(b)
# ...but will fail during concretization if there are
# multiple values set
with pytest.raises(MultipleValuesInExclusiveVariantError):
a.concretize()
# FIXME: remove after having checked the correctness of the semantics
# check_unsatisfiable('multivalue_variant foo="bar,baz"',
# 'multivalue_variant foo="bar,baz,quux"',
# concrete=True)
# check_unsatisfiable('multivalue_variant foo="bar,baz"',
# 'multivalue_variant foo="bar,baz,quux"',
# concrete=True)
# but succeed for abstract ones (b/c they COULD satisfy the
# constraint if constrained)
# check_satisfies('multivalue_variant foo="bar"',
# 'multivalue_variant foo="bar,baz"')
# check_satisfies('multivalue_variant foo="bar,baz"',
# 'multivalue_variant foo="bar,baz,quux"')
def test_unsatisfiable_variant_types(self):
# These should fail due to incompatible types
check_unsatisfiable('multivalue_variant +foo',
'multivalue_variant foo="bar"')
check_unsatisfiable('multivalue_variant ~foo',
'multivalue_variant foo="bar"')
check_unsatisfiable('multivalue_variant foo="bar"',
'multivalue_variant +foo')
check_unsatisfiable('multivalue_variant foo="bar"',
'multivalue_variant ~foo')
def test_satisfies_unconstrained_variant(self):
# only asked for mpich, no constraints. Either will do.
check_satisfies('mpich+foo', 'mpich')
@@ -266,7 +390,7 @@ def test_unsatisfiable_variant_mismatch(self):
# No matchi in specs
check_unsatisfiable('mpich~foo', 'mpich+foo')
check_unsatisfiable('mpich+foo', 'mpich~foo')
check_unsatisfiable('mpich foo=1', 'mpich foo=2')
check_unsatisfiable('mpich foo=True', 'mpich foo=False')
def test_satisfies_matching_compiler_flag(self):
check_satisfies('mpich cppflags="-O3"', 'mpich cppflags="-O3"')
@@ -416,6 +540,19 @@ def test_constrain_variants(self):
'libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo'
)
def test_constrain_multi_value_variant(self):
check_constrain(
'multivalue_variant foo="bar,baz"',
'multivalue_variant foo="bar"',
'multivalue_variant foo="baz"'
)
check_constrain(
'multivalue_variant foo="bar,baz,barbaz"',
'multivalue_variant foo="bar,barbaz"',
'multivalue_variant foo="baz"'
)
def test_constrain_compiler_flags(self):
check_constrain(
'libelf cflags="-O3" cppflags="-Wall"',
@@ -455,7 +592,7 @@ def test_invalid_constraint(self):
check_invalid_constraint('libelf+debug', 'libelf~debug')
check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo')
check_invalid_constraint('libelf debug=2', 'libelf debug=1')
check_invalid_constraint('libelf debug=True', 'libelf debug=False')
check_invalid_constraint(
'libelf cppflags="-O3"', 'libelf cppflags="-O2"')

6
lib/spack/spack/test/spec_yaml.py
View File

@@ -74,6 +74,12 @@ def test_concrete_spec(config, builtin_mock):
check_yaml_round_trip(spec)
def test_yaml_multivalue():
spec = Spec('multivalue_variant foo="bar,baz"')
spec.concretize()
check_yaml_round_trip(spec)
def test_yaml_subdag(config, builtin_mock):
spec = Spec('mpileaks^mpich+debug')
spec.concretize()

656
lib/spack/spack/test/variant.py Normal file
View File

@@ -0,0 +1,656 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import pytest
import numbers
from spack.variant import *
class TestMultiValuedVariant(object):
def test_initialization(self):
# Basic properties
a = MultiValuedVariant('foo', 'bar,baz')
assert repr(a) == "MultiValuedVariant('foo', 'bar,baz')"
assert str(a) == 'foo=bar,baz'
assert a.value == ('bar', 'baz')
assert 'bar' in a
assert 'baz' in a
assert eval(repr(a)) == a
# Spaces are trimmed
b = MultiValuedVariant('foo', 'bar, baz')
assert repr(b) == "MultiValuedVariant('foo', 'bar, baz')"
assert str(b) == 'foo=bar,baz'
assert b.value == ('bar', 'baz')
assert 'bar' in b
assert 'baz' in b
assert a == b
assert hash(a) == hash(b)
assert eval(repr(b)) == a
# Order is not important
c = MultiValuedVariant('foo', 'baz, bar')
assert repr(c) == "MultiValuedVariant('foo', 'baz, bar')"
assert str(c) == 'foo=bar,baz'
assert c.value == ('bar', 'baz')
assert 'bar' in c
assert 'baz' in c
assert a == c
assert hash(a) == hash(c)
assert eval(repr(c)) == a
# Check the copy
d = a.copy()
assert repr(a) == repr(d)
assert str(a) == str(d)
assert d.value == ('bar', 'baz')
assert 'bar' in d
assert 'baz' in d
assert a == d
assert a is not d
assert hash(a) == hash(d)
assert eval(repr(d)) == a
def test_satisfies(self):
a = MultiValuedVariant('foo', 'bar,baz')
b = MultiValuedVariant('foo', 'bar')
c = MultiValuedVariant('fee', 'bar,baz')
d = MultiValuedVariant('foo', 'True')
# 'foo=bar,baz' satisfies 'foo=bar'
assert a.satisfies(b)
# 'foo=bar' does not satisfy 'foo=bar,baz'
assert not b.satisfies(a)
# 'foo=bar,baz' does not satisfy 'foo=bar,baz' and vice-versa
assert not a.satisfies(c)
assert not c.satisfies(a)
# Cannot satisfy the constraint with an object of
# another type
b_sv = SingleValuedVariant('foo', 'bar')
assert not b.satisfies(b_sv)
d_bv = BoolValuedVariant('foo', 'True')
assert not d.satisfies(d_bv)
def test_compatible(self):
a = MultiValuedVariant('foo', 'bar,baz')
b = MultiValuedVariant('foo', 'True')
c = MultiValuedVariant('fee', 'bar,baz')
d = MultiValuedVariant('foo', 'bar,barbaz')
# If the name of two multi-valued variants is the same,
# they are compatible
assert a.compatible(b)
assert not a.compatible(c)
assert a.compatible(d)
assert b.compatible(a)
assert not b.compatible(c)
assert b.compatible(d)
assert not c.compatible(a)
assert not c.compatible(b)
assert not c.compatible(d)
assert d.compatible(a)
assert d.compatible(b)
assert not d.compatible(c)
# Can't be compatible with other types
b_bv = BoolValuedVariant('foo', 'True')
assert not b.compatible(b_bv)
b_sv = SingleValuedVariant('foo', 'True')
assert not b.compatible(b_sv)
def test_constrain(self):
# Try to constrain on a value with less constraints than self
a = MultiValuedVariant('foo', 'bar,baz')
b = MultiValuedVariant('foo', 'bar')
changed = a.constrain(b)
assert not changed
t = MultiValuedVariant('foo', 'bar,baz')
assert a == t
# Try to constrain on a value with more constraints than self
a = MultiValuedVariant('foo', 'bar,baz')
b = MultiValuedVariant('foo', 'bar')
changed = b.constrain(a)
assert changed
t = MultiValuedVariant('foo', 'bar,baz')
assert a == t
# Try to constrain on the same value
a = MultiValuedVariant('foo', 'bar,baz')
b = a.copy()
changed = a.constrain(b)
assert not changed
t = MultiValuedVariant('foo', 'bar,baz')
assert a == t
# Try to constrain on a different name
a = MultiValuedVariant('foo', 'bar,baz')
b = MultiValuedVariant('fee', 'bar')
with pytest.raises(ValueError):
a.constrain(b)
# Try to constrain on other types
a = MultiValuedVariant('foo', 'bar,baz')
sv = SingleValuedVariant('foo', 'bar')
bv = BoolValuedVariant('foo', 'True')
for v in (sv, bv):
with pytest.raises(TypeError):
a.constrain(v)
def test_yaml_entry(self):
a = MultiValuedVariant('foo', 'bar,baz,barbaz')
b = MultiValuedVariant('foo', 'bar, baz, barbaz')
expected = ('foo', sorted(['bar', 'baz', 'barbaz']))
assert a.yaml_entry() == expected
assert b.yaml_entry() == expected
a = MultiValuedVariant('foo', 'bar')
expected = ('foo', sorted(['bar']))
assert a.yaml_entry() == expected
class TestSingleValuedVariant(object):
def test_initialization(self):
# Basic properties
a = SingleValuedVariant('foo', 'bar')
assert repr(a) == "SingleValuedVariant('foo', 'bar')"
assert str(a) == 'foo=bar'
assert a.value == 'bar'
assert 'bar' in a
assert eval(repr(a)) == a
# Raise if multiple values are passed
with pytest.raises(ValueError):
SingleValuedVariant('foo', 'bar, baz')
# Check the copy
b = a.copy()
assert repr(a) == repr(b)
assert str(a) == str(b)
assert b.value == 'bar'
assert 'bar' in b
assert a == b
assert a is not b
assert hash(a) == hash(b)
assert eval(repr(b)) == a
def test_satisfies(self):
a = SingleValuedVariant('foo', 'bar')
b = SingleValuedVariant('foo', 'bar')
c = SingleValuedVariant('foo', 'baz')
d = SingleValuedVariant('fee', 'bar')
e = SingleValuedVariant('foo', 'True')
# 'foo=bar' can only satisfy 'foo=bar'
assert a.satisfies(b)
assert not a.satisfies(c)
assert not a.satisfies(d)
assert b.satisfies(a)
assert not b.satisfies(c)
assert not b.satisfies(d)
assert not c.satisfies(a)
assert not c.satisfies(b)
assert not c.satisfies(d)
# Cannot satisfy the constraint with an object of
# another type
a_mv = MultiValuedVariant('foo', 'bar')
assert not a.satisfies(a_mv)
e_bv = BoolValuedVariant('foo', 'True')
assert not e.satisfies(e_bv)
def test_compatible(self):
a = SingleValuedVariant('foo', 'bar')
b = SingleValuedVariant('fee', 'bar')
c = SingleValuedVariant('foo', 'baz')
d = SingleValuedVariant('foo', 'bar')
# If the name of two multi-valued variants is the same,
# they are compatible
assert not a.compatible(b)
assert not a.compatible(c)
assert a.compatible(d)
assert not b.compatible(a)
assert not b.compatible(c)
assert not b.compatible(d)
assert not c.compatible(a)
assert not c.compatible(b)
assert not c.compatible(d)
assert d.compatible(a)
assert not d.compatible(b)
assert not d.compatible(c)
# Can't be compatible with other types
a_mv = MultiValuedVariant('foo', 'bar')
assert not a.compatible(a_mv)
e = SingleValuedVariant('foo', 'True')
e_bv = BoolValuedVariant('foo', 'True')
assert not e.compatible(e_bv)
def test_constrain(self):
# Try to constrain on a value equal to self
a = SingleValuedVariant('foo', 'bar')
b = SingleValuedVariant('foo', 'bar')
changed = a.constrain(b)
assert not changed
t = SingleValuedVariant('foo', 'bar')
assert a == t
# Try to constrain on a value with a different value
a = SingleValuedVariant('foo', 'bar')
b = SingleValuedVariant('foo', 'baz')
with pytest.raises(UnsatisfiableVariantSpecError):
b.constrain(a)
# Try to constrain on a value with a different value
a = SingleValuedVariant('foo', 'bar')
b = SingleValuedVariant('fee', 'bar')
with pytest.raises(ValueError):
b.constrain(a)
# Try to constrain on the same value
a = SingleValuedVariant('foo', 'bar')
b = a.copy()
changed = a.constrain(b)
assert not changed
t = SingleValuedVariant('foo', 'bar')
assert a == t
# Try to constrain on other values
a = SingleValuedVariant('foo', 'True')
mv = MultiValuedVariant('foo', 'True')
bv = BoolValuedVariant('foo', 'True')
for v in (mv, bv):
with pytest.raises(TypeError):
a.constrain(v)
def test_yaml_entry(self):
a = SingleValuedVariant('foo', 'bar')
expected = ('foo', 'bar')
assert a.yaml_entry() == expected
class TestBoolValuedVariant(object):
def test_initialization(self):
# Basic properties - True value
for v in (True, 'True', 'TRUE', 'TrUe'):
a = BoolValuedVariant('foo', v)
assert repr(a) == "BoolValuedVariant('foo', {0})".format(repr(v))
assert str(a) == '+foo'
assert a.value is True
assert True in a
assert eval(repr(a)) == a
# Copy - True value
b = a.copy()
assert repr(a) == repr(b)
assert str(a) == str(b)
assert b.value is True
assert True in b
assert a == b
assert a is not b
assert hash(a) == hash(b)
assert eval(repr(b)) == a
# Basic properties - False value
for v in (False, 'False', 'FALSE', 'FaLsE'):
a = BoolValuedVariant('foo', v)
assert repr(a) == "BoolValuedVariant('foo', {0})".format(repr(v))
assert str(a) == '~foo'
assert a.value is False
assert False in a
assert eval(repr(a)) == a
# Copy - True value
b = a.copy()
assert repr(a) == repr(b)
assert str(a) == str(b)
assert b.value is False
assert False in b
assert a == b
assert a is not b
assert eval(repr(b)) == a
# Invalid values
for v in ('bar', 'bar,baz'):
with pytest.raises(ValueError):
BoolValuedVariant('foo', v)
def test_satisfies(self):
a = BoolValuedVariant('foo', True)
b = BoolValuedVariant('foo', False)
c = BoolValuedVariant('fee', False)
d = BoolValuedVariant('foo', 'True')
assert not a.satisfies(b)
assert not a.satisfies(c)
assert a.satisfies(d)
assert not b.satisfies(a)
assert not b.satisfies(c)
assert not b.satisfies(d)
assert not c.satisfies(a)
assert not c.satisfies(b)
assert not c.satisfies(d)
assert d.satisfies(a)
assert not d.satisfies(b)
assert not d.satisfies(c)
# Cannot satisfy the constraint with an object of
# another type
d_mv = MultiValuedVariant('foo', 'True')
assert not d.satisfies(d_mv)
d_sv = SingleValuedVariant('foo', 'True')
assert not d.satisfies(d_sv)
def test_compatible(self):
a = BoolValuedVariant('foo', True)
b = BoolValuedVariant('fee', True)
c = BoolValuedVariant('foo', False)
d = BoolValuedVariant('foo', 'True')
# If the name of two multi-valued variants is the same,
# they are compatible
assert not a.compatible(b)
assert not a.compatible(c)
assert a.compatible(d)
assert not b.compatible(a)
assert not b.compatible(c)
assert not b.compatible(d)
assert not c.compatible(a)
assert not c.compatible(b)
assert not c.compatible(d)
assert d.compatible(a)
assert not d.compatible(b)
assert not d.compatible(c)
# Can't be compatible with other types
d_mv = MultiValuedVariant('foo', 'True')
assert not d.compatible(d_mv)
d_sv = SingleValuedVariant('foo', 'True')
assert not d.compatible(d_sv)
def test_constrain(self):
# Try to constrain on a value equal to self
a = BoolValuedVariant('foo', 'True')
b = BoolValuedVariant('foo', True)
changed = a.constrain(b)
assert not changed
t = BoolValuedVariant('foo', True)
assert a == t
# Try to constrain on a value with a different value
a = BoolValuedVariant('foo', True)
b = BoolValuedVariant('foo', False)
with pytest.raises(UnsatisfiableVariantSpecError):
b.constrain(a)
# Try to constrain on a value with a different value
a = BoolValuedVariant('foo', True)
b = BoolValuedVariant('fee', True)
with pytest.raises(ValueError):
b.constrain(a)
# Try to constrain on the same value
a = BoolValuedVariant('foo', True)
b = a.copy()
changed = a.constrain(b)
assert not changed
t = BoolValuedVariant('foo', True)
assert a == t
# Try to constrain on other values
a = BoolValuedVariant('foo', 'True')
sv = SingleValuedVariant('foo', 'True')
mv = MultiValuedVariant('foo', 'True')
for v in (sv, mv):
with pytest.raises(TypeError):
a.constrain(v)
def test_yaml_entry(self):
a = BoolValuedVariant('foo', 'True')
expected = ('foo', True)
assert a.yaml_entry() == expected
a = BoolValuedVariant('foo', 'False')
expected = ('foo', False)
assert a.yaml_entry() == expected
def test_from_node_dict():
a = MultiValuedVariant.from_node_dict('foo', ['bar'])
assert type(a) == MultiValuedVariant
a = MultiValuedVariant.from_node_dict('foo', 'bar')
assert type(a) == SingleValuedVariant
a = MultiValuedVariant.from_node_dict('foo', 'true')
assert type(a) == BoolValuedVariant
class TestVariant(object):
def test_validation(self):
a = Variant(
'foo',
default='',
description='',
values=('bar', 'baz', 'foobar'),
multi=False
)
# Valid vspec, shouldn't raise
vspec = a.make_variant('bar')
a.validate_or_raise(vspec)
# Multiple values are not allowed
with pytest.raises(MultipleValuesInExclusiveVariantError):
vspec.value = 'bar,baz'
# Inconsistent vspec
vspec.name = 'FOO'
with pytest.raises(InconsistentValidationError):
a.validate_or_raise(vspec)
# Valid multi-value vspec
a.multi = True
vspec = a.make_variant('bar,baz')
a.validate_or_raise(vspec)
# Add an invalid value
vspec.value = 'bar,baz,barbaz'
with pytest.raises(InvalidVariantValueError):
a.validate_or_raise(vspec)
def test_callable_validator(self):
def validator(x):
try:
return isinstance(int(x), numbers.Integral)
except ValueError:
return False
a = Variant(
'foo',
default=1024,
description='',
values=validator,
multi=False
)
vspec = a.make_default()
a.validate_or_raise(vspec)
vspec.value = 2056
a.validate_or_raise(vspec)
vspec.value = 'foo'
with pytest.raises(InvalidVariantValueError):
a.validate_or_raise(vspec)
def test_representation(self):
a = Variant(
'foo',
default='',
description='',
values=('bar', 'baz', 'foobar'),
multi=False
)
assert a.allowed_values == 'bar, baz, foobar'
class TestVariantMapTest(object):
def test_invalid_values(self):
# Value with invalid type
a = VariantMap(None)
with pytest.raises(TypeError):
a['foo'] = 2
# Duplicate variant
a['foo'] = MultiValuedVariant('foo', 'bar,baz')
with pytest.raises(DuplicateVariantError):
a['foo'] = MultiValuedVariant('foo', 'bar')
with pytest.raises(DuplicateVariantError):
a['foo'] = SingleValuedVariant('foo', 'bar')
with pytest.raises(DuplicateVariantError):
a['foo'] = BoolValuedVariant('foo', True)
# Non matching names between key and vspec.name
with pytest.raises(KeyError):
a['bar'] = MultiValuedVariant('foo', 'bar')
def test_set_item(self):
# Check that all the three types of variants are accepted
a = VariantMap(None)
a['foo'] = BoolValuedVariant('foo', True)
a['bar'] = SingleValuedVariant('bar', 'baz')
a['foobar'] = MultiValuedVariant('foobar', 'a, b, c, d, e')
def test_substitute(self):
# Check substitution of a key that exists
a = VariantMap(None)
a['foo'] = BoolValuedVariant('foo', True)
a.substitute(SingleValuedVariant('foo', 'bar'))
# Trying to substitute something that is not
# in the map will raise a KeyError
with pytest.raises(KeyError):
a.substitute(BoolValuedVariant('bar', True))
def test_satisfies_and_constrain(self):
# foo=bar foobar=fee feebar=foo
a = VariantMap(None)
a['foo'] = MultiValuedVariant('foo', 'bar')
a['foobar'] = SingleValuedVariant('foobar', 'fee')
a['feebar'] = SingleValuedVariant('feebar', 'foo')
# foo=bar,baz foobar=fee shared=True
b = VariantMap(None)
b['foo'] = MultiValuedVariant('foo', 'bar, baz')
b['foobar'] = SingleValuedVariant('foobar', 'fee')
b['shared'] = BoolValuedVariant('shared', True)
assert not a.satisfies(b)
assert b.satisfies(a)
assert not a.satisfies(b, strict=True)
assert not b.satisfies(a, strict=True)
# foo=bar,baz foobar=fee feebar=foo shared=True
c = VariantMap(None)
c['foo'] = MultiValuedVariant('foo', 'bar, baz')
c['foobar'] = SingleValuedVariant('foobar', 'fee')
c['feebar'] = SingleValuedVariant('feebar', 'foo')
c['shared'] = BoolValuedVariant('shared', True)
assert a.constrain(b)
assert a == c
def test_copy(self):
a = VariantMap(None)
a['foo'] = BoolValuedVariant('foo', True)
a['bar'] = SingleValuedVariant('bar', 'baz')
a['foobar'] = MultiValuedVariant('foobar', 'a, b, c, d, e')
c = a.copy()
assert a == c
def test_str(self):
c = VariantMap(None)
c['foo'] = MultiValuedVariant('foo', 'bar, baz')
c['foobar'] = SingleValuedVariant('foobar', 'fee')
c['feebar'] = SingleValuedVariant('feebar', 'foo')
c['shared'] = BoolValuedVariant('shared', True)
assert str(c) == ' feebar=foo foo=bar,baz foobar=fee +shared'

582
lib/spack/spack/variant.py
View File

@@ -22,17 +22,583 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Variant is a class describing flags on builds, or "variants".
Could be generalized later to describe aribitrary parameters, but
currently variants are just flags.
"""The variant module contains data structures that are needed to manage
variants both in packages and in specs.
"""
import inspect
import re
import llnl.util.lang as lang
import spack.error as error
from six import StringIO
class Variant(object):
"""Represents a variant on a build. Can be either on or off."""
"""Represents a variant in a package, as declared in the
variant directive.
"""
def __init__(self, default, description):
self.default = default
def __init__(
self,
name,
default,
description,
values=(True, False),
multi=False,
validator=None
):
"""Initialize a package variant.
Args:
name (str): name of the variant
default (str): default value for the variant in case
nothing has been specified
description (str): purpose of the variant
values (sequence): sequence of allowed values or a callable
accepting a single value as argument and returning True if the
value is good, False otherwise
multi (bool): whether multiple CSV are allowed
validator (callable): optional callable used to enforce
additional logic on the set of values being validated
"""
self.name = name
self.default = default
self.description = str(description)
if callable(values):
# If 'values' is a callable, assume it is a single value
# validator and reset the values to be explicit during debug
self.single_value_validator = values
self.values = None
else:
# Otherwise assume values is the set of allowed explicit values
self.values = tuple(values)
allowed = self.values + (self.default,)
self.single_value_validator = lambda x: x in allowed
self.multi = multi
self.group_validator = validator
def validate_or_raise(self, vspec, pkg=None):
"""Validate a variant spec against this package variant. Raises an
exception if any error is found.
Args:
vspec (VariantSpec): instance to be validated
pkg (Package): the package that required the validation,
if available
Raises:
InconsistentValidationError: if ``vspec.name != self.name``
MultipleValuesInExclusiveVariantError: if ``vspec`` has
multiple values but ``self.multi == False``
InvalidVariantValueError: if ``vspec.value`` contains
invalid values
"""
# Check the name of the variant
if self.name != vspec.name:
raise InconsistentValidationError(vspec, self)
# Check the values of the variant spec
value = vspec.value
if isinstance(vspec.value, (bool, str)):
value = (vspec.value,)
# If the value is exclusive there must be at most one
if not self.multi and len(value) != 1:
raise MultipleValuesInExclusiveVariantError(vspec, pkg)
# Check and record the values that are not allowed
not_allowed_values = [
x for x in value if not self.single_value_validator(x)
]
if not_allowed_values:
raise InvalidVariantValueError(self, not_allowed_values, pkg)
# Validate the group of values if needed
if self.group_validator is not None:
self.group_validator(value)
@property
def allowed_values(self):
"""Returns a string representation of the allowed values for
printing purposes
Returns:
str: representation of the allowed values
"""
# Join an explicit set of allowed values
if self.values is not None:
v = tuple(str(x) for x in self.values)
return ', '.join(v)
# In case we were given a single-value validator
# print the docstring
docstring = inspect.getdoc(self.single_value_validator)
v = docstring if docstring else ''
return v
def make_default(self):
"""Factory that creates a variant holding the default value.
Returns:
MultiValuedVariant or SingleValuedVariant or BoolValuedVariant:
instance of the proper variant
"""
return self.make_variant(self.default)
def make_variant(self, value):
"""Factory that creates a variant holding the value passed as
a parameter.
Args:
value: value that will be hold by the variant
Returns:
MultiValuedVariant or SingleValuedVariant or BoolValuedVariant:
instance of the proper variant
"""
return self.variant_cls(self.name, value)
@property
def variant_cls(self):
"""Proper variant class to be used for this configuration."""
if self.multi:
return MultiValuedVariant
elif self.values == (True, False):
return BoolValuedVariant
return SingleValuedVariant
@lang.key_ordering
class MultiValuedVariant(object):
"""A variant that can hold multiple values at once."""
@staticmethod
def from_node_dict(name, value):
"""Reconstruct a variant from a node dict."""
if isinstance(value, list):
value = ','.join(value)
return MultiValuedVariant(name, value)
elif str(value).upper() == 'TRUE' or str(value).upper() == 'FALSE':
return BoolValuedVariant(name, value)
return SingleValuedVariant(name, value)
def __init__(self, name, value):
self.name = name
# Stores 'value' after a bit of massaging
# done by the property setter
self._value = None
self._original_value = None
# Invokes property setter
self.value = value
@property
def value(self):
"""Returns a tuple of strings containing the values stored in
the variant.
Returns:
tuple of str: values stored in the variant
"""
return self._value
@value.setter
def value(self, value):
self._value_setter(value)
def _value_setter(self, value):
# Store the original value
self._original_value = value
# Store a tuple of CSV string representations
# Tuple is necessary here instead of list because the
# values need to be hashed
t = re.split(r'\s*,\s*', str(value))
# With multi-value variants it is necessary
# to remove duplicates and give an order
# to a set
self._value = tuple(sorted(set(t)))
def _cmp_key(self):
return self.name, self.value
def copy(self):
"""Returns an instance of a variant equivalent to self
Returns:
any variant type: a copy of self
>>> a = MultiValuedVariant('foo', True)
>>> b = a.copy()
>>> assert a == b
>>> assert a is not b
"""
return type(self)(self.name, self._original_value)
def satisfies(self, other):
"""Returns true if ``other.name == self.name`` and ``other.value`` is
a strict subset of self. Does not try to validate.
Args:
other: constraint to be met for the method to return True
Returns:
bool: True or False
"""
# If types are different the constraint is not satisfied
if type(other) != type(self):
return False
# If names are different then `self` does not satisfy `other`
# (`foo=bar` does not satisfy `baz=bar`)
if other.name != self.name:
return False
# Otherwise we want all the values in `other` to be also in `self`
return all(v in self.value for v in other.value)
def compatible(self, other):
"""Returns True if self and other are compatible, False otherwise.
As there is no semantic check, two VariantSpec are compatible if
either they contain the same value or they are both multi-valued.
Args:
other: instance against which we test compatibility
Returns:
bool: True or False
"""
# If types are different they are not compatible
if type(other) != type(self):
return False
# If names are different then they are not compatible
return other.name == self.name
def constrain(self, other):
"""Modify self to match all the constraints for other if both
instances are multi-valued. Returns True if self changed,
False otherwise.
Args:
other: instance against which we constrain self
Returns:
bool: True or False
"""
# If types are different they are not compatible
if type(other) != type(self):
msg = 'other must be of type \'{0.__name__}\''
raise TypeError(msg.format(type(self)))
if self.name != other.name:
raise ValueError('variants must have the same name')
old_value = self.value
self.value = ','.join(sorted(set(self.value + other.value)))
return old_value != self.value
def yaml_entry(self):
"""Returns a key, value tuple suitable to be an entry in a yaml dict.
Returns:
tuple: (name, value_representation)
"""
return self.name, list(self.value)
def __contains__(self, item):
return item in self._value
def __repr__(self):
cls = type(self)
return '{0.__name__}({1}, {2})'.format(
cls, repr(self.name), repr(self._original_value)
)
def __str__(self):
return '{0}={1}'.format(
self.name, ','.join(str(x) for x in self.value)
)
class SingleValuedVariant(MultiValuedVariant):
"""A variant that can hold multiple values, but one at a time."""
def _value_setter(self, value):
# Treat the value as a multi-valued variant
super(SingleValuedVariant, self)._value_setter(value)
# Then check if there's only a single value
if len(self._value) != 1:
raise MultipleValuesInExclusiveVariantError(self, None)
self._value = str(self._value[0])
def __str__(self):
return '{0}={1}'.format(self.name, self.value)
def satisfies(self, other):
# If types are different the constraint is not satisfied
if type(other) != type(self):
return False
# If names are different then `self` does not satisfy `other`
# (`foo=bar` does not satisfy `baz=bar`)
if other.name != self.name:
return False
return self.value == other.value
def compatible(self, other):
return self.satisfies(other)
def constrain(self, other):
if type(other) != type(self):
msg = 'other must be of type \'{0.__name__}\''
raise TypeError(msg.format(type(self)))
if self.name != other.name:
raise ValueError('variants must have the same name')
if self.value != other.value:
raise UnsatisfiableVariantSpecError(other.value, self.value)
return False
def __contains__(self, item):
return item == self.value
def yaml_entry(self):
return self.name, self.value
class BoolValuedVariant(SingleValuedVariant):
"""A variant that can hold either True or False."""
def _value_setter(self, value):
# Check the string representation of the value and turn
# it to a boolean
if str(value).upper() == 'TRUE':
self._original_value = value
self._value = True
elif str(value).upper() == 'FALSE':
self._original_value = value
self._value = False
else:
msg = 'cannot construct a BoolValuedVariant from '
msg += 'a value that does not represent a bool'
raise ValueError(msg)
def __contains__(self, item):
return item is self.value
def __str__(self):
return '{0}{1}'.format('+' if self.value else '~', self.name)
class VariantMap(lang.HashableMap):
"""Map containing variant instances. New values can be added only
if the key is not already present.
"""
def __init__(self, spec):
super(VariantMap, self).__init__()
self.spec = spec
def __setitem__(self, name, vspec):
# Raise a TypeError if vspec is not of the right type
if not isinstance(vspec, MultiValuedVariant):
msg = 'VariantMap accepts only values of type VariantSpec'
raise TypeError(msg)
# Raise an error if the variant was already in this map
if name in self.dict:
msg = 'Cannot specify variant "{0}" twice'.format(name)
raise DuplicateVariantError(msg)
# Raise an error if name and vspec.name don't match
if name != vspec.name:
msg = 'Inconsistent key "{0}", must be "{1}" to match VariantSpec'
raise KeyError(msg.format(name, vspec.name))
# Set the item
super(VariantMap, self).__setitem__(name, vspec)
def substitute(self, vspec):
"""Substitutes the entry under ``vspec.name`` with ``vspec``.
Args:
vspec: variant spec to be substituted
"""
if vspec.name not in self:
msg = 'cannot substitute a key that does not exist [{0}]'
raise KeyError(msg.format(vspec.name))
# Set the item
super(VariantMap, self).__setitem__(vspec.name, vspec)
def satisfies(self, other, strict=False):
"""Returns True if this VariantMap is more constrained than other,
False otherwise.
Args:
other (VariantMap): VariantMap instance to satisfy
strict (bool): if True return False if a key is in other and
not in self, otherwise discard that key and proceed with
evaluation
Returns:
bool: True or False
"""
to_be_checked = [k for k in other]
strict_or_concrete = strict
if self.spec is not None:
strict_or_concrete |= self.spec._concrete
if not strict_or_concrete:
to_be_checked = filter(lambda x: x in self, to_be_checked)
return all(k in self and self[k].satisfies(other[k])
for k in to_be_checked)
def constrain(self, other):
"""Add all variants in other that aren't in self to self. Also
constrain all multi-valued variants that are already present.
Return True if self changed, False otherwise
Args:
other (VariantMap): instance against which we constrain self
Returns:
bool: True or False
"""
if other.spec is not None and other.spec._concrete:
for k in self:
if k not in other:
raise UnsatisfiableVariantSpecError(self[k], '<absent>')
changed = False
for k in other:
if k in self:
# If they are not compatible raise an error
if not self[k].compatible(other[k]):
raise UnsatisfiableVariantSpecError(self[k], other[k])
# If they are compatible merge them
changed |= self[k].constrain(other[k])
else:
# If it is not present copy it straight away
self[k] = other[k].copy()
changed = True
return changed
@property
def concrete(self):
"""Returns True if the spec is concrete in terms of variants.
Returns:
bool: True or False
"""
return self.spec._concrete or all(
v in self for v in self.spec.package_class.variants
)
def copy(self):
"""Return an instance of VariantMap equivalent to self.
Returns:
VariantMap: a copy of self
"""
clone = VariantMap(self.spec)
for name, variant in self.items():
clone[name] = variant.copy()
return clone
def __str__(self):
# print keys in order
sorted_keys = sorted(self.keys())
# add spaces before and after key/value variants.
string = StringIO()
kv = False
for key in sorted_keys:
vspec = self[key]
if not isinstance(vspec.value, bool):
# add space before all kv pairs.
string.write(' ')
kv = True
else:
# not a kv pair this time
if kv:
# if it was LAST time, then pad after.
string.write(' ')
kv = False
string.write(str(vspec))
return string.getvalue()
class DuplicateVariantError(error.SpecError):
"""Raised when the same variant occurs in a spec twice."""
class UnknownVariantError(error.SpecError):
"""Raised when an unknown variant occurs in a spec."""
def __init__(self, pkg, variant):
super(UnknownVariantError, self).__init__(
'Package {0} has no variant {1}!'.format(pkg, variant)
)
class InconsistentValidationError(error.SpecError):
"""Raised if the wrong validator is used to validate a variant."""
def __init__(self, vspec, variant):
msg = ('trying to validate variant "{0.name}" '
'with the validator of "{1.name}"')
super(InconsistentValidationError, self).__init__(
msg.format(vspec, variant)
)
class MultipleValuesInExclusiveVariantError(error.SpecError, ValueError):
"""Raised when multiple values are present in a variant that wants
only one.
"""
def __init__(self, variant, pkg):
msg = 'multiple values are not allowed for variant "{0.name}"{1}'
pkg_info = ''
if pkg is not None:
pkg_info = ' in package "{0}"'.format(pkg.name)
super(MultipleValuesInExclusiveVariantError, self).__init__(
msg.format(variant, pkg_info)
)
class InvalidVariantValueError(error.SpecError):
"""Raised when a valid variant has at least an invalid value."""
def __init__(self, variant, invalid_values, pkg):
msg = 'invalid values for variant "{0.name}"{2}: {1}\n'
pkg_info = ''
if pkg is not None:
pkg_info = ' in package "{0}"'.format(pkg.name)
super(InvalidVariantValueError, self).__init__(
msg.format(variant, invalid_values, pkg_info)
)
class UnsatisfiableVariantSpecError(error.UnsatisfiableSpecError):
"""Raised when a spec variant conflicts with package constraints."""
def __init__(self, provided, required):
super(UnsatisfiableVariantSpecError, self).__init__(
provided, required, "variant")

32
var/spack/repos/builtin.mock/packages/a/package.py
View File

@@ -25,7 +25,7 @@
from spack import *
class A(Package):
class A(AutotoolsPackage):
"""Simple package with no dependencies"""
homepage = "http://www.example.com"
@@ -33,5 +33,35 @@ class A(Package):
version('1.0', '0123456789abcdef0123456789abcdef')
variant(
'foo',
values=('bar', 'baz', 'fee'),
default='bar',
description='',
multi=True
)
variant(
'foobar',
values=('bar', 'baz', 'fee'),
default='bar',
description='',
multi=False
)
def with_or_without_fee(self, activated):
if not activated:
return '--no-fee'
return '--fee-all-the-time'
def autoreconf(self, spec, prefix):
pass
def configure(self, spec, prefix):
pass
def build(self, spec, prefix):
pass
def install(self, spec, prefix):
pass

57
var/spack/repos/builtin.mock/packages/multivalue_variant/package.py Normal file
View File

@@ -0,0 +1,57 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class MultivalueVariant(Package):
homepage = "http://www.llnl.gov"
url = "http://www.llnl.gov/mpileaks-1.0.tar.gz"
version(1.0, 'foobarbaz')
version(2.1, 'foobarbaz')
version(2.2, 'foobarbaz')
version(2.3, 'foobarbaz')
variant('debug', default=False, description='Debug variant')
variant(
'foo',
description='Multi-valued variant',
values=('bar', 'baz', 'barbaz'),
multi=True
)
variant(
'fee',
description='Single-valued variant',
default='bar',
values=('bar', 'baz', 'barbaz'),
multi=False
)
depends_on('mpi')
depends_on('callpath')
def install(self, spec, prefix):
pass

3
var/spack/repos/builtin/packages/cdo/package.py
View File

@@ -27,7 +27,8 @@
class Cdo(Package):
"""CDO is a collection of command line Operators to manipulate and analyse
Climate and NWP model Data. """
Climate and NWP model Data.
"""
homepage = "https://code.zmaw.de/projects/cdo"
url = "https://code.zmaw.de/attachments/download/12760/cdo-1.7.2.tar.gz"

274
var/spack/repos/builtin/packages/mvapich2/package.py
View File

@@ -22,11 +22,20 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import sys
from spack import *
from spack.error import SpackError
class Mvapich2(Package):
def _process_manager_validator(values):
if len(values) > 1 and 'slurm' in values:
raise SpackError(
'slurm cannot be activated along with other process managers'
)
class Mvapich2(AutotoolsPackage):
"""MVAPICH2 is an MPI implementation for Infiniband networks."""
homepage = "http://mvapich.cse.ohio-state.edu/"
url = "http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.2.tar.gz"
@@ -55,62 +64,39 @@ class Mvapich2(Package):
# serialized - User serializes calls to MPI (MPI_THREAD_SERIALIZED)
# multiple - Fully multi-threaded (MPI_THREAD_MULTIPLE)
# runtime - Alias to "multiple"
variant('threads', default='multiple',
description='Control the level of thread support')
variant(
'threads',
default='multiple',
values=('single', 'funneled', 'serialized', 'multiple'),
multi=False,
description='Control the level of thread support'
)
# 32 is needed when job size exceeds 32768 cores
variant('ch3_rank_bits', default=32,
description='Number of bits allocated to the rank field (16 or 32)'
)
variant(
'ch3_rank_bits',
default='32',
values=('16', '32'),
multi=False,
description='Number of bits allocated to the rank field (16 or 32)'
)
##########
# TODO : Process managers should be grouped into the same variant,
# as soon as variant capabilities will be extended See
# https://groups.google.com/forum/#!topic/spack/F8-f8B4_0so
SLURM = 'slurm'
HYDRA = 'hydra'
GFORKER = 'gforker'
REMSHELL = 'remshell'
SLURM_INCOMPATIBLE_PMS = (HYDRA, GFORKER, REMSHELL)
variant(SLURM, default=False,
description='Set slurm as the only process manager')
variant(HYDRA, default=False,
description='Set hydra as one of the process managers')
variant(GFORKER, default=False,
description='Set gforker as one of the process managers')
variant(REMSHELL, default=False,
description='Set remshell as one of the process managers')
##########
variant(
'process_managers',
description='List of the process managers to activate',
values=('slurm', 'hydra', 'gforker', 'remshell'),
multi=True,
validator=_process_manager_validator
)
##########
# TODO : Network types should be grouped into the same variant, as
# soon as variant capabilities will be extended
PSM = 'psm'
SOCK = 'sock'
NEMESISIBTCP = 'nemesisibtcp'
NEMESISIB = 'nemesisib'
NEMESIS = 'nemesis'
MRAIL = 'mrail'
SUPPORTED_NETWORKS = (PSM, SOCK, NEMESIS, NEMESISIB, NEMESISIBTCP)
variant(
PSM, default=False,
description='Configure for QLogic PSM-CH3')
variant(
SOCK, default=False,
description='Configure for TCP/IP-CH3')
variant(
NEMESISIBTCP, default=False,
description='Configure for both OFA-IB-Nemesis and TCP/IP-Nemesis')
variant(
NEMESISIB, default=False,
description='Configure for OFA-IB-Nemesis')
variant(
NEMESIS, default=False,
description='Configure for TCP/IP-Nemesis')
variant(
MRAIL, default=False,
description='Configure for OFA-IB-CH3')
##########
'fabrics',
description='The fabric enabled for this build',
default='psm',
values=(
'psm', 'sock', 'nemesisib', 'nemesis', 'mrail', 'nemesisibtcp'
)
)
# FIXME : CUDA support is missing
depends_on('bison')
@@ -123,110 +109,52 @@ def url_for_version(self, version):
else:
return "%s/mvapich/mv2/mvapich2-%s.tar.gz" % (base_url, version)
@staticmethod
def enabled(x):
"""Given a variant name returns the string that means the variant is
enabled
@property
def process_manager_options(self):
spec = self.spec
:param x: variant name
:return:
"""
return '+' + x
other_pms = []
for x in ('hydra', 'gforker', 'remshell'):
if 'process_managers={0}'.format(x) in spec:
other_pms.append(x)
opts = ['--with-pm=%s' % ':'.join(other_pms)]
def set_build_type(self, spec, configure_args):
"""Appends to configure_args the flags that depends only on the build
type (i.e. release or debug)
:param spec: spec
:param configure_args: list of current configure arguments
"""
if '+debug' in spec:
build_type_options = [
"--disable-fast",
"--enable-error-checking=runtime",
"--enable-error-messages=all",
# Permits debugging with TotalView
"--enable-g=dbg", "--enable-debuginfo"
]
else:
build_type_options = ["--enable-fast=all"]
configure_args.extend(build_type_options)
def set_process_manager(self, spec, configure_args):
"""Appends to configure_args the flags that will enable the
appropriate process managers
:param spec: spec
:param configure_args: list of current configure arguments
"""
# Check that slurm variant is not activated together with
# other pm variants
has_slurm_incompatible_variants = \
any(self.enabled(x) in spec
for x in Mvapich2.SLURM_INCOMPATIBLE_PMS)
if self.enabled(Mvapich2.SLURM) in spec and \
has_slurm_incompatible_variants:
raise RuntimeError(" %s : 'slurm' cannot be activated \
together with other process managers" % self.name)
process_manager_options = []
# See: http://slurm.schedmd.com/mpi_guide.html#mvapich2
if self.enabled(Mvapich2.SLURM) in spec:
if 'process_managers=slurm' in spec:
if self.version > Version('2.0'):
process_manager_options = [
"--with-pmi=pmi2",
"--with-pm=slurm"
opts = [
'--with-pmi=pmi2',
'--with-pm=slurm'
]
else:
process_manager_options = [
"--with-pmi=slurm",
"--with-pm=no"
opts = [
'--with-pmi=slurm',
'--with-pm=no'
]
elif has_slurm_incompatible_variants:
pms = []
# The variant name is equal to the process manager name in
# the configuration options
for x in Mvapich2.SLURM_INCOMPATIBLE_PMS:
if self.enabled(x) in spec:
pms.append(x)
process_manager_options = [
"--with-pm=%s" % ':'.join(pms)
]
configure_args.extend(process_manager_options)
return opts
def set_network_type(self, spec, configure_args):
# Check that at most one variant has been activated
count = 0
for x in Mvapich2.SUPPORTED_NETWORKS:
if self.enabled(x) in spec:
count += 1
if count > 1:
raise RuntimeError('network variants are mutually exclusive \
(only one can be selected at a time)')
network_options = []
@property
def network_options(self):
opts = []
# From here on I can suppose that only one variant has been selected
if self.enabled(Mvapich2.PSM) in spec:
network_options = ["--with-device=ch3:psm"]
elif self.enabled(Mvapich2.SOCK) in spec:
network_options = ["--with-device=ch3:sock"]
elif self.enabled(Mvapich2.NEMESISIBTCP) in spec:
network_options = ["--with-device=ch3:nemesis:ib,tcp"]
elif self.enabled(Mvapich2.NEMESISIB) in spec:
network_options = ["--with-device=ch3:nemesis:ib"]
elif self.enabled(Mvapich2.NEMESIS) in spec:
network_options = ["--with-device=ch3:nemesis"]
elif self.enabled(Mvapich2.MRAIL) in spec:
network_options = ["--with-device=ch3:mrail", "--with-rdma=gen2"]
configure_args.extend(network_options)
if 'fabrics=psm' in self.spec:
opts = ["--with-device=ch3:psm"]
elif 'fabrics=sock' in self.spec:
opts = ["--with-device=ch3:sock"]
elif 'fabrics=nemesisibtcp' in self.spec:
opts = ["--with-device=ch3:nemesis:ib,tcp"]
elif 'fabrics=nemesisib' in self.spec:
opts = ["--with-device=ch3:nemesis:ib"]
elif 'fabrics=nemesis' in self.spec:
opts = ["--with-device=ch3:nemesis"]
elif 'fabrics=mrail' in self.spec:
opts = ["--with-device=ch3:mrail", "--with-rdma=gen2"]
return opts
def setup_environment(self, spack_env, run_env):
if self.enabled(Mvapich2.SLURM) in self.spec and \
self.version > Version('2.0'):
spec = self.spec
if 'process_managers=slurm' in spec and spec.satisfies('@2.0:'):
run_env.set('SLURM_MPI_TYPE', 'pmi2')
def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
@@ -251,48 +179,44 @@ def setup_dependent_package(self, module, dependent_spec):
join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
]
def install(self, spec, prefix):
@run_before('configure')
def die_without_fortran(self):
# Until we can pass variants such as +fortran through virtual
# dependencies depends_on('mpi'), require Fortran compiler to
# avoid delayed build errors in dependents.
if (self.compiler.f77 is None) or (self.compiler.fc is None):
raise InstallError('Mvapich2 requires both C and Fortran ',
'compilers!')
raise InstallError(
'Mvapich2 requires both C and Fortran compilers!'
)
# we'll set different configure flags depending on our
# environment
configure_args = [
"--prefix=%s" % prefix,
"--enable-shared",
"--enable-romio",
"--disable-silent-rules",
def configure_args(self):
args = [
'--enable-shared',
'--enable-romio',
'-disable-silent-rules',
'--enable-fortran=all',
"--enable-threads={0}".format(spec.variants['threads'].value),
"--with-ch3-rank-bits={0}".format(
spec.variants['ch3_rank_bits'].value),
]
if self.compiler.f77 and self.compiler.fc:
configure_args.append("--enable-fortran=all")
elif self.compiler.f77:
configure_args.append("--enable-fortran=f77")
elif self.compiler.fc:
configure_args.append("--enable-fortran=fc")
if '+debug' in self.spec:
args.extend([
'--disable-fast',
'--enable-error-checking=runtime',
'--enable-error-messages=all',
# Permits debugging with TotalView
'--enable-g=dbg',
'--enable-debuginfo'
])
else:
configure_args.append("--enable-fortran=none")
args.append('--enable-fast=all')
# Set the type of the build (debug, release)
self.set_build_type(spec, configure_args)
# Set the process manager
self.set_process_manager(spec, configure_args)
# Determine network type by variant
self.set_network_type(spec, configure_args)
configure(*configure_args)
make()
make("install")
self.filter_compilers()
args.extend(self.process_manager_options)
args.extend(self.network_options)
return args
@run_after('install')
def filter_compilers(self):
"""Run after install to make the MPI compilers use the
compilers that Spack built the package with.
@@ -302,7 +226,7 @@ def filter_compilers(self):
be bound to whatever compiler they were built with.
"""
bin = self.prefix.bin
mpicc = join_path(bin, 'mpicc')
mpicc = join_path(bin, 'mpicc')
mpicxx = join_path(bin, 'mpicxx')
mpif77 = join_path(bin, 'mpif77')
mpif90 = join_path(bin, 'mpif90')

26
var/spack/repos/builtin/packages/netcdf/package.py
View File

@@ -24,6 +24,16 @@
##############################################################################
from spack import *
import numbers
def is_integral(x):
"""Any integer value"""
try:
return isinstance(int(x), numbers.Integral) and not isinstance(x, bool)
except ValueError:
return False
class Netcdf(AutotoolsPackage):
"""NetCDF is a set of software libraries and self-describing,
@@ -52,10 +62,18 @@ class Netcdf(AutotoolsPackage):
# These variants control the number of dimensions (i.e. coordinates and
# attributes) and variables (e.g. time, entity ID, number of coordinates)
# that can be used in any particular NetCDF file.
variant('maxdims', default=1024,
description='Defines the maximum dimensions of NetCDF files.')
variant('maxvars', default=8192,
description='Defines the maximum variables of NetCDF files.')
variant(
'maxdims',
default=1024,
description='Defines the maximum dimensions of NetCDF files.',
values=is_integral
)
variant(
'maxvars',
default=8192,
description='Defines the maximum variables of NetCDF files.',
values=is_integral
)
depends_on("m4", type='build')
depends_on("hdf", when='+hdf4')

111
var/spack/repos/builtin/packages/openmpi/package.py
View File

@@ -22,13 +22,16 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
import os
from spack import *
def _verbs_dir():
"""Try to find the directory where the OpenFabrics verbs package is
installed. Return None if not found."""
installed. Return None if not found.
"""
try:
# Try to locate Verbs by looking for a utility in the path
ibv_devices = which("ibv_devices")
@@ -43,7 +46,7 @@ def _verbs_dir():
if path == "/":
path = "/usr"
return path
except:
except TypeError:
return None
@@ -82,22 +85,20 @@ class Openmpi(AutotoolsPackage):
patch('configure.patch', when="@1.10.0:1.10.1")
patch('fix_multidef_pmi_class.patch', when="@2.0.0:2.0.1")
# Fabrics
variant('psm', default=False, description='Build support for the PSM library')
variant('psm2', default=False,
description='Build support for the Intel PSM2 library')
variant('pmi', default=False,
description='Build support for PMI-based launchers')
variant('verbs', default=_verbs_dir() is not None,
description='Build support for OpenFabrics verbs')
variant('mxm', default=False, description='Build Mellanox Messaging support')
variant(
'fabrics',
default=None if _verbs_dir() is None else 'verbs',
description='List of fabrics that are enabled',
values=('psm', 'psm2', 'pmi', 'verbs', 'mxm'),
multi=True
)
# Schedulers
# TODO: support for alps and loadleveler is missing
variant('tm', default=False,
description='Build TM (Torque, PBSPro, and compatible) support')
variant('slurm', default=False,
description='Build SLURM scheduler component')
variant(
'schedulers',
description='List of schedulers for which support is enabled',
values=('alps', 'lsf', 'tm', 'slurm', 'sge', 'loadleveler'),
multi=True
)
# Additional support options
variant('java', default=False, description='Build Java support')
@@ -114,7 +115,7 @@ class Openmpi(AutotoolsPackage):
depends_on('hwloc')
depends_on('hwloc +cuda', when='+cuda')
depends_on('jdk', when='+java')
depends_on('sqlite', when='+sqlite3')
depends_on('sqlite', when='+sqlite3@:1.11')
def url_for_version(self, version):
return "http://www.open-mpi.org/software/ompi/v%s/downloads/openmpi-%s.tar.bz2" % (
@@ -153,15 +154,22 @@ def setup_dependent_package(self, module, dependent_spec):
join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
]
@property
def verbs(self):
def with_or_without_verbs(self, activated):
# Up through version 1.6, this option was previously named
# --with-openib
if self.spec.satisfies('@:1.6'):
return 'openib'
opt = 'openib'
# In version 1.7, it was renamed to be --with-verbs
elif self.spec.satisfies('@1.7:'):
return 'verbs'
if self.spec.satisfies('@1.7:'):
opt = 'verbs'
# If the option has not been activated return
# --without-openib or --without-verbs
if not activated:
return '--without-{0}'.format(opt)
line = '--with-{0}'.format(opt)
path = _verbs_dir()
if (path is not None) and (path not in ('/usr', '/usr/local')):
line += '={0}'.format(path)
return line
@run_before('autoreconf')
def die_without_fortran(self):
@@ -175,48 +183,17 @@ def die_without_fortran(self):
def configure_args(self):
spec = self.spec
config_args = [
'--enable-shared',
'--enable-static',
'--enable-mpi-cxx',
# Schedulers
'--with-tm' if '+tm' in spec else '--without-tm',
'--with-slurm' if '+slurm' in spec else '--without-slurm',
# Fabrics
'--with-psm' if '+psm' in spec else '--without-psm',
'--enable-static'
]
if self.spec.satisfies('@2.0:'):
# for Open-MPI 2.0:, C++ bindings are disabled by default.
config_args.extend(['--enable-mpi-cxx'])
# Intel PSM2 support
if spec.satisfies('@1.10:'):
if '+psm2' in spec:
config_args.append('--with-psm2')
else:
config_args.append('--without-psm2')
# PMI support
if spec.satisfies('@1.5.5:'):
if '+pmi' in spec:
config_args.append('--with-pmi')
else:
config_args.append('--without-pmi')
# Mellanox Messaging support
if spec.satisfies('@1.5.4:'):
if '+mxm' in spec:
config_args.append('--with-mxm')
else:
config_args.append('--without-mxm')
# OpenFabrics verbs support
if '+verbs' in spec:
path = _verbs_dir()
if path is not None and path not in ('/usr', '/usr/local'):
config_args.append('--with-{0}={1}'.format(self.verbs, path))
else:
config_args.append('--with-{0}'.format(self.verbs))
else:
config_args.append('--without-{0}'.format(self.verbs))
# Fabrics and schedulers
config_args.extend(self.with_or_without('fabrics'))
config_args.extend(self.with_or_without('schedulers'))
# Hwloc support
if spec.satisfies('@1.5.2:'):
@@ -270,11 +247,11 @@ def configure_args(self):
@run_after('install')
def filter_compilers(self):
"""Run after install to make the MPI compilers use the
compilers that Spack built the package with.
compilers that Spack built the package with.
If this isn't done, they'll have CC, CXX and FC set
to Spack's generic cc, c++ and f90. We want them to
be bound to whatever compiler they were built with.
If this isn't done, they'll have CC, CXX and FC set
to Spack's generic cc, c++ and f90. We want them to
be bound to whatever compiler they were built with.
"""
kwargs = {'ignore_absent': True, 'backup': False, 'string': False}
wrapper_basepath = join_path(self.prefix, 'share', 'openmpi')