30daf95ae8
Packages built targeting a backend may depend on packages like cmake which can be built against the frontend. With this commit, any build dependency or child of a build dependency will target the frontend by default. In compiler concretization when packages copy compilers from nearby packages, build dependencies use compiler information from other build dependencies, and link dependencies avoid using compiler information from build dependencies.
3112 lines
114 KiB
Python
3112 lines
114 KiB
Python
##############################################################################
|
|
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
|
|
# Produced at the Lawrence Livermore National Laboratory.
|
|
#
|
|
# This file is part of Spack.
|
|
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
|
# LLNL-CODE-647188
|
|
#
|
|
# For details, see https://github.com/llnl/spack
|
|
# Please also see the LICENSE file for our notice and the LGPL.
|
|
#
|
|
# This program is free software; you can redistribute it and/or modify
|
|
# it under the terms of the GNU Lesser General Public License (as
|
|
# published by the Free Software Foundation) version 2.1, February 1999.
|
|
#
|
|
# This program is distributed in the hope that it will be useful, but
|
|
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
|
# conditions of the GNU Lesser General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Lesser General Public License
|
|
# along with this program; if not, write to the Free Software Foundation,
|
|
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
|
##############################################################################
|
|
"""
|
|
Spack allows very fine-grained control over how packages are installed and
|
|
over how they are built and configured. To make this easy, it has its own
|
|
syntax for declaring a dependence. We call a descriptor of a particular
|
|
package configuration a "spec".
|
|
|
|
The syntax looks like this:
|
|
|
|
.. code-block:: sh
|
|
|
|
$ spack install mpileaks ^openmpi @1.2:1.4 +debug %intel @12.1 =bgqos_0
|
|
0 1 2 3 4 5 6
|
|
|
|
The first part of this is the command, 'spack install'. The rest of the
|
|
line is a spec for a particular installation of the mpileaks package.
|
|
|
|
0. The package to install
|
|
|
|
1. A dependency of the package, prefixed by ^
|
|
|
|
2. A version descriptor for the package. This can either be a specific
|
|
version, like "1.2", or it can be a range of versions, e.g. "1.2:1.4".
|
|
If multiple specific versions or multiple ranges are acceptable, they
|
|
can be separated by commas, e.g. if a package will only build with
|
|
versions 1.0, 1.2-1.4, and 1.6-1.8 of mavpich, you could say:
|
|
|
|
depends_on("mvapich@1.0,1.2:1.4,1.6:1.8")
|
|
|
|
3. A compile-time variant of the package. If you need openmpi to be
|
|
built in debug mode for your package to work, you can require it by
|
|
adding +debug to the openmpi spec when you depend on it. If you do
|
|
NOT want the debug option to be enabled, then replace this with -debug.
|
|
|
|
4. The name of the compiler to build with.
|
|
|
|
5. The versions of the compiler to build with. Note that the identifier
|
|
for a compiler version is the same '@' that is used for a package version.
|
|
A version list denoted by '@' is associated with the compiler only if
|
|
if it comes immediately after the compiler name. Otherwise it will be
|
|
associated with the current package spec.
|
|
|
|
6. The architecture to build with. This is needed on machines where
|
|
cross-compilation is required
|
|
|
|
Here is the EBNF grammar for a spec::
|
|
|
|
spec-list = { spec [ dep-list ] }
|
|
dep_list = { ^ spec }
|
|
spec = id [ options ]
|
|
options = { @version-list | +variant | -variant | ~variant |
|
|
%compiler | arch=architecture | [ flag ]=value}
|
|
flag = { cflags | cxxflags | fcflags | fflags | cppflags |
|
|
ldflags | ldlibs }
|
|
variant = id
|
|
architecture = id
|
|
compiler = id [ version-list ]
|
|
version-list = version [ { , version } ]
|
|
version = id | id: | :id | id:id
|
|
id = [A-Za-z0-9_][A-Za-z0-9_.-]*
|
|
|
|
Identifiers using the <name>=<value> command, such as architectures and
|
|
compiler flags, require a space before the name.
|
|
|
|
There is one context-sensitive part: ids in versions may contain '.', while
|
|
other ids may not.
|
|
|
|
There is one ambiguity: since '-' is allowed in an id, you need to put
|
|
whitespace space before -variant for it to be tokenized properly. You can
|
|
either use whitespace, or you can just use ~variant since it means the same
|
|
thing. Spack uses ~variant in directory names and in the canonical form of
|
|
specs to avoid ambiguity. Both are provided because ~ can cause shell
|
|
expansion when it is the first character in an id typed on the command line.
|
|
"""
|
|
import base64
|
|
import hashlib
|
|
import ctypes
|
|
from StringIO import StringIO
|
|
from operator import attrgetter
|
|
|
|
from yaml.error import MarkedYAMLError
|
|
|
|
import llnl.util.tty as tty
|
|
from llnl.util.lang import *
|
|
from llnl.util.tty.color import *
|
|
|
|
import spack
|
|
import spack.architecture
|
|
import spack.store
|
|
import spack.compilers as compilers
|
|
import spack.error
|
|
import spack.parse
|
|
from spack.build_environment import get_path_from_module, load_module
|
|
from spack.util.prefix import Prefix
|
|
from spack.util.string import *
|
|
import spack.util.spack_yaml as syaml
|
|
import spack.util.spack_json as sjson
|
|
from spack.util.spack_yaml import syaml_dict
|
|
from spack.util.crypto import prefix_bits
|
|
from spack.version import *
|
|
from spack.provider_index import ProviderIndex
|
|
|
|
__all__ = [
|
|
'Spec',
|
|
'alldeps',
|
|
'nolink',
|
|
'nobuild',
|
|
'canonical_deptype',
|
|
'validate_deptype',
|
|
'parse',
|
|
'parse_anonymous_spec',
|
|
'SpecError',
|
|
'SpecParseError',
|
|
'DuplicateDependencyError',
|
|
'DuplicateVariantError',
|
|
'DuplicateCompilerSpecError',
|
|
'UnsupportedCompilerError',
|
|
'UnknownVariantError',
|
|
'DuplicateArchitectureError',
|
|
'InconsistentSpecError',
|
|
'InvalidDependencyError',
|
|
'InvalidDependencyTypeError',
|
|
'NoProviderError',
|
|
'MultipleProviderError',
|
|
'UnsatisfiableSpecError',
|
|
'UnsatisfiableSpecNameError',
|
|
'UnsatisfiableVersionSpecError',
|
|
'UnsatisfiableCompilerSpecError',
|
|
'UnsatisfiableVariantSpecError',
|
|
'UnsatisfiableCompilerFlagSpecError',
|
|
'UnsatisfiableArchitectureSpecError',
|
|
'UnsatisfiableProviderSpecError',
|
|
'UnsatisfiableDependencySpecError',
|
|
'AmbiguousHashError']
|
|
|
|
# Valid pattern for an identifier in Spack
|
|
identifier_re = r'\w[\w-]*'
|
|
|
|
# Convenient names for color formats so that other things can use them
|
|
compiler_color = '@g'
|
|
version_color = '@c'
|
|
architecture_color = '@m'
|
|
enabled_variant_color = '@B'
|
|
disabled_variant_color = '@r'
|
|
dependency_color = '@.'
|
|
hash_color = '@K'
|
|
|
|
"""This map determines the coloring of specs when using color output.
|
|
We make the fields different colors to enhance readability.
|
|
See spack.color for descriptions of the color codes. """
|
|
color_formats = {'%': compiler_color,
|
|
'@': version_color,
|
|
'=': architecture_color,
|
|
'+': enabled_variant_color,
|
|
'~': disabled_variant_color,
|
|
'^': dependency_color,
|
|
'#': hash_color}
|
|
|
|
"""Regex used for splitting by spec field separators."""
|
|
_separators = '[%s]' % ''.join(color_formats.keys())
|
|
|
|
"""Versionlist constant so we don't have to build a list
|
|
every time we call str()"""
|
|
_any_version = VersionList([':'])
|
|
|
|
# Special types of dependencies.
|
|
alldeps = ('build', 'link', 'run')
|
|
nolink = ('build', 'run')
|
|
nobuild = ('link', 'run')
|
|
norun = ('link', 'build')
|
|
special_types = {
|
|
'alldeps': alldeps,
|
|
'all': alldeps, # allow "all" as string but not symbol.
|
|
'nolink': nolink,
|
|
'nobuild': nobuild,
|
|
'norun': norun,
|
|
}
|
|
|
|
legal_deps = tuple(special_types) + alldeps
|
|
|
|
"""Max integer helps avoid passing too large a value to cyaml."""
|
|
maxint = 2 ** (ctypes.sizeof(ctypes.c_int) * 8 - 1) - 1
|
|
|
|
|
|
def validate_deptype(deptype):
|
|
if isinstance(deptype, str):
|
|
if deptype not in legal_deps:
|
|
raise InvalidDependencyTypeError(
|
|
"Invalid dependency type: %s" % deptype)
|
|
|
|
elif isinstance(deptype, (list, tuple)):
|
|
for t in deptype:
|
|
validate_deptype(t)
|
|
|
|
elif deptype is None:
|
|
raise InvalidDependencyTypeError("deptype cannot be None!")
|
|
|
|
|
|
def canonical_deptype(deptype):
|
|
if deptype is None:
|
|
return alldeps
|
|
|
|
elif isinstance(deptype, str):
|
|
return special_types.get(deptype, (deptype,))
|
|
|
|
elif isinstance(deptype, (tuple, list)):
|
|
return (sum((canonical_deptype(d) for d in deptype), ()))
|
|
|
|
return deptype
|
|
|
|
|
|
def colorize_spec(spec):
|
|
"""Returns a spec colorized according to the colors specified in
|
|
color_formats."""
|
|
class insert_color:
|
|
|
|
def __init__(self):
|
|
self.last = None
|
|
|
|
def __call__(self, match):
|
|
# ignore compiler versions (color same as compiler)
|
|
sep = match.group(0)
|
|
if self.last == '%' and sep == '@':
|
|
return cescape(sep)
|
|
self.last = sep
|
|
|
|
return '%s%s' % (color_formats[sep], cescape(sep))
|
|
|
|
return colorize(re.sub(_separators, insert_color(), str(spec)) + '@.')
|
|
|
|
|
|
@key_ordering
|
|
class ArchSpec(object):
|
|
""" The ArchSpec class represents an abstract architecture specification
|
|
that a package should be built with. At its core, each ArchSpec is
|
|
comprised of three elements: a platform (e.g. Linux), an OS (e.g.
|
|
RHEL6), and a target (e.g. x86_64).
|
|
"""
|
|
|
|
# TODO: Formalize the specifications for architectures and then use
|
|
# the appropriate parser here to read these specifications.
|
|
def __init__(self, *args):
|
|
to_attr_string = lambda s: str(s) if s and s != "None" else None
|
|
|
|
self.platform, self.platform_os, self.target = (None, None, None)
|
|
|
|
if len(args) == 1:
|
|
spec_like = args[0]
|
|
if isinstance(spec_like, ArchSpec):
|
|
self._dup(spec_like)
|
|
elif isinstance(spec_like, basestring):
|
|
spec_fields = spec_like.split("-")
|
|
|
|
if len(spec_fields) == 3:
|
|
self.platform, self.platform_os, self.target = tuple(
|
|
to_attr_string(f) for f in spec_fields)
|
|
else:
|
|
raise ValueError("%s is an invalid arch spec" % spec_like)
|
|
elif len(args) == 3:
|
|
self.platform = to_attr_string(args[0])
|
|
self.platform_os = to_attr_string(args[1])
|
|
self.target = to_attr_string(args[2])
|
|
elif len(args) != 0:
|
|
raise TypeError("Can't make arch spec from %s" % args)
|
|
|
|
def _autospec(self, spec_like):
|
|
if isinstance(spec_like, ArchSpec):
|
|
return spec_like
|
|
return ArchSpec(spec_like)
|
|
|
|
def _cmp_key(self):
|
|
return (self.platform, self.platform_os, self.target)
|
|
|
|
def _dup(self, other):
|
|
self.platform = other.platform
|
|
self.platform_os = other.platform_os
|
|
self.target = other.target
|
|
|
|
@property
|
|
def platform(self):
|
|
return self._platform
|
|
|
|
@platform.setter
|
|
def platform(self, value):
|
|
""" The platform of the architecture spec will be verified as a
|
|
supported Spack platform before it's set to ensure all specs
|
|
refer to valid platforms.
|
|
"""
|
|
value = str(value) if value is not None else None
|
|
self._platform = value
|
|
|
|
@property
|
|
def platform_os(self):
|
|
return self._platform_os
|
|
|
|
@platform_os.setter
|
|
def platform_os(self, value):
|
|
""" The OS of the architecture spec will update the platform field
|
|
if the OS is set to one of the reserved OS types so that the
|
|
default OS type can be resolved. Since the reserved OS
|
|
information is only available for the host machine, the platform
|
|
will assumed to be the host machine's platform.
|
|
"""
|
|
value = str(value) if value is not None else None
|
|
|
|
if value in spack.architecture.Platform.reserved_oss:
|
|
curr_platform = str(spack.architecture.platform())
|
|
self.platform = self.platform or curr_platform
|
|
|
|
if self.platform != curr_platform:
|
|
raise ValueError(
|
|
"Can't set arch spec OS to reserved value '%s' when the "
|
|
"arch platform (%s) isn't the current platform (%s)" %
|
|
(value, self.platform, curr_platform))
|
|
|
|
spec_platform = spack.architecture.get_platform(self.platform)
|
|
value = str(spec_platform.operating_system(value))
|
|
|
|
self._platform_os = value
|
|
|
|
@property
|
|
def target(self):
|
|
return self._target
|
|
|
|
@target.setter
|
|
def target(self, value):
|
|
""" The target of the architecture spec will update the platform field
|
|
if the target is set to one of the reserved target types so that
|
|
the default target type can be resolved. Since the reserved target
|
|
information is only available for the host machine, the platform
|
|
will assumed to be the host machine's platform.
|
|
"""
|
|
value = str(value) if value is not None else None
|
|
|
|
if value in spack.architecture.Platform.reserved_targets:
|
|
curr_platform = str(spack.architecture.platform())
|
|
self.platform = self.platform or curr_platform
|
|
|
|
if self.platform != curr_platform:
|
|
raise ValueError(
|
|
"Can't set arch spec target to reserved value '%s' when "
|
|
"the arch platform (%s) isn't the current platform (%s)" %
|
|
(value, self.platform, curr_platform))
|
|
|
|
spec_platform = spack.architecture.get_platform(self.platform)
|
|
value = str(spec_platform.target(value))
|
|
|
|
self._target = value
|
|
|
|
def satisfies(self, other, strict=False):
|
|
other = self._autospec(other)
|
|
sdict, odict = self.to_cmp_dict(), other.to_cmp_dict()
|
|
|
|
if strict or self.concrete:
|
|
return all(getattr(self, attr) == getattr(other, attr)
|
|
for attr in odict if odict[attr])
|
|
else:
|
|
return all(getattr(self, attr) == getattr(other, attr)
|
|
for attr in odict if sdict[attr] and odict[attr])
|
|
|
|
def constrain(self, other):
|
|
""" Projects all architecture fields that are specified in the given
|
|
spec onto the instance spec if they're missing from the instance
|
|
spec. This will only work if the two specs are compatible.
|
|
"""
|
|
other = self._autospec(other)
|
|
|
|
if not self.satisfies(other):
|
|
raise UnsatisfiableArchitectureSpecError(self, other)
|
|
|
|
constrained = False
|
|
for attr, svalue in self.to_cmp_dict().iteritems():
|
|
ovalue = getattr(other, attr)
|
|
if svalue is None and ovalue is not None:
|
|
setattr(self, attr, ovalue)
|
|
constrained = True
|
|
|
|
return constrained
|
|
|
|
def copy(self):
|
|
clone = ArchSpec.__new__(ArchSpec)
|
|
clone._dup(self)
|
|
return clone
|
|
|
|
@property
|
|
def concrete(self):
|
|
return all(v for k, v in self.to_cmp_dict().iteritems())
|
|
|
|
def to_cmp_dict(self):
|
|
"""Returns a dictionary that can be used for field comparison."""
|
|
return dict([
|
|
('platform', self.platform),
|
|
('platform_os', self.platform_os),
|
|
('target', self.target)])
|
|
|
|
def to_dict(self):
|
|
d = syaml_dict([
|
|
('platform', self.platform),
|
|
('platform_os', self.platform_os),
|
|
('target', self.target)])
|
|
return syaml_dict([('arch', d)])
|
|
|
|
@staticmethod
|
|
def from_dict(d):
|
|
"""Import an ArchSpec from raw YAML/JSON data.
|
|
|
|
This routine implements a measure of compatibility with older
|
|
versions of Spack. Spack releases before 0.10 used a single
|
|
string with no OS or platform identifiers. We import old Spack
|
|
architectures with platform ``spack09``, OS ``unknown``, and the
|
|
old arch string as the target.
|
|
|
|
Specs from `0.10` or later have a more fleshed out architecture
|
|
descriptor with a platform, an OS, and a target.
|
|
|
|
"""
|
|
if not isinstance(d['arch'], dict):
|
|
return ArchSpec('spack09', 'unknown', d['arch'])
|
|
|
|
d = d['arch']
|
|
return ArchSpec(d['platform'], d['platform_os'], d['target'])
|
|
|
|
def __str__(self):
|
|
return "%s-%s-%s" % (self.platform, self.platform_os, self.target)
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
def __contains__(self, string):
|
|
return string in str(self)
|
|
|
|
|
|
@key_ordering
|
|
class CompilerSpec(object):
|
|
"""The CompilerSpec field represents the compiler or range of compiler
|
|
versions that a package should be built with. CompilerSpecs have a
|
|
name and a version list. """
|
|
|
|
def __init__(self, *args):
|
|
nargs = len(args)
|
|
if nargs == 1:
|
|
arg = args[0]
|
|
# If there is one argument, it's either another CompilerSpec
|
|
# to copy or a string to parse
|
|
if isinstance(arg, basestring):
|
|
c = SpecParser().parse_compiler(arg)
|
|
self.name = c.name
|
|
self.versions = c.versions
|
|
|
|
elif isinstance(arg, CompilerSpec):
|
|
self.name = arg.name
|
|
self.versions = arg.versions.copy()
|
|
|
|
else:
|
|
raise TypeError(
|
|
"Can only build CompilerSpec from string or " +
|
|
"CompilerSpec. Found %s" % type(arg))
|
|
|
|
elif nargs == 2:
|
|
name, version = args
|
|
self.name = name
|
|
self.versions = VersionList()
|
|
self.versions.add(ver(version))
|
|
|
|
else:
|
|
raise TypeError(
|
|
"__init__ takes 1 or 2 arguments. (%d given)" % nargs)
|
|
|
|
def _add_version(self, version):
|
|
self.versions.add(version)
|
|
|
|
def _autospec(self, compiler_spec_like):
|
|
if isinstance(compiler_spec_like, CompilerSpec):
|
|
return compiler_spec_like
|
|
return CompilerSpec(compiler_spec_like)
|
|
|
|
def satisfies(self, other, strict=False):
|
|
other = self._autospec(other)
|
|
return (self.name == other.name and
|
|
self.versions.satisfies(other.versions, strict=strict))
|
|
|
|
def constrain(self, other):
|
|
"""Intersect self's versions with other.
|
|
|
|
Return whether the CompilerSpec changed.
|
|
"""
|
|
other = self._autospec(other)
|
|
|
|
# ensure that other will actually constrain this spec.
|
|
if not other.satisfies(self):
|
|
raise UnsatisfiableCompilerSpecError(other, self)
|
|
|
|
return self.versions.intersect(other.versions)
|
|
|
|
@property
|
|
def concrete(self):
|
|
"""A CompilerSpec is concrete if its versions are concrete and there
|
|
is an available compiler with the right version."""
|
|
return self.versions.concrete
|
|
|
|
@property
|
|
def version(self):
|
|
if not self.concrete:
|
|
raise SpecError("Spec is not concrete: " + str(self))
|
|
return self.versions[0]
|
|
|
|
def copy(self):
|
|
clone = CompilerSpec.__new__(CompilerSpec)
|
|
clone.name = self.name
|
|
clone.versions = self.versions.copy()
|
|
return clone
|
|
|
|
def _cmp_key(self):
|
|
return (self.name, self.versions)
|
|
|
|
def to_dict(self):
|
|
d = syaml_dict([('name', self.name)])
|
|
d.update(self.versions.to_dict())
|
|
|
|
return syaml_dict([('compiler', d)])
|
|
|
|
@staticmethod
|
|
def from_dict(d):
|
|
d = d['compiler']
|
|
return CompilerSpec(d['name'], VersionList.from_dict(d))
|
|
|
|
def __str__(self):
|
|
out = self.name
|
|
if self.versions and self.versions != _any_version:
|
|
vlist = ",".join(str(v) for v in self.versions)
|
|
out += "@%s" % vlist
|
|
return out
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
|
|
@key_ordering
|
|
class DependencySpec(object):
|
|
"""Dependencies can be one (or more) of several types:
|
|
|
|
- build: needs to be in the PATH at build time.
|
|
- link: is linked to and added to compiler flags.
|
|
- run: needs to be in the PATH for the package to run.
|
|
|
|
Fields:
|
|
- spec: the spack.spec.Spec description of a dependency.
|
|
- deptypes: strings representing the type of dependency this is.
|
|
"""
|
|
|
|
def __init__(self, spec, deptypes):
|
|
self.spec = spec
|
|
self.deptypes = deptypes
|
|
|
|
def _cmp_key(self):
|
|
return self.spec
|
|
|
|
def copy(self):
|
|
return DependencySpec(self.spec.copy(), self.deptype)
|
|
|
|
def __str__(self):
|
|
return str(self.spec)
|
|
|
|
|
|
@key_ordering
|
|
class VariantSpec(object):
|
|
|
|
"""Variants are named, build-time options for a package. Names depend
|
|
on the particular package being built, and each named variant can
|
|
be enabled or disabled.
|
|
"""
|
|
|
|
def __init__(self, name, value):
|
|
self.name = name
|
|
self.value = value
|
|
|
|
def _cmp_key(self):
|
|
return (self.name, self.value)
|
|
|
|
def copy(self):
|
|
return VariantSpec(self.name, self.value)
|
|
|
|
def __str__(self):
|
|
if type(self.value) == bool:
|
|
return '{0}{1}'.format('+' if self.value else '~', self.name)
|
|
else:
|
|
return ' {0}={1}'.format(self.name, self.value)
|
|
|
|
|
|
class VariantMap(HashableMap):
|
|
|
|
def __init__(self, spec):
|
|
super(VariantMap, self).__init__()
|
|
self.spec = spec
|
|
|
|
def satisfies(self, other, strict=False):
|
|
if strict or self.spec._concrete:
|
|
return all(k in self and self[k].value == other[k].value
|
|
for k in other)
|
|
else:
|
|
return all(self[k].value == other[k].value
|
|
for k in other if k in self)
|
|
|
|
def constrain(self, other):
|
|
"""Add all variants in other that aren't in self to self.
|
|
|
|
Raises an error if any common variants don't match.
|
|
Return whether the spec changed.
|
|
"""
|
|
if other.spec._concrete:
|
|
for k in self:
|
|
if k not in other:
|
|
raise UnsatisfiableVariantSpecError(self[k], '<absent>')
|
|
|
|
changed = False
|
|
for k in other:
|
|
if k in self:
|
|
if self[k].value != other[k].value:
|
|
raise UnsatisfiableVariantSpecError(self[k], other[k])
|
|
else:
|
|
self[k] = other[k].copy()
|
|
changed = True
|
|
return changed
|
|
|
|
@property
|
|
def concrete(self):
|
|
return self.spec._concrete or all(
|
|
v in self for v in self.spec.package_class.variants)
|
|
|
|
def copy(self):
|
|
clone = VariantMap(None)
|
|
for name, variant in self.items():
|
|
clone[name] = variant.copy()
|
|
return clone
|
|
|
|
def __str__(self):
|
|
sorted_keys = sorted(self.keys())
|
|
return ''.join(str(self[key]) for key in sorted_keys)
|
|
|
|
|
|
_valid_compiler_flags = [
|
|
'cflags', 'cxxflags', 'fflags', 'ldflags', 'ldlibs', 'cppflags']
|
|
|
|
|
|
class FlagMap(HashableMap):
|
|
|
|
def __init__(self, spec):
|
|
super(FlagMap, self).__init__()
|
|
self.spec = spec
|
|
|
|
def satisfies(self, other, strict=False):
|
|
if strict or (self.spec and self.spec._concrete):
|
|
return all(f in self and set(self[f]) == set(other[f])
|
|
for f in other)
|
|
else:
|
|
return all(set(self[f]) == set(other[f])
|
|
for f in other if (other[f] != [] and f in self))
|
|
|
|
def constrain(self, other):
|
|
"""Add all flags in other that aren't in self to self.
|
|
|
|
Return whether the spec changed.
|
|
"""
|
|
if other.spec and other.spec._concrete:
|
|
for k in self:
|
|
if k not in other:
|
|
raise UnsatisfiableCompilerFlagSpecError(
|
|
self[k], '<absent>')
|
|
|
|
changed = False
|
|
for k in other:
|
|
if k in self and not set(self[k]) <= set(other[k]):
|
|
raise UnsatisfiableCompilerFlagSpecError(
|
|
' '.join(f for f in self[k]),
|
|
' '.join(f for f in other[k]))
|
|
elif k not in self:
|
|
self[k] = other[k]
|
|
changed = True
|
|
return changed
|
|
|
|
@staticmethod
|
|
def valid_compiler_flags():
|
|
return _valid_compiler_flags
|
|
|
|
@property
|
|
def concrete(self):
|
|
return all(flag in self for flag in _valid_compiler_flags)
|
|
|
|
def copy(self):
|
|
clone = FlagMap(None)
|
|
for name, value in self.items():
|
|
clone[name] = value
|
|
return clone
|
|
|
|
def _cmp_key(self):
|
|
return tuple((k, tuple(v)) for k, v in sorted(self.iteritems()))
|
|
|
|
def __str__(self):
|
|
sorted_keys = filter(
|
|
lambda flag: self[flag] != [], sorted(self.keys()))
|
|
cond_symbol = ' ' if len(sorted_keys) > 0 else ''
|
|
return cond_symbol + ' '.join(
|
|
str(key) + '=\"' + ' '.join(
|
|
str(f) for f in self[key]) + '\"' for key in sorted_keys)
|
|
|
|
|
|
class DependencyMap(HashableMap):
|
|
|
|
"""Each spec has a DependencyMap containing specs for its dependencies.
|
|
The DependencyMap is keyed by name. """
|
|
@property
|
|
def concrete(self):
|
|
return all(d.spec.concrete for d in self.values())
|
|
|
|
def __str__(self):
|
|
return ''.join(
|
|
["^" + self[name].format() for name in sorted(self.keys())])
|
|
|
|
|
|
@key_ordering
|
|
class Spec(object):
|
|
|
|
def __init__(self, spec_like, *dep_like, **kwargs):
|
|
# Copy if spec_like is a Spec.
|
|
if isinstance(spec_like, Spec):
|
|
self._dup(spec_like)
|
|
return
|
|
|
|
# Parse if the spec_like is a string.
|
|
if not isinstance(spec_like, basestring):
|
|
raise TypeError("Can't make spec out of %s" % type(spec_like))
|
|
|
|
spec_list = SpecParser().parse(spec_like)
|
|
if len(spec_list) > 1:
|
|
raise ValueError("More than one spec in string: " + spec_like)
|
|
if len(spec_list) < 1:
|
|
raise ValueError("String contains no specs: " + spec_like)
|
|
|
|
# Take all the attributes from the first parsed spec without copying.
|
|
# This is safe b/c we throw out the parsed spec. It's a bit nasty,
|
|
# but it's nastier to implement the constructor so that the parser
|
|
# writes directly into this Spec object.
|
|
other = spec_list[0]
|
|
self.name = other.name
|
|
self.versions = other.versions
|
|
self.architecture = other.architecture
|
|
self.compiler = other.compiler
|
|
self.compiler_flags = other.compiler_flags
|
|
self.compiler_flags.spec = self
|
|
self._dependencies = other._dependencies
|
|
self._dependents = other._dependents
|
|
self.variants = other.variants
|
|
self.variants.spec = self
|
|
self.namespace = other.namespace
|
|
self._hash = other._hash
|
|
self._cmp_key_cache = other._cmp_key_cache
|
|
|
|
# Specs are by default not assumed to be normal, but in some
|
|
# cases we've read them from a file want to assume normal.
|
|
# This allows us to manipulate specs that Spack doesn't have
|
|
# package.py files for.
|
|
self._normal = kwargs.get('normal', False)
|
|
self._concrete = kwargs.get('concrete', False)
|
|
|
|
# Allow a spec to be constructed with an external path.
|
|
self.external = kwargs.get('external', None)
|
|
self.external_module = kwargs.get('external_module', None)
|
|
|
|
# This allows users to construct a spec DAG with literals.
|
|
# Note that given two specs a and b, Spec(a) copies a, but
|
|
# Spec(a, b) will copy a but just add b as a dep.
|
|
for dep in dep_like:
|
|
spec = dep if isinstance(dep, Spec) else Spec(dep)
|
|
# XXX(deptype): default deptypes
|
|
self._add_dependency(spec, ('build', 'link'))
|
|
|
|
def __getattr__(self, item):
|
|
"""Delegate to self.package if the attribute is not in the spec"""
|
|
# This line is to avoid infinite recursion in case package is
|
|
# not present among self attributes
|
|
if item.endswith('libs'):
|
|
return getattr(self.package, item)
|
|
raise AttributeError()
|
|
|
|
def get_dependency(self, name):
|
|
dep = self._dependencies.get(name)
|
|
if dep is not None:
|
|
return dep
|
|
raise InvalidDependencyException(
|
|
self.name + " does not depend on " + comma_or(name))
|
|
|
|
def _find_deps(self, where, deptype):
|
|
deptype = canonical_deptype(deptype)
|
|
|
|
return [dep.spec
|
|
for dep in where.values()
|
|
if deptype and any(d in deptype for d in dep.deptypes)]
|
|
|
|
def dependencies(self, deptype=None):
|
|
return self._find_deps(self._dependencies, deptype)
|
|
|
|
def dependents(self, deptype=None):
|
|
return self._find_deps(self._dependents, deptype)
|
|
|
|
def _find_deps_dict(self, where, deptype):
|
|
deptype = canonical_deptype(deptype)
|
|
|
|
return dict((dep.spec.name, dep)
|
|
for dep in where.values()
|
|
if deptype and any(d in deptype for d in dep.deptypes))
|
|
|
|
def dependencies_dict(self, deptype=None):
|
|
return self._find_deps_dict(self._dependencies, deptype)
|
|
|
|
def dependents_dict(self, deptype=None):
|
|
return self._find_deps_dict(self._dependents, deptype)
|
|
|
|
#
|
|
# Private routines here are called by the parser when building a spec.
|
|
#
|
|
def _add_version(self, version):
|
|
"""Called by the parser to add an allowable version."""
|
|
self.versions.add(version)
|
|
|
|
def _add_variant(self, name, value):
|
|
"""Called by the parser to add a variant."""
|
|
if name in self.variants:
|
|
raise DuplicateVariantError(
|
|
"Cannot specify variant '%s' twice" % name)
|
|
if isinstance(value, basestring) and value.upper() == 'TRUE':
|
|
value = True
|
|
elif isinstance(value, basestring) and value.upper() == 'FALSE':
|
|
value = False
|
|
self.variants[name] = VariantSpec(name, value)
|
|
|
|
def _add_flag(self, name, value):
|
|
"""Called by the parser to add a known flag.
|
|
Known flags currently include "arch"
|
|
"""
|
|
valid_flags = FlagMap.valid_compiler_flags()
|
|
if name == 'arch' or name == 'architecture':
|
|
parts = tuple(value.split('-'))
|
|
plat, os, tgt = parts if len(parts) == 3 else (None, None, value)
|
|
self._set_architecture(platform=plat, platform_os=os, target=tgt)
|
|
elif name == 'platform':
|
|
self._set_architecture(platform=value)
|
|
elif name == 'os' or name == 'operating_system':
|
|
self._set_architecture(platform_os=value)
|
|
elif name == 'target':
|
|
self._set_architecture(target=value)
|
|
elif name in valid_flags:
|
|
assert(self.compiler_flags is not None)
|
|
self.compiler_flags[name] = value.split()
|
|
else:
|
|
self._add_variant(name, value)
|
|
|
|
def _set_architecture(self, **kwargs):
|
|
"""Called by the parser to set the architecture."""
|
|
arch_attrs = ['platform', 'platform_os', 'target']
|
|
if self.architecture and self.architecture.concrete:
|
|
raise DuplicateArchitectureError(
|
|
"Spec for '%s' cannot have two architectures." % self.name)
|
|
|
|
if not self.architecture:
|
|
new_vals = tuple(kwargs.get(arg, None) for arg in arch_attrs)
|
|
self.architecture = ArchSpec(*new_vals)
|
|
else:
|
|
new_attrvals = [(a, v) for a, v in kwargs.iteritems()
|
|
if a in arch_attrs]
|
|
for new_attr, new_value in new_attrvals:
|
|
if getattr(self.architecture, new_attr):
|
|
raise DuplicateArchitectureError(
|
|
"Spec for '%s' cannot have two '%s' specified "
|
|
"for its architecture" % (self.name, new_attr))
|
|
else:
|
|
setattr(self.architecture, new_attr, new_value)
|
|
|
|
def _set_compiler(self, compiler):
|
|
"""Called by the parser to set the compiler."""
|
|
if self.compiler:
|
|
raise DuplicateCompilerSpecError(
|
|
"Spec for '%s' cannot have two compilers." % self.name)
|
|
self.compiler = compiler
|
|
|
|
def _add_dependency(self, spec, deptypes):
|
|
"""Called by the parser to add another spec as a dependency."""
|
|
if spec.name in self._dependencies:
|
|
raise DuplicateDependencyError(
|
|
"Cannot depend on '%s' twice" % spec)
|
|
self._dependencies[spec.name] = DependencySpec(spec, deptypes)
|
|
spec._dependents[self.name] = DependencySpec(self, deptypes)
|
|
|
|
#
|
|
# Public interface
|
|
#
|
|
@property
|
|
def fullname(self):
|
|
return (
|
|
('%s.%s' % (self.namespace, self.name)) if self.namespace else
|
|
(self.name if self.name else ''))
|
|
|
|
@property
|
|
def root(self):
|
|
"""Follow dependent links and find the root of this spec's DAG.
|
|
In spack specs, there should be a single root (the package being
|
|
installed). This will throw an assertion error if that is not
|
|
the case.
|
|
"""
|
|
if not self._dependents:
|
|
return self
|
|
|
|
# If the spec has multiple dependents, ensure that they all
|
|
# lead to the same place. Spack shouldn't deal with any DAGs
|
|
# with multiple roots, so something's wrong if we find one.
|
|
depiter = iter(self._dependents.values())
|
|
first_root = next(depiter).spec.root
|
|
assert(all(first_root is d.spec.root for d in depiter))
|
|
return first_root
|
|
|
|
@property
|
|
def package(self):
|
|
return spack.repo.get(self)
|
|
|
|
@property
|
|
def package_class(self):
|
|
"""Internal package call gets only the class object for a package.
|
|
Use this to just get package metadata.
|
|
"""
|
|
return spack.repo.get_pkg_class(self.fullname)
|
|
|
|
@property
|
|
def virtual(self):
|
|
"""Right now, a spec is virtual if no package exists with its name.
|
|
|
|
TODO: revisit this -- might need to use a separate namespace and
|
|
be more explicit about this.
|
|
Possible idea: just use conventin and make virtual deps all
|
|
caps, e.g., MPI vs mpi.
|
|
"""
|
|
return Spec.is_virtual(self.name)
|
|
|
|
@staticmethod
|
|
def is_virtual(name):
|
|
"""Test if a name is virtual without requiring a Spec."""
|
|
return (name is not None) and (not spack.repo.exists(name))
|
|
|
|
@property
|
|
def concrete(self):
|
|
"""A spec is concrete if it can describe only ONE build of a package.
|
|
If any of the name, version, architecture, compiler,
|
|
variants, or depdenencies are ambiguous,then it is not concrete.
|
|
"""
|
|
if self._concrete:
|
|
return True
|
|
|
|
self._concrete = bool(not self.virtual and
|
|
self.namespace is not None and
|
|
self.versions.concrete and
|
|
self.variants.concrete and
|
|
self.architecture and
|
|
self.architecture.concrete and
|
|
self.compiler and self.compiler.concrete and
|
|
self.compiler_flags.concrete and
|
|
self._dependencies.concrete)
|
|
return self._concrete
|
|
|
|
def traverse(self, visited=None, deptype=None, **kwargs):
|
|
traversal = self.traverse_with_deptype(visited=visited,
|
|
deptype=deptype,
|
|
**kwargs)
|
|
if kwargs.get('depth', False):
|
|
return [(s[0], s[1].spec) for s in traversal]
|
|
else:
|
|
return [s.spec for s in traversal]
|
|
|
|
def traverse_with_deptype(self, visited=None, d=0, deptype=None,
|
|
deptype_query=None, _self_deptype=None,
|
|
**kwargs):
|
|
"""Generic traversal of the DAG represented by this spec.
|
|
This will yield each node in the spec. Options:
|
|
|
|
order [=pre|post]
|
|
Order to traverse spec nodes. Defaults to preorder traversal.
|
|
Options are:
|
|
|
|
'pre': Pre-order traversal; each node is yielded before its
|
|
children in the dependency DAG.
|
|
'post': Post-order traversal; each node is yielded after its
|
|
children in the dependency DAG.
|
|
|
|
cover [=nodes|edges|paths]
|
|
Determines how extensively to cover the dag. Possible values:
|
|
|
|
'nodes': Visit each node in the dag only once. Every node
|
|
yielded by this function will be unique.
|
|
'edges': If a node has been visited once but is reached along a
|
|
new path from the root, yield it but do not descend
|
|
into it. This traverses each 'edge' in the DAG once.
|
|
'paths': Explore every unique path reachable from the root.
|
|
This descends into visited subtrees and will yield
|
|
nodes twice if they're reachable by multiple paths.
|
|
|
|
depth [=False]
|
|
Defaults to False. When True, yields not just nodes in the
|
|
spec, but also their depth from the root in a (depth, node)
|
|
tuple.
|
|
|
|
key [=id]
|
|
Allow a custom key function to track the identity of nodes
|
|
in the traversal.
|
|
|
|
root [=True]
|
|
If False, this won't yield the root node, just its descendents.
|
|
|
|
direction [=children|parents]
|
|
If 'children', does a traversal of this spec's children. If
|
|
'parents', traverses upwards in the DAG towards the root.
|
|
|
|
"""
|
|
# get initial values for kwargs
|
|
depth = kwargs.get('depth', False)
|
|
key_fun = kwargs.get('key', id)
|
|
if isinstance(key_fun, basestring):
|
|
key_fun = attrgetter(key_fun)
|
|
yield_root = kwargs.get('root', True)
|
|
cover = kwargs.get('cover', 'nodes')
|
|
direction = kwargs.get('direction', 'children')
|
|
order = kwargs.get('order', 'pre')
|
|
|
|
if deptype is None:
|
|
deptype = alldeps
|
|
|
|
if deptype_query is None:
|
|
deptype_query = ('link', 'run')
|
|
|
|
# Make sure kwargs have legal values; raise ValueError if not.
|
|
def validate(name, val, allowed_values):
|
|
if val not in allowed_values:
|
|
raise ValueError("Invalid value for %s: %s. Choices are %s"
|
|
% (name, val, ",".join(allowed_values)))
|
|
validate('cover', cover, ('nodes', 'edges', 'paths'))
|
|
validate('direction', direction, ('children', 'parents'))
|
|
validate('order', order, ('pre', 'post'))
|
|
|
|
if visited is None:
|
|
visited = set()
|
|
key = key_fun(self)
|
|
|
|
# Node traversal does not yield visited nodes.
|
|
if key in visited and cover == 'nodes':
|
|
return
|
|
|
|
def return_val(res):
|
|
return (d, res) if depth else res
|
|
|
|
yield_me = yield_root or d > 0
|
|
|
|
# Preorder traversal yields before successors
|
|
if yield_me and order == 'pre':
|
|
yield return_val(DependencySpec(self, _self_deptype))
|
|
|
|
deps = self.dependencies_dict(deptype)
|
|
|
|
# Edge traversal yields but skips children of visited nodes
|
|
if not (key in visited and cover == 'edges'):
|
|
# This code determines direction and yields the children/parents
|
|
|
|
successors = deps
|
|
if direction == 'parents':
|
|
successors = self.dependents_dict(deptype)
|
|
|
|
visited.add(key)
|
|
for name in sorted(successors):
|
|
child = successors[name]
|
|
children = child.spec.traverse_with_deptype(
|
|
visited, d=d + 1, deptype=deptype,
|
|
deptype_query=deptype_query,
|
|
_self_deptype=child.deptypes, **kwargs)
|
|
for elt in children:
|
|
yield elt
|
|
|
|
# Postorder traversal yields after successors
|
|
if yield_me and order == 'post':
|
|
yield return_val(DependencySpec(self, _self_deptype))
|
|
|
|
@property
|
|
def short_spec(self):
|
|
"""Returns a version of the spec with the dependencies hashed
|
|
instead of completely enumerated."""
|
|
return self.format('$_$@$%@$+$=$#')
|
|
|
|
@property
|
|
def cshort_spec(self):
|
|
"""Returns a version of the spec with the dependencies hashed
|
|
instead of completely enumerated."""
|
|
return self.format('$_$@$%@$+$=$#', color=True)
|
|
|
|
@property
|
|
def prefix(self):
|
|
return Prefix(spack.store.layout.path_for_spec(self))
|
|
|
|
def dag_hash(self, length=None):
|
|
"""Return a hash of the entire spec DAG, including connectivity."""
|
|
if self._hash:
|
|
return self._hash[:length]
|
|
else:
|
|
yaml_text = syaml.dump(
|
|
self.to_node_dict(), default_flow_style=True, width=maxint)
|
|
sha = hashlib.sha1(yaml_text)
|
|
b32_hash = base64.b32encode(sha.digest()).lower()
|
|
if self.concrete:
|
|
self._hash = b32_hash
|
|
return b32_hash[:length]
|
|
|
|
def dag_hash_bit_prefix(self, bits):
|
|
"""Get the first <bits> bits of the DAG hash as an integer type."""
|
|
return base32_prefix_bits(self.dag_hash(), bits)
|
|
|
|
def to_node_dict(self):
|
|
d = syaml_dict()
|
|
|
|
if self.versions:
|
|
d.update(self.versions.to_dict())
|
|
|
|
if self.architecture:
|
|
d.update(self.architecture.to_dict())
|
|
|
|
if self.compiler:
|
|
d.update(self.compiler.to_dict())
|
|
|
|
if self.namespace:
|
|
d['namespace'] = self.namespace
|
|
|
|
params = syaml_dict(sorted(
|
|
(name, v.value) for name, v in self.variants.items()))
|
|
params.update(sorted(self.compiler_flags.items()))
|
|
if params:
|
|
d['parameters'] = params
|
|
|
|
# TODO: restore build dependencies here once we have less picky
|
|
# TODO: concretization.
|
|
deps = self.dependencies_dict(deptype=('link', 'run'))
|
|
if deps:
|
|
d['dependencies'] = syaml_dict([
|
|
(name,
|
|
syaml_dict([
|
|
('hash', dspec.spec.dag_hash()),
|
|
('type', sorted(str(s) for s in dspec.deptypes))])
|
|
) for name, dspec in sorted(deps.items())
|
|
])
|
|
|
|
return syaml_dict([(self.name, d)])
|
|
|
|
def to_dict(self):
|
|
node_list = []
|
|
for s in self.traverse(order='pre', deptype=('link', 'run')):
|
|
node = s.to_node_dict()
|
|
node[s.name]['hash'] = s.dag_hash()
|
|
node_list.append(node)
|
|
|
|
return syaml_dict([('spec', node_list)])
|
|
|
|
def to_yaml(self, stream=None):
|
|
return syaml.dump(
|
|
self.to_dict(), stream=stream, default_flow_style=False)
|
|
|
|
def to_json(self, stream=None):
|
|
return sjson.dump(self.to_dict(), stream)
|
|
|
|
@staticmethod
|
|
def from_node_dict(node):
|
|
name = next(iter(node))
|
|
node = node[name]
|
|
|
|
spec = Spec(name)
|
|
spec.namespace = node.get('namespace', None)
|
|
spec._hash = node.get('hash', None)
|
|
|
|
if 'version' in node or 'versions' in node:
|
|
spec.versions = VersionList.from_dict(node)
|
|
|
|
if 'arch' in node:
|
|
spec.architecture = ArchSpec.from_dict(node)
|
|
|
|
if 'compiler' in node:
|
|
spec.compiler = CompilerSpec.from_dict(node)
|
|
else:
|
|
spec.compiler = None
|
|
|
|
if 'parameters' in node:
|
|
for name, value in node['parameters'].items():
|
|
if name in _valid_compiler_flags:
|
|
spec.compiler_flags[name] = value
|
|
else:
|
|
spec.variants[name] = VariantSpec(name, value)
|
|
|
|
elif 'variants' in node:
|
|
for name, value in node['variants'].items():
|
|
spec.variants[name] = VariantSpec(name, value)
|
|
for name in FlagMap.valid_compiler_flags():
|
|
spec.compiler_flags[name] = []
|
|
|
|
# Don't read dependencies here; from_node_dict() is used by
|
|
# from_yaml() to read the root *and* each dependency spec.
|
|
|
|
return spec
|
|
|
|
@staticmethod
|
|
def read_yaml_dep_specs(dependency_dict):
|
|
"""Read the DependencySpec portion of a YAML-formatted Spec.
|
|
|
|
This needs to be backward-compatible with older spack spec
|
|
formats so that reindex will work on old specs/databases.
|
|
"""
|
|
for dep_name, elt in dependency_dict.items():
|
|
if isinstance(elt, basestring):
|
|
# original format, elt is just the dependency hash.
|
|
dag_hash, deptypes = elt, ['build', 'link']
|
|
elif isinstance(elt, tuple):
|
|
# original deptypes format: (used tuples, not future-proof)
|
|
dag_hash, deptypes = elt
|
|
elif isinstance(elt, dict):
|
|
# new format: elements of dependency spec are keyed.
|
|
dag_hash, deptypes = elt['hash'], elt['type']
|
|
else:
|
|
raise SpecError("Couldn't parse dependency types in spec.")
|
|
|
|
yield dep_name, dag_hash, list(deptypes)
|
|
|
|
@staticmethod
|
|
def from_dict(data):
|
|
"""Construct a spec from YAML.
|
|
|
|
Parameters:
|
|
data -- a nested dict/list data structure read from YAML or JSON.
|
|
"""
|
|
nodes = data['spec']
|
|
|
|
# Read nodes out of list. Root spec is the first element;
|
|
# dependencies are the following elements.
|
|
dep_list = [Spec.from_node_dict(node) for node in nodes]
|
|
if not dep_list:
|
|
raise SpecError("YAML spec contains no nodes.")
|
|
deps = dict((spec.name, spec) for spec in dep_list)
|
|
spec = dep_list[0]
|
|
|
|
for node in nodes:
|
|
# get dependency dict from the node.
|
|
name = next(iter(node))
|
|
|
|
if 'dependencies' not in node[name]:
|
|
continue
|
|
|
|
yaml_deps = node[name]['dependencies']
|
|
for dname, dhash, dtypes in Spec.read_yaml_dep_specs(yaml_deps):
|
|
# Fill in dependencies by looking them up by name in deps dict
|
|
deps[name]._dependencies[dname] = DependencySpec(
|
|
deps[dname], set(dtypes))
|
|
|
|
return spec
|
|
|
|
@staticmethod
|
|
def from_yaml(stream):
|
|
"""Construct a spec from YAML.
|
|
|
|
Parameters:
|
|
stream -- string or file object to read from.
|
|
"""
|
|
try:
|
|
data = syaml.load(stream)
|
|
return Spec.from_dict(data)
|
|
except MarkedYAMLError as e:
|
|
raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))
|
|
|
|
@staticmethod
|
|
def from_json(stream):
|
|
"""Construct a spec from JSON.
|
|
|
|
Parameters:
|
|
stream -- string or file object to read from.
|
|
"""
|
|
try:
|
|
data = sjson.load(stream)
|
|
return Spec.from_dict(data)
|
|
except Exception as e:
|
|
raise sjson.SpackJSONError("error parsing JSON spec:", str(e))
|
|
|
|
def build_dep(self):
|
|
# If this spec is the root, it will automatically be included in
|
|
# traverse
|
|
return not (self.root in
|
|
self.traverse(
|
|
deptype=('link', 'run'), direction='parents'))
|
|
|
|
def link_root(self):
|
|
parents = list(self.traverse(deptype=('link',), direction='parents',
|
|
order='pre'))
|
|
return parents[-1]
|
|
|
|
def disjoint_build_tree(self):
|
|
link_root = self.link_root()
|
|
build_subtree = list(link_root.traverse(direction='children'))
|
|
return all(x.build_dep() for x in build_subtree)
|
|
|
|
def _concretize_helper(self, presets=None, visited=None):
|
|
"""Recursive helper function for concretize().
|
|
This concretizes everything bottom-up. As things are
|
|
concretized, they're added to the presets, and ancestors
|
|
will prefer the settings of their children.
|
|
"""
|
|
if presets is None:
|
|
presets = {}
|
|
if visited is None:
|
|
visited = set()
|
|
|
|
if self.name in visited:
|
|
return False
|
|
|
|
changed = False
|
|
|
|
# Concretize deps first -- this is a bottom-up process.
|
|
for name in sorted(self._dependencies.keys()):
|
|
dep = self._dependencies[name]
|
|
changed |= dep.spec._concretize_helper(presets, visited)
|
|
|
|
if self.name in presets:
|
|
changed |= self.constrain(presets[self.name])
|
|
else:
|
|
# Concretize virtual dependencies last. Because they're added
|
|
# to presets below, their constraints will all be merged, but we'll
|
|
# still need to select a concrete package later.
|
|
if not self.virtual:
|
|
changed |= any(
|
|
(spack.concretizer.concretize_architecture(self),
|
|
spack.concretizer.concretize_compiler(self),
|
|
spack.concretizer.concretize_compiler_flags(
|
|
self), # has to be concretized after compiler
|
|
spack.concretizer.concretize_version(self),
|
|
spack.concretizer.concretize_variants(self)))
|
|
presets[self.name] = self
|
|
|
|
visited.add(self.name)
|
|
return changed
|
|
|
|
def _replace_with(self, concrete):
|
|
"""Replace this virtual spec with a concrete spec."""
|
|
assert(self.virtual)
|
|
for name, dep_spec in self._dependents.items():
|
|
dependent = dep_spec.spec
|
|
deptypes = dep_spec.deptypes
|
|
|
|
# remove self from all dependents.
|
|
del dependent._dependencies[self.name]
|
|
|
|
# add the replacement, unless it is already a dep of dependent.
|
|
if concrete.name not in dependent._dependencies:
|
|
dependent._add_dependency(concrete, deptypes)
|
|
|
|
def _replace_node(self, replacement):
|
|
"""Replace this spec with another.
|
|
|
|
Connects all dependents of this spec to its replacement, and
|
|
disconnects this spec from any dependencies it has. New spec
|
|
will have any dependencies the replacement had, and may need
|
|
to be normalized.
|
|
|
|
"""
|
|
for name, dep_spec in self._dependents.items():
|
|
dependent = dep_spec.spec
|
|
deptypes = dep_spec.deptypes
|
|
del dependent._dependencies[self.name]
|
|
dependent._add_dependency(replacement, deptypes)
|
|
|
|
for name, dep_spec in self._dependencies.items():
|
|
del dep_spec.spec.dependents[self.name]
|
|
del self._dependencies[dep.name]
|
|
|
|
def _expand_virtual_packages(self):
|
|
"""Find virtual packages in this spec, replace them with providers,
|
|
and normalize again to include the provider's (potentially virtual)
|
|
dependencies. Repeat until there are no virtual deps.
|
|
|
|
Precondition: spec is normalized.
|
|
|
|
.. todo::
|
|
|
|
If a provider depends on something that conflicts with
|
|
other dependencies in the spec being expanded, this can
|
|
produce a conflicting spec. For example, if mpich depends
|
|
on hwloc@:1.3 but something in the spec needs hwloc1.4:,
|
|
then we should choose an MPI other than mpich. Cases like
|
|
this are infrequent, but should implement this before it is
|
|
a problem.
|
|
"""
|
|
# Make an index of stuff this spec already provides
|
|
# XXX(deptype): 'link' and 'run'?
|
|
self_index = ProviderIndex(self.traverse(), restrict=True)
|
|
changed = False
|
|
done = False
|
|
|
|
while not done:
|
|
done = True
|
|
# XXX(deptype): 'link' and 'run'?
|
|
for spec in list(self.traverse()):
|
|
replacement = None
|
|
if spec.virtual:
|
|
replacement = self._find_provider(spec, self_index)
|
|
if replacement:
|
|
# TODO: may break if in-place on self but
|
|
# shouldn't happen if root is traversed first.
|
|
spec._replace_with(replacement)
|
|
done = False
|
|
break
|
|
|
|
if not replacement:
|
|
# Get a list of possible replacements in order of
|
|
# preference.
|
|
candidates = spack.concretizer.choose_virtual_or_external(
|
|
spec)
|
|
|
|
# Try the replacements in order, skipping any that cause
|
|
# satisfiability problems.
|
|
for replacement in candidates:
|
|
if replacement is spec:
|
|
break
|
|
|
|
# Replace spec with the candidate and normalize
|
|
copy = self.copy()
|
|
copy[spec.name]._dup(replacement.copy(deps=False))
|
|
|
|
try:
|
|
# If there are duplicate providers or duplicate
|
|
# provider deps, consolidate them and merge
|
|
# constraints.
|
|
copy.normalize(force=True)
|
|
break
|
|
except SpecError:
|
|
# On error, we'll try the next replacement.
|
|
continue
|
|
|
|
# If replacement is external then trim the dependencies
|
|
if replacement.external or replacement.external_module:
|
|
if (spec._dependencies):
|
|
changed = True
|
|
spec._dependencies = DependencyMap()
|
|
replacement._dependencies = DependencyMap()
|
|
replacement.architecture = self.architecture
|
|
|
|
# TODO: could this and the stuff in _dup be cleaned up?
|
|
def feq(cfield, sfield):
|
|
return (not cfield) or (cfield == sfield)
|
|
|
|
if replacement is spec or (
|
|
feq(replacement.name, spec.name) and
|
|
feq(replacement.versions, spec.versions) and
|
|
feq(replacement.compiler, spec.compiler) and
|
|
feq(replacement.architecture, spec.architecture) and
|
|
feq(replacement._dependencies, spec._dependencies) and
|
|
feq(replacement.variants, spec.variants) and
|
|
feq(replacement.external, spec.external) and
|
|
feq(replacement.external_module,
|
|
spec.external_module)):
|
|
continue
|
|
# Refine this spec to the candidate. This uses
|
|
# replace_with AND dup so that it can work in
|
|
# place. TODO: make this more efficient.
|
|
if spec.virtual:
|
|
spec._replace_with(replacement)
|
|
changed = True
|
|
if spec._dup(replacement, deps=False, cleardeps=False):
|
|
changed = True
|
|
|
|
self_index.update(spec)
|
|
done = False
|
|
break
|
|
|
|
return changed
|
|
|
|
def concretize(self):
|
|
"""A spec is concrete if it describes one build of a package uniquely.
|
|
This will ensure that this spec is concrete.
|
|
|
|
If this spec could describe more than one version, variant, or build
|
|
of a package, this will add constraints to make it concrete.
|
|
|
|
Some rigorous validation and checks are also performed on the spec.
|
|
Concretizing ensures that it is self-consistent and that it's
|
|
consistent with requirements of its pacakges. See flatten() and
|
|
normalize() for more details on this.
|
|
"""
|
|
if not self.name:
|
|
raise SpecError("Attempting to concretize anonymous spec")
|
|
|
|
if self._concrete:
|
|
return
|
|
|
|
changed = True
|
|
force = False
|
|
|
|
while changed:
|
|
changes = (self.normalize(force),
|
|
self._expand_virtual_packages(),
|
|
self._concretize_helper())
|
|
changed = any(changes)
|
|
force = True
|
|
|
|
for s in self.traverse(deptype_query=alldeps):
|
|
# After concretizing, assign namespaces to anything left.
|
|
# Note that this doesn't count as a "change". The repository
|
|
# configuration is constant throughout a spack run, and
|
|
# normalize and concretize evaluate Packages using Repo.get(),
|
|
# which respects precedence. So, a namespace assignment isn't
|
|
# changing how a package name would have been interpreted and
|
|
# we can do it as late as possible to allow as much
|
|
# compatibility across repositories as possible.
|
|
if s.namespace is None:
|
|
s.namespace = spack.repo.repo_for_pkg(s.name).namespace
|
|
|
|
for s in self.traverse(root=False):
|
|
if s.external_module:
|
|
compiler = spack.compilers.compiler_for_spec(
|
|
s.compiler, s.architecture)
|
|
for mod in compiler.modules:
|
|
load_module(mod)
|
|
|
|
s.external = get_path_from_module(s.external_module)
|
|
|
|
# Mark everything in the spec as concrete, as well.
|
|
self._mark_concrete()
|
|
|
|
def _mark_concrete(self, value=True):
|
|
"""Mark this spec and its dependencies as concrete.
|
|
|
|
Only for internal use -- client code should use "concretize"
|
|
unless there is a need to force a spec to be concrete.
|
|
"""
|
|
for s in self.traverse(deptype_query=alldeps):
|
|
s._normal = value
|
|
s._concrete = value
|
|
|
|
def concretized(self):
|
|
"""This is a non-destructive version of concretize(). First clones,
|
|
then returns a concrete version of this package without modifying
|
|
this package. """
|
|
clone = self.copy()
|
|
clone.concretize()
|
|
return clone
|
|
|
|
def flat_dependencies(self, **kwargs):
|
|
flat_deps = DependencyMap()
|
|
flat_deps_deptypes = self.flat_dependencies_with_deptype(**kwargs)
|
|
for name, depspec in flat_deps_deptypes.items():
|
|
flat_deps[name] = depspec.spec
|
|
return flat_deps
|
|
|
|
def flat_dependencies_with_deptype(self, **kwargs):
|
|
"""Return a DependencyMap containing all of this spec's
|
|
dependencies with their constraints merged.
|
|
|
|
If copy is True, returns merged copies of its dependencies
|
|
without modifying the spec it's called on.
|
|
|
|
If copy is False, clears this spec's dependencies and
|
|
returns them.
|
|
"""
|
|
copy = kwargs.get('copy', True)
|
|
deptype_query = kwargs.get('deptype_query')
|
|
|
|
flat_deps = DependencyMap()
|
|
try:
|
|
deptree = self.traverse_with_deptype(root=False,
|
|
deptype_query=deptype_query)
|
|
for depspec in deptree:
|
|
spec = depspec.spec
|
|
deptypes = depspec.deptypes
|
|
|
|
if spec.name not in flat_deps:
|
|
if copy:
|
|
dep_spec = DependencySpec(spec.copy(deps=False),
|
|
deptypes)
|
|
else:
|
|
dep_spec = DependencySpec(spec, deptypes)
|
|
flat_deps[spec.name] = dep_spec
|
|
else:
|
|
flat_deps[spec.name].spec.constrain(spec)
|
|
|
|
if not copy:
|
|
for depspec in flat_deps.values():
|
|
depspec.spec._dependencies.clear()
|
|
depspec.spec._dependents.clear()
|
|
self._dependencies.clear()
|
|
|
|
return flat_deps
|
|
|
|
except UnsatisfiableSpecError as e:
|
|
# Here, the DAG contains two instances of the same package
|
|
# with inconsistent constraints. Users cannot produce
|
|
# inconsistent specs like this on the command line: the
|
|
# parser doesn't allow it. Spack must be broken!
|
|
raise InconsistentSpecError("Invalid Spec DAG: %s" % e.message)
|
|
|
|
def index(self, deptype=None):
|
|
"""Return DependencyMap that points to all the dependencies in this
|
|
spec."""
|
|
dm = DependencyMap()
|
|
for spec in self.traverse(deptype=deptype):
|
|
dm[spec.name] = spec
|
|
return dm
|
|
|
|
def _evaluate_dependency_conditions(self, name):
|
|
"""Evaluate all the conditions on a dependency with this name.
|
|
|
|
If the package depends on <name> in this configuration, return
|
|
the dependency. If no conditions are True (and we don't
|
|
depend on it), return None.
|
|
"""
|
|
pkg = spack.repo.get(self.fullname)
|
|
conditions = pkg.dependencies[name]
|
|
|
|
# evaluate when specs to figure out constraints on the dependency.
|
|
dep = None
|
|
for when_spec, dep_spec in conditions.items():
|
|
sat = self.satisfies(when_spec, strict=True)
|
|
if sat:
|
|
if dep is None:
|
|
dep = Spec(name)
|
|
try:
|
|
dep.constrain(dep_spec)
|
|
except UnsatisfiableSpecError as e:
|
|
e.message = ("Conflicting conditional dependencies on"
|
|
"package %s for spec %s" % (self.name, self))
|
|
raise e
|
|
return dep
|
|
|
|
def _find_provider(self, vdep, provider_index):
|
|
"""Find provider for a virtual spec in the provider index.
|
|
Raise an exception if there is a conflicting virtual
|
|
dependency already in this spec.
|
|
"""
|
|
assert(vdep.virtual)
|
|
providers = provider_index.providers_for(vdep)
|
|
|
|
# If there is a provider for the vpkg, then use that instead of
|
|
# the virtual package.
|
|
if providers:
|
|
# Remove duplicate providers that can concretize to the same
|
|
# result.
|
|
for provider in providers:
|
|
for spec in providers:
|
|
if spec is not provider and provider.satisfies(spec):
|
|
providers.remove(spec)
|
|
# Can't have multiple providers for the same thing in one spec.
|
|
if len(providers) > 1:
|
|
raise MultipleProviderError(vdep, providers)
|
|
return providers[0]
|
|
else:
|
|
# The user might have required something insufficient for
|
|
# pkg_dep -- so we'll get a conflict. e.g., user asked for
|
|
# mpi@:1.1 but some package required mpi@2.1:.
|
|
required = provider_index.providers_for(vdep.name)
|
|
if len(required) > 1:
|
|
raise MultipleProviderError(vdep, required)
|
|
elif required:
|
|
raise UnsatisfiableProviderSpecError(required[0], vdep)
|
|
|
|
def _merge_dependency(self, dep, deptypes, visited, spec_deps,
|
|
provider_index):
|
|
"""Merge the dependency into this spec.
|
|
|
|
This is the core of normalize(). There are some basic steps:
|
|
|
|
* If dep is virtual, evaluate whether it corresponds to an
|
|
existing concrete dependency, and merge if so.
|
|
|
|
* If it's real and it provides some virtual dep, see if it provides
|
|
what some virtual dependency wants and merge if so.
|
|
|
|
* Finally, if none of the above, merge dependency and its
|
|
constraints into this spec.
|
|
|
|
This method returns True if the spec was changed, False otherwise.
|
|
"""
|
|
changed = False
|
|
|
|
# If it's a virtual dependency, try to find an existing
|
|
# provider in the spec, and merge that.
|
|
if dep.virtual:
|
|
visited.add(dep.name)
|
|
provider = self._find_provider(dep, provider_index)
|
|
if provider:
|
|
dep = provider
|
|
else:
|
|
index = ProviderIndex([dep], restrict=True)
|
|
for vspec in (v.spec
|
|
for v in spec_deps.values()
|
|
if v.spec.virtual):
|
|
if index.providers_for(vspec):
|
|
vspec._replace_with(dep)
|
|
del spec_deps[vspec.name]
|
|
changed = True
|
|
else:
|
|
required = index.providers_for(vspec.name)
|
|
if required:
|
|
raise UnsatisfiableProviderSpecError(required[0], dep)
|
|
provider_index.update(dep)
|
|
# If the spec isn't already in the set of dependencies, clone
|
|
# it from the package description.
|
|
if dep.name not in spec_deps:
|
|
spec_deps[dep.name] = DependencySpec(dep.copy(), deptypes)
|
|
changed = True
|
|
# Constrain package information with spec info
|
|
try:
|
|
changed |= spec_deps[dep.name].spec.constrain(dep)
|
|
|
|
except UnsatisfiableSpecError as e:
|
|
e.message = "Invalid spec: '%s'. "
|
|
e.message += "Package %s requires %s %s, but spec asked for %s"
|
|
e.message %= (spec_deps[dep.name].spec, dep.name,
|
|
e.constraint_type, e.required, e.provided)
|
|
raise e
|
|
|
|
# Add merged spec to my deps and recurse
|
|
dependency = spec_deps[dep.name]
|
|
if dep.name not in self._dependencies:
|
|
self._add_dependency(dependency.spec, dependency.deptypes)
|
|
|
|
changed |= dependency.spec._normalize_helper(
|
|
visited, spec_deps, provider_index)
|
|
return changed
|
|
|
|
def _normalize_helper(self, visited, spec_deps, provider_index):
|
|
"""Recursive helper function for _normalize."""
|
|
if self.name in visited:
|
|
return False
|
|
visited.add(self.name)
|
|
|
|
# if we descend into a virtual spec, there's nothing more
|
|
# to normalize. Concretize will finish resolving it later.
|
|
if self.virtual or self.external or self.external_module:
|
|
return False
|
|
|
|
# Combine constraints from package deps with constraints from
|
|
# the spec, until nothing changes.
|
|
any_change = False
|
|
changed = True
|
|
|
|
pkg = spack.repo.get(self.fullname)
|
|
while changed:
|
|
changed = False
|
|
for dep_name in pkg.dependencies:
|
|
# Do we depend on dep_name? If so pkg_dep is not None.
|
|
pkg_dep = self._evaluate_dependency_conditions(dep_name)
|
|
deptypes = pkg._deptypes[dep_name]
|
|
# If pkg_dep is a dependency, merge it.
|
|
if pkg_dep:
|
|
changed |= self._merge_dependency(
|
|
pkg_dep, deptypes, visited, spec_deps, provider_index)
|
|
any_change |= changed
|
|
|
|
return any_change
|
|
|
|
def normalize(self, force=False):
|
|
"""When specs are parsed, any dependencies specified are hanging off
|
|
the root, and ONLY the ones that were explicitly provided are there.
|
|
Normalization turns a partial flat spec into a DAG, where:
|
|
|
|
1. Known dependencies of the root package are in the DAG.
|
|
2. Each node's dependencies dict only contains its known direct
|
|
deps.
|
|
3. There is only ONE unique spec for each package in the DAG.
|
|
|
|
* This includes virtual packages. If there a non-virtual
|
|
package that provides a virtual package that is in the spec,
|
|
then we replace the virtual package with the non-virtual one.
|
|
|
|
TODO: normalize should probably implement some form of cycle
|
|
detection, to ensure that the spec is actually a DAG.
|
|
"""
|
|
if not self.name:
|
|
raise SpecError("Attempting to normalize anonymous spec")
|
|
|
|
if self._normal and not force:
|
|
return False
|
|
|
|
# avoid any assumptions about concreteness when forced
|
|
if force:
|
|
self._mark_concrete(False)
|
|
|
|
# Ensure first that all packages & compilers in the DAG exist.
|
|
self.validate_names()
|
|
# Get all the dependencies into one DependencyMap
|
|
spec_deps = self.flat_dependencies_with_deptype(
|
|
copy=False, deptype_query=alldeps)
|
|
|
|
# Initialize index of virtual dependency providers if
|
|
# concretize didn't pass us one already
|
|
provider_index = ProviderIndex(
|
|
[s.spec for s in spec_deps.values()], restrict=True)
|
|
|
|
# traverse the package DAG and fill out dependencies according
|
|
# to package files & their 'when' specs
|
|
visited = set()
|
|
any_change = self._normalize_helper(visited, spec_deps, provider_index)
|
|
|
|
# If there are deps specified but not visited, they're not
|
|
# actually deps of this package. Raise an error.
|
|
extra = set(spec_deps.keys()).difference(visited)
|
|
if extra:
|
|
raise InvalidDependencyError(
|
|
self.name + " does not depend on " + comma_or(extra))
|
|
|
|
# Mark the spec as normal once done.
|
|
self._normal = True
|
|
return any_change
|
|
|
|
def normalized(self):
|
|
"""
|
|
Return a normalized copy of this spec without modifying this spec.
|
|
"""
|
|
clone = self.copy()
|
|
clone.normalize()
|
|
return clone
|
|
|
|
def validate_names(self):
|
|
"""This checks that names of packages and compilers in this spec are real.
|
|
If they're not, it will raise either UnknownPackageError or
|
|
UnsupportedCompilerError.
|
|
"""
|
|
for spec in self.traverse():
|
|
# raise an UnknownPackageError if the spec's package isn't real.
|
|
if (not spec.virtual) and spec.name:
|
|
spack.repo.get(spec.fullname)
|
|
|
|
# validate compiler in addition to the package name.
|
|
if spec.compiler:
|
|
if not compilers.supported(spec.compiler):
|
|
raise UnsupportedCompilerError(spec.compiler.name)
|
|
|
|
# Ensure that variants all exist.
|
|
for vname, variant in spec.variants.items():
|
|
if vname not in spec.package_class.variants:
|
|
raise UnknownVariantError(spec.name, vname)
|
|
|
|
def constrain(self, other, deps=True):
|
|
"""Merge the constraints of other with self.
|
|
|
|
Returns True if the spec changed as a result, False if not.
|
|
"""
|
|
other = self._autospec(other)
|
|
|
|
if not (self.name == other.name or
|
|
(not self.name) or
|
|
(not other.name)):
|
|
raise UnsatisfiableSpecNameError(self.name, other.name)
|
|
|
|
if (other.namespace is not None and
|
|
self.namespace is not None and
|
|
other.namespace != self.namespace):
|
|
raise UnsatisfiableSpecNameError(self.fullname, other.fullname)
|
|
|
|
if not self.versions.overlaps(other.versions):
|
|
raise UnsatisfiableVersionSpecError(self.versions, other.versions)
|
|
|
|
for v in other.variants:
|
|
if (v in self.variants and
|
|
self.variants[v].value != other.variants[v].value):
|
|
raise UnsatisfiableVariantSpecError(self.variants[v],
|
|
other.variants[v])
|
|
|
|
# TODO: Check out the logic here
|
|
sarch, oarch = self.architecture, other.architecture
|
|
if sarch is not None and oarch is not None:
|
|
if sarch.platform is not None and oarch.platform is not None:
|
|
if sarch.platform != oarch.platform:
|
|
raise UnsatisfiableArchitectureSpecError(sarch, oarch)
|
|
if sarch.platform_os is not None and oarch.platform_os is not None:
|
|
if sarch.platform_os != oarch.platform_os:
|
|
raise UnsatisfiableArchitectureSpecError(sarch, oarch)
|
|
if sarch.target is not None and oarch.target is not None:
|
|
if sarch.target != oarch.target:
|
|
raise UnsatisfiableArchitectureSpecError(sarch, oarch)
|
|
|
|
changed = False
|
|
if self.compiler is not None and other.compiler is not None:
|
|
changed |= self.compiler.constrain(other.compiler)
|
|
elif self.compiler is None:
|
|
changed |= (self.compiler != other.compiler)
|
|
self.compiler = other.compiler
|
|
|
|
changed |= self.versions.intersect(other.versions)
|
|
changed |= self.variants.constrain(other.variants)
|
|
|
|
changed |= self.compiler_flags.constrain(other.compiler_flags)
|
|
|
|
old = str(self.architecture)
|
|
sarch, oarch = self.architecture, other.architecture
|
|
if sarch is None or other.architecture is None:
|
|
self.architecture = sarch or oarch
|
|
else:
|
|
if sarch.platform is None or oarch.platform is None:
|
|
self.architecture.platform = sarch.platform or oarch.platform
|
|
if sarch.platform_os is None or oarch.platform_os is None:
|
|
sarch.platform_os = sarch.platform_os or oarch.platform_os
|
|
if sarch.target is None or oarch.target is None:
|
|
sarch.target = sarch.target or oarch.target
|
|
changed |= (str(self.architecture) != old)
|
|
|
|
if deps:
|
|
changed |= self._constrain_dependencies(other)
|
|
|
|
return changed
|
|
|
|
def _constrain_dependencies(self, other):
|
|
"""Apply constraints of other spec's dependencies to this spec."""
|
|
other = self._autospec(other)
|
|
|
|
if not self._dependencies or not other._dependencies:
|
|
return False
|
|
|
|
# TODO: might want more detail than this, e.g. specific deps
|
|
# in violation. if this becomes a priority get rid of this
|
|
# check and be more specific about what's wrong.
|
|
if not other.satisfies_dependencies(self):
|
|
raise UnsatisfiableDependencySpecError(other, self)
|
|
|
|
# Handle common first-order constraints directly
|
|
changed = False
|
|
for name in self.common_dependencies(other):
|
|
changed |= self[name].constrain(other[name], deps=False)
|
|
|
|
# Update with additional constraints from other spec
|
|
for name in other.dep_difference(self):
|
|
dep_spec_copy = other.get_dependency(name)
|
|
dep_copy = dep_spec_copy.spec
|
|
deptypes = dep_spec_copy.deptypes
|
|
self._add_dependency(dep_copy.copy(), deptypes)
|
|
changed = True
|
|
|
|
return changed
|
|
|
|
def common_dependencies(self, other):
|
|
"""Return names of dependencies that self an other have in common."""
|
|
# XXX(deptype): handle deptypes via deptype kwarg.
|
|
common = set(
|
|
s.name for s in self.traverse(root=False))
|
|
common.intersection_update(
|
|
s.name for s in other.traverse(root=False))
|
|
return common
|
|
|
|
def constrained(self, other, deps=True):
|
|
"""Return a constrained copy without modifying this spec."""
|
|
clone = self.copy(deps=deps)
|
|
clone.constrain(other, deps)
|
|
return clone
|
|
|
|
def dep_difference(self, other):
|
|
"""Returns dependencies in self that are not in other."""
|
|
mine = set(s.name for s in self.traverse(root=False))
|
|
mine.difference_update(
|
|
s.name for s in other.traverse(root=False))
|
|
return mine
|
|
|
|
def _autospec(self, spec_like):
|
|
"""
|
|
Used to convert arguments to specs. If spec_like is a spec, returns
|
|
it. If it's a string, tries to parse a string. If that fails, tries
|
|
to parse a local spec from it (i.e. name is assumed to be self's name).
|
|
"""
|
|
if isinstance(spec_like, spack.spec.Spec):
|
|
return spec_like
|
|
|
|
try:
|
|
spec = spack.spec.Spec(spec_like)
|
|
if not spec.name:
|
|
raise SpecError(
|
|
"anonymous package -- this will always be handled")
|
|
return spec
|
|
except SpecError:
|
|
return parse_anonymous_spec(spec_like, self.name)
|
|
|
|
def satisfies(self, other, deps=True, strict=False):
|
|
"""Determine if this spec satisfies all constraints of another.
|
|
|
|
There are two senses for satisfies:
|
|
|
|
* `loose` (default): the absence of a constraint in self
|
|
implies that it *could* be satisfied by other, so we only
|
|
check that there are no conflicts with other for
|
|
constraints that this spec actually has.
|
|
|
|
* `strict`: strict means that we *must* meet all the
|
|
constraints specified on other.
|
|
"""
|
|
other = self._autospec(other)
|
|
|
|
# A concrete provider can satisfy a virtual dependency.
|
|
if not self.virtual and other.virtual:
|
|
pkg = spack.repo.get(self.fullname)
|
|
if pkg.provides(other.name):
|
|
for provided, when_spec in pkg.provided.items():
|
|
if self.satisfies(when_spec, deps=False, strict=strict):
|
|
if provided.satisfies(other):
|
|
return True
|
|
return False
|
|
|
|
# Otherwise, first thing we care about is whether the name matches
|
|
if self.name != other.name and self.name and other.name:
|
|
return False
|
|
|
|
# namespaces either match, or other doesn't require one.
|
|
if (other.namespace is not None and
|
|
self.namespace is not None and
|
|
self.namespace != other.namespace):
|
|
return False
|
|
if self.versions and other.versions:
|
|
if not self.versions.satisfies(other.versions, strict=strict):
|
|
return False
|
|
elif strict and (self.versions or other.versions):
|
|
return False
|
|
|
|
# None indicates no constraints when not strict.
|
|
if self.compiler and other.compiler:
|
|
if not self.compiler.satisfies(other.compiler, strict=strict):
|
|
return False
|
|
elif strict and (other.compiler and not self.compiler):
|
|
return False
|
|
|
|
var_strict = strict
|
|
if (not self.name) or (not other.name):
|
|
var_strict = True
|
|
if not self.variants.satisfies(other.variants, strict=var_strict):
|
|
return False
|
|
|
|
# Architecture satisfaction is currently just string equality.
|
|
# If not strict, None means unconstrained.
|
|
if self.architecture and other.architecture:
|
|
if not self.architecture.satisfies(other.architecture, strict):
|
|
return False
|
|
elif strict and (other.architecture and not self.architecture):
|
|
return False
|
|
|
|
if not self.compiler_flags.satisfies(
|
|
other.compiler_flags,
|
|
strict=strict):
|
|
return False
|
|
|
|
# If we need to descend into dependencies, do it, otherwise we're done.
|
|
if deps:
|
|
deps_strict = strict
|
|
if not (self.name and other.name):
|
|
deps_strict = True
|
|
return self.satisfies_dependencies(other, strict=deps_strict)
|
|
else:
|
|
return True
|
|
|
|
def satisfies_dependencies(self, other, strict=False):
|
|
"""
|
|
This checks constraints on common dependencies against each other.
|
|
"""
|
|
other = self._autospec(other)
|
|
|
|
if strict:
|
|
if other._dependencies and not self._dependencies:
|
|
return False
|
|
|
|
if not all(dep in self._dependencies
|
|
for dep in other._dependencies):
|
|
return False
|
|
|
|
elif not self._dependencies or not other._dependencies:
|
|
# if either spec doesn't restrict dependencies then both are
|
|
# compatible.
|
|
return True
|
|
|
|
# Handle first-order constraints directly
|
|
for name in self.common_dependencies(other):
|
|
if not self[name].satisfies(other[name], deps=False):
|
|
return False
|
|
|
|
# For virtual dependencies, we need to dig a little deeper.
|
|
self_index = ProviderIndex(self.traverse(), restrict=True)
|
|
other_index = ProviderIndex(other.traverse(), restrict=True)
|
|
|
|
# This handles cases where there are already providers for both vpkgs
|
|
if not self_index.satisfies(other_index):
|
|
return False
|
|
|
|
# These two loops handle cases where there is an overly restrictive
|
|
# vpkg in one spec for a provider in the other (e.g., mpi@3: is not
|
|
# compatible with mpich2)
|
|
for spec in self.virtual_dependencies():
|
|
if (spec.name in other_index and
|
|
not other_index.providers_for(spec)):
|
|
return False
|
|
|
|
for spec in other.virtual_dependencies():
|
|
if spec.name in self_index and not self_index.providers_for(spec):
|
|
return False
|
|
|
|
return True
|
|
|
|
def virtual_dependencies(self):
|
|
"""Return list of any virtual deps in this spec."""
|
|
return [spec for spec in self.traverse() if spec.virtual]
|
|
|
|
def _dup(self, other, deps=True, cleardeps=True):
|
|
"""Copy the spec other into self. This is an overwriting
|
|
copy. It does not copy any dependents (parents), but by default
|
|
copies dependencies.
|
|
|
|
To duplicate an entire DAG, call _dup() on the root of the DAG.
|
|
|
|
Options:
|
|
dependencies[=True]
|
|
Whether deps should be copied too. Set to False to copy a
|
|
spec but not its dependencies.
|
|
"""
|
|
# We don't count dependencies as changes here
|
|
changed = True
|
|
if hasattr(self, 'name'):
|
|
changed = (self.name != other.name and
|
|
self.versions != other.versions and
|
|
self.architecture != other.architecture and
|
|
self.compiler != other.compiler and
|
|
self.variants != other.variants and
|
|
self._normal != other._normal and
|
|
self.concrete != other.concrete and
|
|
self.external != other.external and
|
|
self.external_module != other.external_module and
|
|
self.compiler_flags != other.compiler_flags)
|
|
|
|
# Local node attributes get copied first.
|
|
self.name = other.name
|
|
self.versions = other.versions.copy()
|
|
self.architecture = other.architecture.copy() if other.architecture \
|
|
else None
|
|
self.compiler = other.compiler.copy() if other.compiler else None
|
|
if cleardeps:
|
|
self._dependents = DependencyMap()
|
|
self._dependencies = DependencyMap()
|
|
self.compiler_flags = other.compiler_flags.copy()
|
|
self.variants = other.variants.copy()
|
|
self.variants.spec = self
|
|
self.external = other.external
|
|
self.external_module = other.external_module
|
|
self.namespace = other.namespace
|
|
|
|
self.external = other.external
|
|
self.external_module = other.external_module
|
|
|
|
# If we copy dependencies, preserve DAG structure in the new spec
|
|
if deps:
|
|
# This copies the deps from other using _dup(deps=False)
|
|
deptypes = alldeps
|
|
if isinstance(deps, (tuple, list)):
|
|
deptypes = deps
|
|
new_nodes = other.flat_dependencies(deptypes=deptypes)
|
|
new_nodes[self.name] = self
|
|
|
|
stack = [other]
|
|
while stack:
|
|
cur_spec = stack.pop(0)
|
|
new_spec = new_nodes[cur_spec.name]
|
|
|
|
for depspec in cur_spec._dependencies.values():
|
|
if not any(d in deptypes for d in depspec.deptypes):
|
|
continue
|
|
|
|
stack.append(depspec.spec)
|
|
|
|
# XXX(deptype): add any new deptypes that may have appeared
|
|
# here.
|
|
if depspec.spec.name not in new_spec._dependencies:
|
|
new_spec._add_dependency(
|
|
new_nodes[depspec.spec.name], depspec.deptypes)
|
|
|
|
# These fields are all cached results of expensive operations.
|
|
# If we preserved the original structure, we can copy them
|
|
# safely. If not, they need to be recomputed.
|
|
if deps is True or deps == alldeps:
|
|
self._hash = other._hash
|
|
self._cmp_key_cache = other._cmp_key_cache
|
|
self._normal = other._normal
|
|
self._concrete = other._concrete
|
|
else:
|
|
self._hash = None
|
|
self._cmp_key_cache = None
|
|
self._normal = False
|
|
self._concrete = False
|
|
|
|
return changed
|
|
|
|
def copy(self, deps=True):
|
|
"""Return a copy of this spec.
|
|
|
|
By default, returns a deep copy. To control how dependencies are
|
|
copied, supply:
|
|
|
|
deps=True: deep copy
|
|
|
|
deps=False: shallow copy (no dependencies)
|
|
|
|
deps=('link', 'build'):
|
|
only build and link dependencies. Similar for other deptypes.
|
|
|
|
"""
|
|
clone = Spec.__new__(Spec)
|
|
clone._dup(self, deps=deps)
|
|
return clone
|
|
|
|
@property
|
|
def version(self):
|
|
if not self.versions.concrete:
|
|
raise SpecError("Spec version is not concrete: " + str(self))
|
|
return self.versions[0]
|
|
|
|
def __getitem__(self, name):
|
|
"""Get a dependency from the spec by its name."""
|
|
for spec in self.traverse():
|
|
if spec.name == name:
|
|
return spec
|
|
|
|
if Spec.is_virtual(name):
|
|
# TODO: this is a kind of kludgy way to find providers
|
|
# TODO: should we just keep virtual deps in the DAG instead of
|
|
# TODO: removing them on concretize?
|
|
for spec in self.traverse():
|
|
if spec.virtual:
|
|
continue
|
|
if spec.package.provides(name):
|
|
return spec
|
|
|
|
raise KeyError("No spec with name %s in %s" % (name, self))
|
|
|
|
def __contains__(self, spec):
|
|
"""True if this spec satisfies the provided spec, or if any dependency
|
|
does. If the spec has no name, then we parse this one first.
|
|
"""
|
|
spec = self._autospec(spec)
|
|
for s in self.traverse():
|
|
if s.satisfies(spec, strict=True):
|
|
return True
|
|
|
|
return False
|
|
|
|
def sorted_deps(self):
|
|
"""Return a list of all dependencies sorted by name."""
|
|
deps = self.flat_dependencies()
|
|
return tuple(deps[name] for name in sorted(deps))
|
|
|
|
def _eq_dag(self, other, vs, vo):
|
|
"""Recursive helper for eq_dag and ne_dag. Does the actual DAG
|
|
traversal."""
|
|
vs.add(id(self))
|
|
vo.add(id(other))
|
|
|
|
if self.ne_node(other):
|
|
return False
|
|
|
|
if len(self._dependencies) != len(other._dependencies):
|
|
return False
|
|
|
|
ssorted = [self._dependencies[name].spec
|
|
for name in sorted(self._dependencies)]
|
|
osorted = [other._dependencies[name].spec
|
|
for name in sorted(other._dependencies)]
|
|
|
|
for s, o in zip(ssorted, osorted):
|
|
visited_s = id(s) in vs
|
|
visited_o = id(o) in vo
|
|
|
|
# Check for duplicate or non-equal dependencies
|
|
if visited_s != visited_o:
|
|
return False
|
|
|
|
# Skip visited nodes
|
|
if visited_s or visited_o:
|
|
continue
|
|
|
|
# Recursive check for equality
|
|
if not s._eq_dag(o, vs, vo):
|
|
return False
|
|
|
|
return True
|
|
|
|
def eq_dag(self, other):
|
|
"""True if the full dependency DAGs of specs are equal"""
|
|
return self._eq_dag(other, set(), set())
|
|
|
|
def ne_dag(self, other):
|
|
"""True if the full dependency DAGs of specs are not equal"""
|
|
return not self.eq_dag(other)
|
|
|
|
def _cmp_node(self):
|
|
"""Comparison key for just *this node* and not its deps."""
|
|
return (self.name,
|
|
self.namespace,
|
|
self.versions,
|
|
self.variants,
|
|
self.architecture,
|
|
self.compiler,
|
|
self.compiler_flags)
|
|
|
|
def eq_node(self, other):
|
|
"""Equality with another spec, not including dependencies."""
|
|
return self._cmp_node() == other._cmp_node()
|
|
|
|
def ne_node(self, other):
|
|
"""Inequality with another spec, not including dependencies."""
|
|
return self._cmp_node() != other._cmp_node()
|
|
|
|
def _cmp_key(self):
|
|
"""This returns a key for the spec *including* DAG structure.
|
|
|
|
The key is the concatenation of:
|
|
1. A tuple describing this node in the DAG.
|
|
2. The hash of each of this node's dependencies' cmp_keys.
|
|
"""
|
|
if self._cmp_key_cache:
|
|
return self._cmp_key_cache
|
|
|
|
dep_tuple = tuple(
|
|
(d.spec.name, hash(d.spec), tuple(sorted(d.deptypes)))
|
|
for name, d in sorted(self._dependencies.items()))
|
|
|
|
key = (self._cmp_node(), dep_tuple)
|
|
if self._concrete:
|
|
self._cmp_key_cache = key
|
|
return key
|
|
|
|
def colorized(self):
|
|
return colorize_spec(self)
|
|
|
|
def format(self, format_string='$_$@$%@+$+$=', **kwargs):
|
|
"""
|
|
Prints out particular pieces of a spec, depending on what is
|
|
in the format string. The format strings you can provide are::
|
|
|
|
$_ Package name
|
|
$. Full package name (with namespace)
|
|
$@ Version with '@' prefix
|
|
$% Compiler with '%' prefix
|
|
$%@ Compiler with '%' prefix & compiler version with '@' prefix
|
|
$%+ Compiler with '%' prefix & compiler flags prefixed by name
|
|
$%@+ Compiler, compiler version, and compiler flags with same
|
|
prefixes as above
|
|
$+ Options
|
|
$= Architecture prefixed by 'arch='
|
|
$# 7-char prefix of DAG hash with '-' prefix
|
|
$$ $
|
|
|
|
You can also use full-string versions, which elide the prefixes::
|
|
|
|
${PACKAGE} Package name
|
|
${VERSION} Version
|
|
${COMPILER} Full compiler string
|
|
${COMPILERNAME} Compiler name
|
|
${COMPILERVER} Compiler version
|
|
${COMPILERFLAGS} Compiler flags
|
|
${OPTIONS} Options
|
|
${ARCHITECTURE} Architecture
|
|
${SHA1} Dependencies 8-char sha1 prefix
|
|
${HASH:len} DAG hash with optional length specifier
|
|
|
|
${SPACK_ROOT} The spack root directory
|
|
${SPACK_INSTALL} The default spack install directory,
|
|
${SPACK_PREFIX}/opt
|
|
${PREFIX} The package prefix
|
|
|
|
Note these are case-insensitive: for example you can specify either
|
|
``${PACKAGE}`` or ``${package}``.
|
|
|
|
Optionally you can provide a width, e.g. ``$20_`` for a 20-wide name.
|
|
Like printf, you can provide '-' for left justification, e.g.
|
|
``$-20_`` for a left-justified name.
|
|
|
|
Anything else is copied verbatim into the output stream.
|
|
|
|
*Example:* ``$_$@$+`` translates to the name, version, and options
|
|
of the package, but no dependencies, arch, or compiler.
|
|
|
|
TODO: allow, e.g., ``$6#`` to customize short hash length
|
|
TODO: allow, e.g., ``$##`` for full hash.
|
|
"""
|
|
color = kwargs.get('color', False)
|
|
length = len(format_string)
|
|
out = StringIO()
|
|
named = escape = compiler = False
|
|
named_str = fmt = ''
|
|
|
|
def write(s, c):
|
|
if color:
|
|
f = color_formats[c] + cescape(s) + '@.'
|
|
cwrite(f, stream=out, color=color)
|
|
else:
|
|
out.write(s)
|
|
|
|
iterator = enumerate(format_string)
|
|
for i, c in iterator:
|
|
if escape:
|
|
fmt = '%'
|
|
if c == '-':
|
|
fmt += c
|
|
i, c = next(iterator)
|
|
|
|
while c in '0123456789':
|
|
fmt += c
|
|
i, c = next(iterator)
|
|
fmt += 's'
|
|
|
|
if c == '_':
|
|
name = self.name if self.name else ''
|
|
out.write(fmt % name)
|
|
elif c == '.':
|
|
out.write(fmt % self.fullname)
|
|
elif c == '@':
|
|
if self.versions and self.versions != _any_version:
|
|
write(fmt % (c + str(self.versions)), c)
|
|
elif c == '%':
|
|
if self.compiler:
|
|
write(fmt % (c + str(self.compiler.name)), c)
|
|
compiler = True
|
|
elif c == '+':
|
|
if self.variants:
|
|
write(fmt % str(self.variants), c)
|
|
elif c == '=':
|
|
if self.architecture and str(self.architecture):
|
|
write(fmt % (' arch' + c + str(self.architecture)), c)
|
|
elif c == '#':
|
|
out.write('-' + fmt % (self.dag_hash(7)))
|
|
elif c == '$':
|
|
if fmt != '%s':
|
|
raise ValueError("Can't use format width with $$.")
|
|
out.write('$')
|
|
elif c == '{':
|
|
named = True
|
|
named_str = ''
|
|
escape = False
|
|
|
|
elif compiler:
|
|
if c == '@':
|
|
if (self.compiler and self.compiler.versions and
|
|
self.compiler.versions != _any_version):
|
|
write(c + str(self.compiler.versions), '%')
|
|
elif c == '+':
|
|
if self.compiler_flags:
|
|
write(fmt % str(self.compiler_flags), '%')
|
|
compiler = False
|
|
elif c == '$':
|
|
escape = True
|
|
compiler = False
|
|
else:
|
|
out.write(c)
|
|
compiler = False
|
|
|
|
elif named:
|
|
if not c == '}':
|
|
if i == length - 1:
|
|
raise ValueError("Error: unterminated ${ in format:"
|
|
"'%s'" % format_string)
|
|
named_str += c
|
|
continue
|
|
named_str = named_str.upper()
|
|
if named_str == 'PACKAGE':
|
|
name = self.name if self.name else ''
|
|
write(fmt % self.name, '@')
|
|
if named_str == 'VERSION':
|
|
if self.versions and self.versions != _any_version:
|
|
write(fmt % str(self.versions), '@')
|
|
elif named_str == 'COMPILER':
|
|
if self.compiler:
|
|
write(fmt % self.compiler, '%')
|
|
elif named_str == 'COMPILERNAME':
|
|
if self.compiler:
|
|
write(fmt % self.compiler.name, '%')
|
|
elif named_str in ['COMPILERVER', 'COMPILERVERSION']:
|
|
if self.compiler:
|
|
write(fmt % self.compiler.versions, '%')
|
|
elif named_str == 'COMPILERFLAGS':
|
|
if self.compiler:
|
|
write(fmt % str(self.compiler_flags), '%')
|
|
elif named_str == 'OPTIONS':
|
|
if self.variants:
|
|
write(fmt % str(self.variants), '+')
|
|
elif named_str == 'ARCHITECTURE':
|
|
if self.architecture and str(self.architecture):
|
|
write(fmt % str(self.architecture), ' arch=')
|
|
elif named_str == 'SHA1':
|
|
if self.dependencies:
|
|
out.write(fmt % str(self.dag_hash(7)))
|
|
elif named_str == 'SPACK_ROOT':
|
|
out.write(fmt % spack.prefix)
|
|
elif named_str == 'SPACK_INSTALL':
|
|
out.write(fmt % spack.store.root)
|
|
elif named_str == 'PREFIX':
|
|
out.write(fmt % self.prefix)
|
|
elif named_str.startswith('HASH'):
|
|
if named_str.startswith('HASH:'):
|
|
_, hashlen = named_str.split(':')
|
|
hashlen = int(hashlen)
|
|
else:
|
|
hashlen = None
|
|
out.write(fmt % (self.dag_hash(hashlen)))
|
|
|
|
named = False
|
|
|
|
elif c == '$':
|
|
escape = True
|
|
if i == length - 1:
|
|
raise ValueError("Error: unterminated $ in format: '%s'"
|
|
% format_string)
|
|
else:
|
|
out.write(c)
|
|
|
|
result = out.getvalue()
|
|
return result
|
|
|
|
def dep_string(self):
|
|
return ''.join("^" + dep.format() for dep in self.sorted_deps())
|
|
|
|
def __cmp__(self, other):
|
|
# Package name sort order is not configurable, always goes alphabetical
|
|
if self.name != other.name:
|
|
return cmp(self.name, other.name)
|
|
|
|
# Package version is second in compare order
|
|
pkgname = self.name
|
|
if self.versions != other.versions:
|
|
return spack.pkgsort.version_compare(
|
|
pkgname, self.versions, other.versions)
|
|
|
|
# Compiler is third
|
|
if self.compiler != other.compiler:
|
|
return spack.pkgsort.compiler_compare(
|
|
pkgname, self.compiler, other.compiler)
|
|
|
|
# Variants
|
|
if self.variants != other.variants:
|
|
return spack.pkgsort.variant_compare(
|
|
pkgname, self.variants, other.variants)
|
|
|
|
# Target
|
|
if self.architecture != other.architecture:
|
|
return spack.pkgsort.architecture_compare(
|
|
pkgname, self.architecture, other.architecture)
|
|
|
|
# Dependency is not configurable
|
|
if self._dependencies != other._dependencies:
|
|
return -1 if self._dependencies < other._dependencies else 1
|
|
|
|
# Equal specs
|
|
return 0
|
|
|
|
def __str__(self):
|
|
return self.format() + self.dep_string()
|
|
|
|
def _install_status(self):
|
|
"""Helper for tree to print DB install status."""
|
|
if not self.concrete:
|
|
return None
|
|
try:
|
|
record = spack.store.db.get_record(self)
|
|
return record.installed
|
|
except KeyError:
|
|
return None
|
|
|
|
def tree(self, **kwargs):
|
|
"""Prints out this spec and its dependencies, tree-formatted
|
|
with indentation."""
|
|
color = kwargs.pop('color', False)
|
|
depth = kwargs.pop('depth', False)
|
|
hashes = kwargs.pop('hashes', True)
|
|
hlen = kwargs.pop('hashlen', None)
|
|
install_status = kwargs.pop('install_status', True)
|
|
cover = kwargs.pop('cover', 'nodes')
|
|
indent = kwargs.pop('indent', 0)
|
|
fmt = kwargs.pop('format', '$_$@$%@+$+$=')
|
|
prefix = kwargs.pop('prefix', None)
|
|
deptypes = kwargs.pop('deptypes', ('build', 'link'))
|
|
check_kwargs(kwargs, self.tree)
|
|
|
|
out = ""
|
|
for d, node in self.traverse(
|
|
order='pre', cover=cover, depth=True, deptypes=deptypes):
|
|
if prefix is not None:
|
|
out += prefix(node)
|
|
out += " " * indent
|
|
if depth:
|
|
out += "%-4d" % d
|
|
if install_status:
|
|
status = node._install_status()
|
|
if status is None:
|
|
out += " " # Package isn't installed
|
|
elif status:
|
|
out += colorize("@g{[+]} ", color=color) # installed
|
|
else:
|
|
out += colorize("@r{[-]} ", color=color) # missing
|
|
|
|
if hashes:
|
|
out += colorize('@K{%s} ', color=color) % node.dag_hash(hlen)
|
|
out += (" " * d)
|
|
if d > 0:
|
|
out += "^"
|
|
out += node.format(fmt, color=color) + "\n"
|
|
return out
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
|
|
#
|
|
# These are possible token types in the spec grammar.
|
|
#
|
|
HASH, DEP, AT, COLON, COMMA, ON, OFF, PCT, EQ, QT, ID = range(11)
|
|
|
|
|
|
class SpecLexer(spack.parse.Lexer):
|
|
|
|
"""Parses tokens that make up spack specs."""
|
|
|
|
def __init__(self):
|
|
super(SpecLexer, self).__init__([
|
|
(r'/', lambda scanner, val: self.token(HASH, val)),
|
|
(r'\^', lambda scanner, val: self.token(DEP, val)),
|
|
(r'\@', lambda scanner, val: self.token(AT, val)),
|
|
(r'\:', lambda scanner, val: self.token(COLON, val)),
|
|
(r'\,', lambda scanner, val: self.token(COMMA, val)),
|
|
(r'\+', lambda scanner, val: self.token(ON, val)),
|
|
(r'\-', lambda scanner, val: self.token(OFF, val)),
|
|
(r'\~', lambda scanner, val: self.token(OFF, val)),
|
|
(r'\%', lambda scanner, val: self.token(PCT, val)),
|
|
(r'\=', lambda scanner, val: self.token(EQ, val)),
|
|
# This is more liberal than identifier_re (see above).
|
|
# Checked by check_identifier() for better error messages.
|
|
(r'([\"\'])(?:(?=(\\?))\2.)*?\1',
|
|
lambda scanner, val: self.token(QT, val)),
|
|
(r'\w[\w.-]*', lambda scanner, val: self.token(ID, val)),
|
|
(r'\s+', lambda scanner, val: None)])
|
|
|
|
|
|
# Lexer is always the same for every parser.
|
|
_lexer = SpecLexer()
|
|
|
|
|
|
class SpecParser(spack.parse.Parser):
|
|
|
|
def __init__(self):
|
|
super(SpecParser, self).__init__(_lexer)
|
|
self.previous = None
|
|
|
|
def do_parse(self):
|
|
specs = []
|
|
try:
|
|
while self.next:
|
|
# TODO: clean this parsing up a bit
|
|
if self.previous:
|
|
specs.append(self.spec(self.previous.value))
|
|
if self.accept(ID):
|
|
self.previous = self.token
|
|
if self.accept(EQ):
|
|
if not specs:
|
|
specs.append(self.spec(None))
|
|
if self.accept(QT):
|
|
self.token.value = self.token.value[1:-1]
|
|
else:
|
|
self.expect(ID)
|
|
specs[-1]._add_flag(
|
|
self.previous.value, self.token.value)
|
|
else:
|
|
specs.append(self.spec(self.previous.value))
|
|
self.previous = None
|
|
elif self.accept(HASH):
|
|
specs.append(self.spec_by_hash())
|
|
|
|
elif self.accept(DEP):
|
|
if not specs:
|
|
self.previous = self.token
|
|
specs.append(self.spec(None))
|
|
self.previous = None
|
|
if self.accept(HASH):
|
|
dep = self.spec_by_hash()
|
|
else:
|
|
self.expect(ID)
|
|
dep = self.spec(self.token.value)
|
|
# XXX(deptype): default deptypes
|
|
def_deptypes = ('build', 'link')
|
|
specs[-1]._add_dependency(dep, def_deptypes)
|
|
|
|
else:
|
|
# Attempt to construct an anonymous spec, but check that
|
|
# the first token is valid
|
|
# TODO: Is this check even necessary, or will it all be Lex
|
|
# errors now?
|
|
specs.append(self.spec(None, True))
|
|
|
|
except spack.parse.ParseError as e:
|
|
raise SpecParseError(e)
|
|
|
|
# If the spec has an os or a target and no platform, give it
|
|
# the default platform
|
|
platform_default = spack.architecture.platform().name
|
|
for spec in specs:
|
|
for s in spec.traverse():
|
|
if s.architecture and not s.architecture.platform and \
|
|
(s.architecture.platform_os or s.architecture.target):
|
|
s._set_architecture(platform=platform_default)
|
|
return specs
|
|
|
|
def parse_compiler(self, text):
|
|
self.setup(text)
|
|
return self.compiler()
|
|
|
|
def spec_by_hash(self):
|
|
self.expect(ID)
|
|
|
|
specs = spack.store.db.query()
|
|
matches = [spec for spec in specs if
|
|
spec.dag_hash()[:len(self.token.value)] == self.token.value]
|
|
|
|
if not matches:
|
|
tty.die("%s does not match any installed packages." %
|
|
self.token.value)
|
|
|
|
if len(matches) != 1:
|
|
raise AmbiguousHashError(
|
|
"Multiple packages specify hash %s." % self.token.value,
|
|
*matches)
|
|
|
|
return matches[0]
|
|
|
|
def spec(self, name, check_valid_token=False):
|
|
"""Parse a spec out of the input. If a spec is supplied, then initialize
|
|
and return it instead of creating a new one."""
|
|
if name:
|
|
spec_namespace, dot, spec_name = name.rpartition('.')
|
|
if not spec_namespace:
|
|
spec_namespace = None
|
|
self.check_identifier(spec_name)
|
|
else:
|
|
spec_namespace = None
|
|
spec_name = None
|
|
|
|
# This will init the spec without calling __init__.
|
|
spec = Spec.__new__(Spec)
|
|
spec.name = spec_name
|
|
spec.versions = VersionList()
|
|
spec.variants = VariantMap(spec)
|
|
spec.architecture = None
|
|
spec.compiler = None
|
|
spec.external = None
|
|
spec.external_module = None
|
|
spec.compiler_flags = FlagMap(spec)
|
|
spec._dependents = DependencyMap()
|
|
spec._dependencies = DependencyMap()
|
|
spec.namespace = spec_namespace
|
|
spec._hash = None
|
|
spec._cmp_key_cache = None
|
|
|
|
spec._normal = False
|
|
spec._concrete = False
|
|
|
|
# record this so that we know whether version is
|
|
# unspecified or not.
|
|
added_version = False
|
|
|
|
if self.previous and self.previous.value == DEP:
|
|
if self.accept(HASH):
|
|
spec.add_dependency(self.spec_by_hash())
|
|
else:
|
|
self.expect(ID)
|
|
if self.accept(EQ):
|
|
raise SpecParseError(spack.parse.ParseError(
|
|
"", "", "Expected dependency received anonymous spec"))
|
|
spec.add_dependency(self.spec(self.token.value))
|
|
|
|
while self.next:
|
|
if self.accept(AT):
|
|
vlist = self.version_list()
|
|
for version in vlist:
|
|
spec._add_version(version)
|
|
added_version = True
|
|
check_valid_token = False
|
|
|
|
elif self.accept(ON):
|
|
spec._add_variant(self.variant(), True)
|
|
check_valid_token = False
|
|
|
|
elif self.accept(OFF):
|
|
spec._add_variant(self.variant(), False)
|
|
check_valid_token = False
|
|
|
|
elif self.accept(PCT):
|
|
spec._set_compiler(self.compiler())
|
|
check_valid_token = False
|
|
|
|
elif self.accept(ID):
|
|
self.previous = self.token
|
|
if self.accept(EQ):
|
|
if self.accept(QT):
|
|
self.token.value = self.token.value[1:-1]
|
|
else:
|
|
self.expect(ID)
|
|
spec._add_flag(self.previous.value, self.token.value)
|
|
self.previous = None
|
|
else:
|
|
return spec
|
|
|
|
else:
|
|
if check_valid_token:
|
|
self.unexpected_token()
|
|
break
|
|
|
|
# If there was no version in the spec, consier it an open range
|
|
if not added_version:
|
|
spec.versions = VersionList(':')
|
|
|
|
return spec
|
|
|
|
def variant(self, name=None):
|
|
# TODO: Make generalized variants possible
|
|
if name:
|
|
return name
|
|
else:
|
|
self.expect(ID)
|
|
self.check_identifier()
|
|
return self.token.value
|
|
|
|
def version(self):
|
|
start = None
|
|
end = None
|
|
if self.accept(ID):
|
|
start = self.token.value
|
|
|
|
if self.accept(COLON):
|
|
if self.accept(ID):
|
|
end = self.token.value
|
|
elif start:
|
|
# No colon, but there was a version.
|
|
return Version(start)
|
|
else:
|
|
# No colon and no id: invalid version.
|
|
self.next_token_error("Invalid version specifier")
|
|
|
|
if start:
|
|
start = Version(start)
|
|
if end:
|
|
end = Version(end)
|
|
return VersionRange(start, end)
|
|
|
|
def version_list(self):
|
|
vlist = []
|
|
vlist.append(self.version())
|
|
while self.accept(COMMA):
|
|
vlist.append(self.version())
|
|
return vlist
|
|
|
|
def compiler(self):
|
|
self.expect(ID)
|
|
self.check_identifier()
|
|
|
|
compiler = CompilerSpec.__new__(CompilerSpec)
|
|
compiler.name = self.token.value
|
|
compiler.versions = VersionList()
|
|
if self.accept(AT):
|
|
vlist = self.version_list()
|
|
for version in vlist:
|
|
compiler._add_version(version)
|
|
else:
|
|
compiler.versions = VersionList(':')
|
|
return compiler
|
|
|
|
def check_identifier(self, id=None):
|
|
"""The only identifiers that can contain '.' are versions, but version
|
|
ids are context-sensitive so we have to check on a case-by-case
|
|
basis. Call this if we detect a version id where it shouldn't be.
|
|
"""
|
|
if not id:
|
|
id = self.token.value
|
|
if '.' in id:
|
|
self.last_token_error("Identifier cannot contain '.'")
|
|
|
|
|
|
def parse(string):
|
|
"""Returns a list of specs from an input string.
|
|
For creating one spec, see Spec() constructor.
|
|
"""
|
|
return SpecParser().parse(string)
|
|
|
|
|
|
def parse_anonymous_spec(spec_like, pkg_name):
|
|
"""Allow the user to omit the package name part of a spec if they
|
|
know what it has to be already.
|
|
|
|
e.g., provides('mpi@2', when='@1.9:') says that this package
|
|
provides MPI-3 when its version is higher than 1.9.
|
|
"""
|
|
if not isinstance(spec_like, (str, Spec)):
|
|
raise TypeError('spec must be Spec or spec string. Found %s'
|
|
% type(spec_like))
|
|
|
|
if isinstance(spec_like, str):
|
|
try:
|
|
anon_spec = Spec(spec_like)
|
|
if anon_spec.name != pkg_name:
|
|
raise SpecParseError(spack.parse.ParseError(
|
|
"",
|
|
"",
|
|
"Expected anonymous spec for package %s but found spec for"
|
|
"package %s" % (pkg_name, anon_spec.name)))
|
|
except SpecParseError:
|
|
anon_spec = Spec(pkg_name + ' ' + spec_like)
|
|
if anon_spec.name != pkg_name:
|
|
raise ValueError(
|
|
"Invalid spec for package %s: %s" % (pkg_name, spec_like))
|
|
else:
|
|
anon_spec = spec_like.copy()
|
|
|
|
if anon_spec.name != pkg_name:
|
|
raise ValueError("Spec name '%s' must match package name '%s'"
|
|
% (anon_spec.name, pkg_name))
|
|
|
|
return anon_spec
|
|
|
|
|
|
def base32_prefix_bits(hash_string, bits):
|
|
"""Return the first <bits> bits of a base32 string as an integer."""
|
|
if bits > len(hash_string) * 5:
|
|
raise ValueError("Too many bits! Requested %d bit prefix of '%s'."
|
|
% (bits, hash_string))
|
|
|
|
hash_bytes = base64.b32decode(hash_string, casefold=True)
|
|
return prefix_bits(hash_bytes, bits)
|
|
|
|
|
|
class SpecError(spack.error.SpackError):
|
|
"""Superclass for all errors that occur while constructing specs."""
|
|
|
|
|
|
class SpecParseError(SpecError):
|
|
"""Wrapper for ParseError for when we're parsing specs."""
|
|
def __init__(self, parse_error):
|
|
super(SpecParseError, self).__init__(parse_error.message)
|
|
self.string = parse_error.string
|
|
self.pos = parse_error.pos
|
|
|
|
|
|
class DuplicateDependencyError(SpecError):
|
|
"""Raised when the same dependency occurs in a spec twice."""
|
|
|
|
|
|
class DuplicateVariantError(SpecError):
|
|
"""Raised when the same variant occurs in a spec twice."""
|
|
|
|
|
|
class DuplicateCompilerSpecError(SpecError):
|
|
"""Raised when the same compiler occurs in a spec twice."""
|
|
|
|
|
|
class UnsupportedCompilerError(SpecError):
|
|
"""Raised when the user asks for a compiler spack doesn't know about."""
|
|
def __init__(self, compiler_name):
|
|
super(UnsupportedCompilerError, self).__init__(
|
|
"The '%s' compiler is not yet supported." % compiler_name)
|
|
|
|
|
|
class UnknownVariantError(SpecError):
|
|
"""Raised when the same variant occurs in a spec twice."""
|
|
def __init__(self, pkg, variant):
|
|
super(UnknownVariantError, self).__init__(
|
|
"Package %s has no variant %s!" % (pkg, variant))
|
|
|
|
|
|
class DuplicateArchitectureError(SpecError):
|
|
"""Raised when the same architecture occurs in a spec twice."""
|
|
|
|
|
|
class InconsistentSpecError(SpecError):
|
|
"""Raised when two nodes in the same spec DAG have inconsistent
|
|
constraints."""
|
|
|
|
|
|
class InvalidDependencyError(SpecError):
|
|
"""Raised when a dependency in a spec is not actually a dependency
|
|
of the package."""
|
|
|
|
|
|
class InvalidDependencyTypeError(SpecError):
|
|
"""Raised when a dependency type is not a legal Spack dep type."""
|
|
|
|
|
|
class NoProviderError(SpecError):
|
|
"""Raised when there is no package that provides a particular
|
|
virtual dependency.
|
|
"""
|
|
def __init__(self, vpkg):
|
|
super(NoProviderError, self).__init__(
|
|
"No providers found for virtual package: '%s'" % vpkg)
|
|
self.vpkg = vpkg
|
|
|
|
|
|
class MultipleProviderError(SpecError):
|
|
"""Raised when there is no package that provides a particular
|
|
virtual dependency.
|
|
"""
|
|
def __init__(self, vpkg, providers):
|
|
"""Takes the name of the vpkg"""
|
|
super(MultipleProviderError, self).__init__(
|
|
"Multiple providers found for '%s': %s"
|
|
% (vpkg, [str(s) for s in providers]))
|
|
self.vpkg = vpkg
|
|
self.providers = providers
|
|
|
|
|
|
class UnsatisfiableSpecError(SpecError):
|
|
"""Raised when a spec conflicts with package constraints.
|
|
Provide the requirement that was violated when raising."""
|
|
def __init__(self, provided, required, constraint_type):
|
|
super(UnsatisfiableSpecError, self).__init__(
|
|
"%s does not satisfy %s" % (provided, required))
|
|
self.provided = provided
|
|
self.required = required
|
|
self.constraint_type = constraint_type
|
|
|
|
|
|
class UnsatisfiableSpecNameError(UnsatisfiableSpecError):
|
|
"""Raised when two specs aren't even for the same package."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableSpecNameError, self).__init__(
|
|
provided, required, "name")
|
|
|
|
|
|
class UnsatisfiableVersionSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a spec version conflicts with package constraints."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableVersionSpecError, self).__init__(
|
|
provided, required, "version")
|
|
|
|
|
|
class UnsatisfiableCompilerSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a spec comiler conflicts with package constraints."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableCompilerSpecError, self).__init__(
|
|
provided, required, "compiler")
|
|
|
|
|
|
class UnsatisfiableVariantSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a spec variant conflicts with package constraints."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableVariantSpecError, self).__init__(
|
|
provided, required, "variant")
|
|
|
|
|
|
class UnsatisfiableCompilerFlagSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a spec variant conflicts with package constraints."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableCompilerFlagSpecError, self).__init__(
|
|
provided, required, "compiler_flags")
|
|
|
|
|
|
class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a spec architecture conflicts with package constraints."""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableArchitectureSpecError, self).__init__(
|
|
provided, required, "architecture")
|
|
|
|
|
|
class UnsatisfiableProviderSpecError(UnsatisfiableSpecError):
|
|
"""Raised when a provider is supplied but constraints don't match
|
|
a vpkg requirement"""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableProviderSpecError, self).__init__(
|
|
provided, required, "provider")
|
|
|
|
|
|
# TODO: get rid of this and be more specific about particular incompatible
|
|
# dep constraints
|
|
class UnsatisfiableDependencySpecError(UnsatisfiableSpecError):
|
|
"""Raised when some dependency of constrained specs are incompatible"""
|
|
def __init__(self, provided, required):
|
|
super(UnsatisfiableDependencySpecError, self).__init__(
|
|
provided, required, "dependency")
|
|
|
|
|
|
class AmbiguousHashError(SpecError):
|
|
def __init__(self, msg, *specs):
|
|
super(AmbiguousHashError, self).__init__(msg)
|
|
for spec in specs:
|
|
print(' ', spec.format('$.$@$%@+$+$=$#'))
|