zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1208 from paulhopkins/bugfix/github1196-specify-preferred-variants

Browse Source 
Bugfix/github1196 specify preferred variants
This commit is contained in:
Todd Gamblin 2016-07-21 08:36:19 -07:00 committed by GitHub
commit 098af17971
4 changed files with 128 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/spack/docs/configuration.rst
View File

@ -142,8 +142,9 @@ Here's an example packages.yaml file that sets preferred packages:
.. code-block:: sh .. code-block:: sh
packages: packages:
dyninst: opencv:
compiler: [gcc@4.9] compiler: [gcc@4.9]
variants: +debug
gperftools: gperftools:
version: [2.2, 2.4, 2.3] version: [2.2, 2.4, 2.3]
all: all:
@ -153,17 +154,17 @@ Here's an example packages.yaml file that sets preferred packages:
At a high level, this example is specifying how packages should be At a high level, this example is specifying how packages should be
concretized. The dyninst package should prefer using gcc 4.9. concretized. The opencv package should prefer using gcc 4.9 and
The gperftools package should prefer version be built with debug options. The gperftools package should prefer version
2.2 over 2.4. Every package on the system should prefer mvapich for 2.2 over 2.4. Every package on the system should prefer mvapich for
its MPI and gcc 4.4.7 (except for Dyninst, which overrides this by preferring gcc 4.9). its MPI and gcc 4.4.7 (except for opencv, which overrides this by preferring gcc 4.9).
These options are used to fill in implicit defaults. Any of them can be overwritten These options are used to fill in implicit defaults. Any of them can be overwritten
on the command line if explicitly requested. on the command line if explicitly requested.
Each packages.yaml file begins with the string ``packages:`` and Each packages.yaml file begins with the string ``packages:`` and
package names are specified on the next level. The special string ``all`` package names are specified on the next level. The special string ``all``
applies settings to each package. Underneath each package name is applies settings to each package. Underneath each package name is
one or more components: ``compiler``, ``version``, one or more components: ``compiler``, ``variants``, ``version``,
or ``providers``. Each component has an ordered list of spec or ``providers``. Each component has an ordered list of spec
``constraints``, with earlier entries in the list being preferred over ``constraints``, with earlier entries in the list being preferred over
later entries. later entries.

143
lib/spack/spack/concretize.py
View File

@ -40,12 +40,12 @@
import spack.error import spack.error
from spack.version import * from spack.version import *
from functools import partial from functools import partial
from spec import DependencyMap
from itertools import chain from itertools import chain
from spack.config import * from spack.config import *
class DefaultConcretizer(object): class DefaultConcretizer(object):
"""This class doesn't have any state, it just provides some methods for """This class doesn't have any state, it just provides some methods for
concretization. You can subclass it to override just some of the concretization. You can subclass it to override just some of the
default concretization strategies, or you can override all of them. default concretization strategies, or you can override all of them.
@ -61,14 +61,17 @@ def _valid_virtuals_and_externals(self, spec):
if not providers: if not providers:
raise UnsatisfiableProviderSpecError(providers[0], spec) raise UnsatisfiableProviderSpecError(providers[0], spec)
spec_w_preferred_providers = find_spec( spec_w_preferred_providers = find_spec(
spec, lambda(x): spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) spec, lambda x: spack.pkgsort.spec_has_preferred_provider(x.name, spec.name)) # NOQA: ignore=E501
if not spec_w_preferred_providers: if not spec_w_preferred_providers:
spec_w_preferred_providers = spec spec_w_preferred_providers = spec
provider_cmp = partial(spack.pkgsort.provider_compare, spec_w_preferred_providers.name, spec.name) provider_cmp = partial(spack.pkgsort.provider_compare,
spec_w_preferred_providers.name,
spec.name)
candidates = sorted(providers, cmp=provider_cmp) candidates = sorted(providers, cmp=provider_cmp)
# For each candidate package, if it has externals, add those to the usable list. # For each candidate package, if it has externals, add those
# if it's not buildable, then *only* add the externals. # to the usable list. if it's not buildable, then *only* add
# the externals.
usable = [] usable = []
for cspec in candidates: for cspec in candidates:
if is_spec_buildable(cspec): if is_spec_buildable(cspec):
@ -114,13 +117,14 @@ def choose_virtual_or_external(self, spec):
# Find the nearest spec in the dag that has a compiler. We'll # Find the nearest spec in the dag that has a compiler. We'll
# use that spec to calibrate compiler compatibility. # use that spec to calibrate compiler compatibility.
abi_exemplar = find_spec(spec, lambda(x): x.compiler) abi_exemplar = find_spec(spec, lambda x: x.compiler)
if not abi_exemplar: if not abi_exemplar:
abi_exemplar = spec.root abi_exemplar = spec.root
# Make a list including ABI compatibility of specs with the exemplar. # Make a list including ABI compatibility of specs with the exemplar.
strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates] strict = [spack.abi.compatible(c, abi_exemplar) for c in candidates]
loose = [spack.abi.compatible(c, abi_exemplar, loose=True) for c in candidates] loose = [spack.abi.compatible(c, abi_exemplar, loose=True)
for c in candidates]
keys = zip(strict, loose, candidates) keys = zip(strict, loose, candidates)
# Sort candidates from most to least compatibility. # Sort candidates from most to least compatibility.
@ -133,7 +137,6 @@ def choose_virtual_or_external(self, spec):
candidates = [c for s, l, c in keys] candidates = [c for s, l, c in keys]
return candidates return candidates
def concretize_version(self, spec): def concretize_version(self, spec):
"""If the spec is already concrete, return. Otherwise take """If the spec is already concrete, return. Otherwise take
the preferred version from spackconfig, and default to the package's the preferred version from spackconfig, and default to the package's
@ -167,7 +170,8 @@ def prefer_key(v):
if valid_versions: if valid_versions:
# Disregard @develop and take the next valid version # Disregard @develop and take the next valid version
if ver(valid_versions[0]) == ver('develop') and len(valid_versions) > 1: if ver(valid_versions[0]) == ver('develop') and \
len(valid_versions) > 1:
spec.versions = ver([valid_versions[1]]) spec.versions = ver([valid_versions[1]])
else: else:
spec.versions = ver([valid_versions[0]]) spec.versions = ver([valid_versions[0]])
@ -193,28 +197,32 @@ def prefer_key(v):
return True # Things changed return True # Things changed
def _concretize_operating_system(self, spec): def _concretize_operating_system(self, spec):
platform = spec.architecture.platform
if spec.architecture.platform_os is not None and isinstance( if spec.architecture.platform_os is not None and isinstance(
spec.architecture.platform_os,spack.architecture.OperatingSystem): spec.architecture.platform_os,
spack.architecture.OperatingSystem):
return False return False
if spec.root.architecture and spec.root.architecture.platform_os: if spec.root.architecture and spec.root.architecture.platform_os:
if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem): if isinstance(spec.root.architecture.platform_os,
spec.architecture.platform_os = spec.root.architecture.platform_os spack.architecture.OperatingSystem):
spec.architecture.platform_os = \
spec.root.architecture.platform_os
else: else:
spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os') spec.architecture.platform_os = \
spec.architecture.platform.operating_system('default_os')
return True # changed return True # changed
def _concretize_target(self, spec): def _concretize_target(self, spec):
platform = spec.architecture.platform
if spec.architecture.target is not None and isinstance( if spec.architecture.target is not None and isinstance(
spec.architecture.target, spack.architecture.Target): spec.architecture.target, spack.architecture.Target):
return False return False
if spec.root.architecture and spec.root.architecture.target: if spec.root.architecture and spec.root.architecture.target:
if isinstance(spec.root.architecture.target,spack.architecture.Target): if isinstance(spec.root.architecture.target,
spack.architecture.Target):
spec.architecture.target = spec.root.architecture.target spec.architecture.target = spec.root.architecture.target
else: else:
spec.architecture.target = spec.architecture.platform.target('default_target') spec.architecture.target = spec.architecture.platform.target(
'default_target')
return True # changed return True # changed
def _concretize_platform(self, spec): def _concretize_platform(self, spec):
@ -222,7 +230,8 @@ def _concretize_platform(self, spec):
spec.architecture.platform, spack.architecture.Platform): spec.architecture.platform, spack.architecture.Platform):
return False return False
if spec.root.architecture and spec.root.architecture.platform: if spec.root.architecture and spec.root.architecture.platform:
if isinstance(spec.root.architecture.platform,spack.architecture.Platform): if isinstance(spec.root.architecture.platform,
spack.architecture.Platform):
spec.architecture.platform = spec.root.architecture.platform spec.architecture.platform = spec.root.architecture.platform
else: else:
spec.architecture.platform = spack.architecture.platform() spec.architecture.platform = spack.architecture.platform()
@ -250,20 +259,24 @@ def concretize_architecture(self, spec):
self._concretize_target(spec))) self._concretize_target(spec)))
return ret return ret
def concretize_variants(self, spec): def concretize_variants(self, spec):
"""If the spec already has variants filled in, return. Otherwise, add """If the spec already has variants filled in, return. Otherwise, add
the default variants from the package specification. the user preferences from packages.yaml or the default variants from
the package specification.
""" """
changed = False changed = False
preferred_variants = spack.pkgsort.spec_preferred_variants(
spec.package_class.name)
for name, variant in spec.package_class.variants.items(): for name, variant in spec.package_class.variants.items():
if name not in spec.variants: if name not in spec.variants:
spec.variants[name] = spack.spec.VariantSpec(name, variant.default)
changed = True changed = True
if name in preferred_variants:
spec.variants[name] = preferred_variants.get(name)
else:
spec.variants[name] = \
spack.spec.VariantSpec(name, variant.default)
return changed return changed
def concretize_compiler(self, spec): def concretize_compiler(self, spec):
"""If the spec already has a compiler, we're done. If not, then take """If the spec already has a compiler, we're done. If not, then take
the compiler used for the nearest ancestor with a compiler the compiler used for the nearest ancestor with a compiler
@ -278,12 +291,14 @@ def concretize_compiler(self, spec):
""" """
# Pass on concretizing the compiler if the target is not yet determined # Pass on concretizing the compiler if the target is not yet determined
if not spec.architecture.platform_os: if not spec.architecture.platform_os:
#Although this usually means changed, this means awaiting other changes # Although this usually means changed, this means awaiting other
# changes
return True return True
# Only use a matching compiler if it is of the proper style # Only use a matching compiler if it is of the proper style
# Takes advantage of the proper logic already existing in compiler_for_spec # Takes advantage of the proper logic already existing in
# Should think whether this can be more efficient # compiler_for_spec Should think whether this can be more
# efficient
def _proper_compiler_style(cspec, arch): def _proper_compiler_style(cspec, arch):
platform = arch.platform platform = arch.platform
compilers = spack.compilers.compilers_for_spec(cspec, compilers = spack.compilers.compilers_for_spec(cspec,
@ -292,7 +307,6 @@ def _proper_compiler_style(cspec, arch):
arch.platform_os, compilers) arch.platform_os, compilers)
# return compilers # return compilers
all_compilers = spack.compilers.all_compilers() all_compilers = spack.compilers.all_compilers()
if (spec.compiler and if (spec.compiler and
@ -301,7 +315,8 @@ def _proper_compiler_style(cspec, arch):
return False return False
# Find the another spec that has a compiler, or the root if none do # Find the another spec that has a compiler, or the root if none do
other_spec = spec if spec.compiler else find_spec(spec, lambda(x) : x.compiler) other_spec = spec if spec.compiler else find_spec(
spec, lambda x: x.compiler)
if not other_spec: if not other_spec:
other_spec = spec.root other_spec = spec.root
@ -313,9 +328,12 @@ def _proper_compiler_style(cspec, arch):
spec.compiler = other_compiler.copy() spec.compiler = other_compiler.copy()
return True return True
# Filter the compilers into a sorted list based on the compiler_order from spackconfig # Filter the compilers into a sorted list based on the compiler_order
compiler_list = all_compilers if not other_compiler else spack.compilers.find(other_compiler) # from spackconfig
cmp_compilers = partial(spack.pkgsort.compiler_compare, other_spec.name) compiler_list = all_compilers if not other_compiler else \
spack.compilers.find(other_compiler)
cmp_compilers = partial(
spack.pkgsort.compiler_compare, other_spec.name)
matches = sorted(compiler_list, cmp=cmp_compilers) matches = sorted(compiler_list, cmp=cmp_compilers)
if not matches: if not matches:
raise UnavailableCompilerVersionError(other_compiler) raise UnavailableCompilerVersionError(other_compiler)
@ -330,7 +348,6 @@ def _proper_compiler_style(cspec, arch):
assert(spec.compiler.concrete) assert(spec.compiler.concrete)
return True # things changed. return True # things changed.
def concretize_compiler_flags(self, spec): def concretize_compiler_flags(self, spec):
""" """
The compiler flags are updated to match those of the spec whose The compiler flags are updated to match those of the spec whose
@ -338,53 +355,65 @@ def concretize_compiler_flags(self, spec):
Default specs set at the compiler level will still be added later. Default specs set at the compiler level will still be added later.
""" """
if not spec.architecture.platform_os: if not spec.architecture.platform_os:
#Although this usually means changed, this means awaiting other changes # Although this usually means changed, this means awaiting other
# changes
return True return True
ret = False ret = False
for flag in spack.spec.FlagMap.valid_compiler_flags(): for flag in spack.spec.FlagMap.valid_compiler_flags():
try: try:
nearest = next(p for p in spec.traverse(direction='parents') nearest = next(p for p in spec.traverse(direction='parents')
if ((p.compiler == spec.compiler and p is not spec) if ((p.compiler == spec.compiler and
and flag in p.compiler_flags)) p is not spec) and
if not flag in spec.compiler_flags or \ flag in p.compiler_flags))
not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): if flag not in spec.compiler_flags or \
not (sorted(spec.compiler_flags[flag]) >=
sorted(nearest.compiler_flags[flag])):
if flag in spec.compiler_flags: if flag in spec.compiler_flags:
spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | spec.compiler_flags[flag] = list(
set(spec.compiler_flags[flag]) |
set(nearest.compiler_flags[flag])) set(nearest.compiler_flags[flag]))
else: else:
spec.compiler_flags[flag] = nearest.compiler_flags[flag] spec.compiler_flags[
flag] = nearest.compiler_flags[flag]
ret = True ret = True
except StopIteration: except StopIteration:
if (flag in spec.root.compiler_flags and ((not flag in spec.compiler_flags) or if (flag in spec.root.compiler_flags and
sorted(spec.compiler_flags[flag]) != sorted(spec.root.compiler_flags[flag]))): ((flag not in spec.compiler_flags) or
sorted(spec.compiler_flags[flag]) !=
sorted(spec.root.compiler_flags[flag]))):
if flag in spec.compiler_flags: if flag in spec.compiler_flags:
spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | spec.compiler_flags[flag] = list(
set(spec.compiler_flags[flag]) |
set(spec.root.compiler_flags[flag])) set(spec.root.compiler_flags[flag]))
else: else:
spec.compiler_flags[flag] = spec.root.compiler_flags[flag] spec.compiler_flags[
flag] = spec.root.compiler_flags[flag]
ret = True ret = True
else: else:
if not flag in spec.compiler_flags: if flag not in spec.compiler_flags:
spec.compiler_flags[flag] = [] spec.compiler_flags[flag] = []
# Include the compiler flag defaults from the config files # Include the compiler flag defaults from the config files
# This ensures that spack will detect conflicts that stem from a change # This ensures that spack will detect conflicts that stem from a change
# in default compiler flags. # in default compiler flags.
compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) compiler = spack.compilers.compiler_for_spec(
spec.compiler, spec.architecture)
for flag in compiler.flags: for flag in compiler.flags:
if flag not in spec.compiler_flags: if flag not in spec.compiler_flags:
spec.compiler_flags[flag] = compiler.flags[flag] spec.compiler_flags[flag] = compiler.flags[flag]
if compiler.flags[flag] != []: if compiler.flags[flag] != []:
ret = True ret = True
else: else:
if ((sorted(spec.compiler_flags[flag]) != sorted(compiler.flags[flag])) and if ((sorted(spec.compiler_flags[flag]) !=
(not set(spec.compiler_flags[flag]) >= set(compiler.flags[flag]))): sorted(compiler.flags[flag])) and
(not set(spec.compiler_flags[flag]) >=
set(compiler.flags[flag]))):
ret = True ret = True
spec.compiler_flags[flag] = list(set(spec.compiler_flags[flag]) | spec.compiler_flags[flag] = list(
set(spec.compiler_flags[flag]) |
set(compiler.flags[flag])) set(compiler.flags[flag]))
return ret return ret
@ -406,8 +435,10 @@ def find_spec(spec, condition):
# Then search all other relatives in the DAG *except* spec # Then search all other relatives in the DAG *except* spec
for relative in spec.root.traverse(deptypes=spack.alldeps): for relative in spec.root.traverse(deptypes=spack.alldeps):
if relative is spec: continue if relative is spec:
if id(relative) in visited: continue continue
if id(relative) in visited:
continue
if condition(relative): if condition(relative):
return relative return relative
@ -454,8 +485,10 @@ def cmp_specs(lhs, rhs):
class UnavailableCompilerVersionError(spack.error.SpackError): class UnavailableCompilerVersionError(spack.error.SpackError):
"""Raised when there is no available compiler that satisfies a """Raised when there is no available compiler that satisfies a
compiler spec.""" compiler spec."""
def __init__(self, compiler_spec): def __init__(self, compiler_spec):
super(UnavailableCompilerVersionError, self).__init__( super(UnavailableCompilerVersionError, self).__init__(
"No available compiler version matches '%s'" % compiler_spec, "No available compiler version matches '%s'" % compiler_spec,
@ -463,16 +496,20 @@ def __init__(self, compiler_spec):
class NoValidVersionError(spack.error.SpackError): class NoValidVersionError(spack.error.SpackError):
"""Raised when there is no way to have a concrete version for a """Raised when there is no way to have a concrete version for a
particular spec.""" particular spec."""
def __init__(self, spec): def __init__(self, spec):
super(NoValidVersionError, self).__init__( super(NoValidVersionError, self).__init__(
"There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) "There are no valid versions for %s that match '%s'" % (spec.name, spec.versions)) # NOQA: ignore=E501
class NoBuildError(spack.error.SpackError): class NoBuildError(spack.error.SpackError):
"""Raised when a package is configured with the buildable option False, but """Raised when a package is configured with the buildable option False, but
no satisfactory external versions can be found""" no satisfactory external versions can be found"""
def __init__(self, spec): def __init__(self, spec):
super(NoBuildError, self).__init__( super(NoBuildError, self).__init__(
"The spec '%s' is configured as not buildable, and no matching external installs were found" % spec.name) "The spec '%s' is configured as not buildable,and no matching external installs were found" % spec.name) # NOQA: ignore=E501

8
lib/spack/spack/config.py
View File

@ -257,7 +257,13 @@
'paths': { 'paths': {
'type' : 'object', 'type' : 'object',
'default' : {}, 'default' : {},
} },
'variants': {
'oneOf' : [
{ 'type' : 'string' },
{ 'type' : 'array',
'items' : { 'type' : 'string' } },
], },
},},},},},}, },},},},},},
'modules': { 'modules': {

7
lib/spack/spack/preferred_packages.py
View File

@ -158,6 +158,13 @@ def spec_has_preferred_provider(self, pkgname, provider_str):
return bool(self._order_for_package(pkgname, 'providers', return bool(self._order_for_package(pkgname, 'providers',
provider_str, False)) provider_str, False))
def spec_preferred_variants(self, pkgname):
"""Return a VariantMap of preferred variants and their values"""
variants = self.preferred.get(pkgname, {}).get('variants', '')
if not isinstance(variants, basestring):
variants = "".join(variants)
return spack.spec.Spec(pkgname + variants).variants
def version_compare(self, pkgname, a, b): def version_compare(self, pkgname, a, b):
"""Return less-than-0, 0, or greater than 0 if version a of pkgname is """Return less-than-0, 0, or greater than 0 if version a of pkgname is
respectively less-than, equal-to, or greater-than version b of respectively less-than, equal-to, or greater-than version b of