zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reworked compiler finding/storing/format to allow for multiple compilers with the same spec for different operating systems. TODO: fix config tests. All others up to date

Browse Source
This commit is contained in:
Gregory Becker 2016-05-04 21:42:59 -07:00
parent 8cd13d4b35
commit 90b7b7ba5c
12 changed files with 451 additions and 317 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

96
lib/spack/spack/architecture.py
View File

@ -22,6 +22,7 @@
# along with this program; if not, write to the Free Software Foundation, # along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
import os
from collections import namedtuple from collections import namedtuple
import imp import imp
import platform as py_platform import platform as py_platform
@ -32,7 +33,10 @@
import llnl.util.tty as tty import llnl.util.tty as tty
import spack import spack
import spack.compilers
from spack.util.naming import mod_to_class from spack.util.naming import mod_to_class
from spack.util.environment import get_path
from spack.util.multiproc import parmap
import spack.error as serr import spack.error as serr
@ -143,6 +147,7 @@ def operating_system(self, name):
return self.operating_sys.get(name, None) return self.operating_sys.get(name, None)
@classmethod @classmethod
def detect(self): def detect(self):
""" Subclass is responsible for implementing this method. """ Subclass is responsible for implementing this method.
@ -170,10 +175,9 @@ class OperatingSystem(object):
find compilers we call find_compilers method for each operating system find compilers we call find_compilers method for each operating system
""" """
def __init__(self, name, version, compiler_strategy="PATH"): def __init__(self, name, version):
self.name = name self.name = name
self.version = version self.version = version
self.compiler_strategy = compiler_strategy
def __str__(self): def __str__(self):
return self.name + self.version return self.name + self.version
@ -182,13 +186,96 @@ def __repr__(self):
return self.__str__() return self.__str__()
def _cmp_key(self): def _cmp_key(self):
return (self.name, self.version, self.compiler_strategy) return (self.name, self.version)
def find_compilers(self, *paths):
"""
Return a list of compilers found in the suppied paths.
This invokes the find() method for each Compiler class,
and appends the compilers detected to a list.
"""
if not paths:
paths = get_path('PATH')
# Make sure path elements exist, and include /bin directories
# under prefixes.
filtered_path = []
for p in paths:
# Eliminate symlinks and just take the real directories.
p = os.path.realpath(p)
if not os.path.isdir(p):
continue
filtered_path.append(p)
# Check for a bin directory, add it if it exists
bin = join_path(p, 'bin')
if os.path.isdir(bin):
filtered_path.append(os.path.realpath(bin))
# Once the paths are cleaned up, do a search for each type of
# compiler. We can spawn a bunch of parallel searches to reduce
# the overhead of spelunking all these directories.
types = spack.compilers.all_compiler_types()
compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *filtered_path), types)
# ensure all the version calls we made are cached in the parent
# process, as well. This speeds up Spack a lot.
clist = reduce(lambda x,y: x+y, compiler_lists)
return clist
def find_compiler(self, cmp_cls, *path):
"""Try to find the given type of compiler in the user's
environment. For each set of compilers found, this returns
compiler objects with the cc, cxx, f77, fc paths and the
version filled in.
This will search for compilers with the names in cc_names,
cxx_names, etc. and it will group them if they have common
prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
be grouped with g++-mp-4.7 and gfortran-mp-4.7.
"""
dicts = parmap(
lambda t: cmp_cls._find_matches_in_path(*t),
[(cmp_cls.cc_names, cmp_cls.cc_version) + tuple(path),
(cmp_cls.cxx_names, cmp_cls.cxx_version) + tuple(path),
(cmp_cls.f77_names, cmp_cls.f77_version) + tuple(path),
(cmp_cls.fc_names, cmp_cls.fc_version) + tuple(path)])
all_keys = set()
for d in dicts:
all_keys.update(d)
compilers = {}
for k in all_keys:
ver, pre, suf = k
# Skip compilers with unknown version.
if ver == 'unknown':
continue
paths = tuple(pn[k] if k in pn else None for pn in dicts)
spec = spack.spec.CompilerSpec(cmp_cls.name, ver)
if ver in compilers:
prev = compilers[ver]
# prefer the one with more compilers.
prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
newcount = len([p for p in paths if p is not None])
prevcount = len([p for p in prev_paths if p is not None])
# Don't add if it's not an improvement over prev compiler.
if newcount <= prevcount:
continue
compilers[ver] = cmp_cls(spec, self, paths)
return list(compilers.values())
def to_dict(self): def to_dict(self):
d = {} d = {}
d['name'] = self.name d['name'] = self.name
d['version'] = self.version d['version'] = self.version
d['compiler_strategy'] = self.compiler_strategy
return d return d
@ -261,7 +348,6 @@ def _operating_system_from_dict(os_dict):
operating_system = OperatingSystem.__new__(OperatingSystem) operating_system = OperatingSystem.__new__(OperatingSystem)
operating_system.name = os_dict['name'] operating_system.name = os_dict['name']
operating_system.version = os_dict['version'] operating_system.version = os_dict['version']
operating_system.compiler_strategy = os_dict['compiler_strategy']
return operating_system return operating_system
def arch_from_dict(d): def arch_from_dict(d):

5
lib/spack/spack/build_environment.py
View File

@ -169,9 +169,8 @@ def set_compiler_environment_variables(pkg):
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler) os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
if compiler.strategy == 'MODULES': for mod in compiler.modules:
for mod in compiler.modules: load_module(mod)
load_module(mod)
def set_build_environment_variables(pkg): def set_build_environment_variables(pkg):
"""This ensures a clean install environment when we build packages. """This ensures a clean install environment when we build packages.

4
lib/spack/spack/cmd/compiler.py
View File

@ -80,7 +80,7 @@ def compiler_add(args):
if not paths: if not paths:
paths = get_path('PATH') paths = get_path('PATH')
compilers = [c for c in spack.compilers.find_compilers(*args.add_paths) compilers = [c for c in spack.compilers.find_compilers(*paths)
if c.spec not in spack.compilers.all_compilers(scope=args.scope)] if c.spec not in spack.compilers.all_compilers(scope=args.scope)]
if compilers: if compilers:
@ -125,6 +125,8 @@ def compiler_info(args):
print "\tcxx = %s" % c.cxx print "\tcxx = %s" % c.cxx
print "\tf77 = %s" % c.f77 print "\tf77 = %s" % c.f77
print "\tfc = %s" % c.fc print "\tfc = %s" % c.fc
print "\tmodules = %s" % c.modules
print "\toperating system = %s" % c.operating_system
def compiler_list(args): def compiler_list(args):

166
lib/spack/spack/compiler.py
View File

@ -106,14 +106,14 @@ class Compiler(object):
PrgEnv_compiler = None PrgEnv_compiler = None
def __init__(self, cspec, strategy, paths, modules=None): def __init__(self, cspec, operating_system, paths, modules=[], alias=None):
def check(exe): def check(exe):
if exe is None: if exe is None:
return None return None
_verify_executables(exe) _verify_executables(exe)
return exe return exe
self.strategy = strategy self.operating_system = operating_system
self.cc = check(paths[0]) self.cc = check(paths[0])
self.cxx = check(paths[1]) self.cxx = check(paths[1])
@ -126,62 +126,33 @@ def check(exe):
self.spec = cspec self.spec = cspec
self.modules = modules self.modules = modules
self.alias = alias
@property @property
def version(self): def version(self):
return self.spec.version return self.spec.version
#
# Compiler classes have methods for querying the version of
# specific compiler executables. This is used when discovering compilers.
#
# Compiler *instances* are just data objects, and can only be
# constructed from an actual set of executables.
#
@classmethod
def default_version(cls, cc):
"""Override just this to override all compiler version functions."""
return dumpversion(cc)
@classmethod
def cc_version(cls, cc):
return cls.default_version(cc)
@classmethod
def cxx_version(cls, cxx):
return cls.default_version(cxx)
@classmethod
def f77_version(cls, f77):
return cls.default_version(f77)
@classmethod
def fc_version(cls, fc):
return cls.default_version(fc)
@classmethod @classmethod
def _find_matches_in_path(cls, compiler_names, detect_version, *path): def _find_matches_in_path(cls, compiler_names, detect_version, *path):
"""Finds compilers in the paths supplied. """Finds compilers in the paths supplied.
Looks for all combinations of ``compiler_names`` with the Looks for all combinations of ``compiler_names`` with the
``prefixes`` and ``suffixes`` defined for this compiler ``prefixes`` and ``suffixes`` defined for this compiler
class. If any compilers match the compiler_names, class. If any compilers match the compiler_names,
prefixes, or suffixes, uses ``detect_version`` to figure prefixes, or suffixes, uses ``detect_version`` to figure
out what version the compiler is. out what version the compiler is.
This returns a dict with compilers grouped by (prefix, This returns a dict with compilers grouped by (prefix,
suffix, version) tuples. This can be further organized by suffix, version) tuples. This can be further organized by
find(). find().
""" """
if not path: if not path:
path = get_path('PATH') path = get_path('PATH')
prefixes = [''] + cls.prefixes prefixes = [''] + cls.prefixes
suffixes = [''] + cls.suffixes suffixes = [''] + cls.suffixes
checks = [] checks = []
for directory in path: for directory in path:
if not (os.path.isdir(directory) and if not (os.path.isdir(directory) and
@ -219,89 +190,34 @@ def check(key):
successful = [key for key in parmap(check, checks) if key is not None] successful = [key for key in parmap(check, checks) if key is not None]
return dict(((v, p, s), path) for v, p, s, path in successful) return dict(((v, p, s), path) for v, p, s, path in successful)
@classmethod #
def find(cls, *path): # Compiler classes have methods for querying the version of
compilers = [] # specific compiler executables. This is used when discovering compilers.
platform = spack.architecture.sys_type() #
strategies = [o.compiler_strategy for o in platform.operating_sys.values()] # Compiler *instances* are just data objects, and can only be
if 'PATH' in strategies: # constructed from an actual set of executables.
compilers.extend(cls.find_in_path(*path)) #
if 'MODULES' in strategies:
compilers.extend(cls.find_in_modules())
return compilers
@classmethod @classmethod
def find_in_path(cls, *path): def default_version(cls, cc):
"""Try to find this type of compiler in the user's """Override just this to override all compiler version functions."""
environment. For each set of compilers found, this returns return dumpversion(cc)
compiler objects with the cc, cxx, f77, fc paths and the
version filled in.
This will search for compilers with the names in cc_names,
cxx_names, etc. and it will group them if they have common
prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
be grouped with g++-mp-4.7 and gfortran-mp-4.7.
"""
dicts = parmap(
lambda t: cls._find_matches_in_path(*t),
[(cls.cc_names, cls.cc_version) + tuple(path),
(cls.cxx_names, cls.cxx_version) + tuple(path),
(cls.f77_names, cls.f77_version) + tuple(path),
(cls.fc_names, cls.fc_version) + tuple(path)])
all_keys = set()
for d in dicts:
all_keys.update(d)
compilers = {}
for k in all_keys:
ver, pre, suf = k
# Skip compilers with unknown version.
if ver == 'unknown':
continue
paths = tuple(pn[k] if k in pn else None for pn in dicts)
spec = spack.spec.CompilerSpec(cls.name, ver)
if ver in compilers:
prev = compilers[ver]
# prefer the one with more compilers.
prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
newcount = len([p for p in paths if p is not None])
prevcount = len([p for p in prev_paths if p is not None])
# Don't add if it's not an improvement over prev compiler.
if newcount <= prevcount:
continue
compilers[ver] = cls(spec, 'PATH', paths)
return list(compilers.values())
@classmethod @classmethod
def find_in_modules(cls): def cc_version(cls, cc):
compilers = [] return cls.default_version(cc)
if cls.PrgEnv:
if not cls.PrgEnv_compiler:
tty.die('Must supply PrgEnv_compiler with PrgEnv')
modulecmd = which('modulecmd') @classmethod
modulecmd.add_default_arg('python') def cxx_version(cls, cxx):
return cls.default_version(cxx)
output = modulecmd('avail', cls.PrgEnv_compiler, output=str, error=str)
matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output)
for name, version in matches:
v = version
comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES',
['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v])
compilers.append(comp) @classmethod
def f77_version(cls, f77):
return cls.default_version(f77)
return compilers @classmethod
def fc_version(cls, fc):
return cls.default_version(fc)
def __repr__(self): def __repr__(self):
@ -311,12 +227,8 @@ def __repr__(self):
def __str__(self): def __str__(self):
"""Return a string represntation of the compiler toolchain.""" """Return a string represntation of the compiler toolchain."""
if self.strategy is 'MODULES': return "%s(%s)" % (
return "%s(%s)" % ( self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc, self.modules, str(self.operating_system)))))
self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules))))
else:
return "%s(%s)" % (
self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc))))
class CompilerAccessError(spack.error.SpackError): class CompilerAccessError(spack.error.SpackError):

224
lib/spack/spack/compilers/__init__.py
View File

@ -29,6 +29,10 @@
import os import os
import platform import platform
import copy import copy
import hashlib
import base64
import yaml
import sys
from llnl.util.lang import memoized, list_modules from llnl.util.lang import memoized, list_modules
from llnl.util.filesystem import join_path from llnl.util.filesystem import join_path
@ -46,8 +50,8 @@
from spack.util.environment import get_path from spack.util.environment import get_path
_imported_compilers_module = 'spack.compilers' _imported_compilers_module = 'spack.compilers'
_required_instance_vars = ['cc', 'cxx', 'f77', 'fc'] _path_instance_vars = ['cc', 'cxx', 'f77', 'fc']
_optional_instance_vars = ['modules', 'strategy'] _other_instance_vars = ['modules', 'operating_system']
_default_order = [] _default_order = []
# TODO: customize order in config file # TODO: customize order in config file
@ -67,93 +71,85 @@ def converter(cspec_like, *args, **kwargs):
def _to_dict(compiler): def _to_dict(compiler):
"""Return a dict version of compiler suitable to insert in YAML.""" """Return a dict version of compiler suitable to insert in YAML."""
d = {}
d['spec'] = str(compiler.spec)
d['paths'] = dict( (attr, getattr(compiler, attr, None)) for attr in _path_instance_vars )
d['operating_system'] = compiler.operating_system.to_dict()
d['modules'] = compiler.modules
if not compiler.alias:
yaml_text = yaml.dump(
d, default_flow_style=True, width=sys.maxint)
sha = hashlib.sha1(yaml_text)
compiler.alias = base64.b32encode(sha.digest()).lower()[:8]
return { return {
str(compiler.spec) : dict( compiler.alias: d
(attr, getattr(compiler, attr, None))
for attr in _required_instance_vars + _optional_instance_vars)
} }
def get_compiler_config(arch=None, scope=None): def get_compiler_config(scope=None):
"""Return the compiler configuration for the specified architecture. """Return the compiler configuration for the specified architecture.
""" """
# Check whether we're on a front-end (native) architecture.
my_arch = spack.architecture.Arch()
if arch is None:
arch = my_arch
if isinstance(arch, basestring) and arch == 'all':
name = 'all'
else:
name = arch.platform.name
def init_compiler_config(): def init_compiler_config():
"""Compiler search used when Spack has no compilers.""" """Compiler search used when Spack has no compilers."""
config[name] = {} config = {}
compilers = find_compilers(*get_path('PATH')) compilers = find_compilers()
for compiler in compilers: for compiler in compilers:
config[name].update(_to_dict(compiler)) config.update(_to_dict(compiler))
spack.config.update_config('compilers', config, scope=scope) spack.config.update_config('compilers', config, scope=scope)
config = spack.config.get_config('compilers', scope=scope) config = spack.config.get_config('compilers', scope=scope)
# Update the configuration if there are currently no compilers # Update the configuration if there are currently no compilers
# configured. Avoid updating automatically if there ARE site # configured. Avoid updating automatically if there ARE site
# compilers configured but no user ones. # compilers configured but no user ones.
if (isinstance(arch, basestring) or arch == my_arch) and arch not in config: # if (isinstance(arch, basestring) or arch == my_arch) and arch not in config:
if scope is None: if scope is None:
# We know no compilers were configured in any scope. # We know no compilers were configured in any scope.
init_compiler_config()
elif scope == 'user':
# Check the site config and update the user config if
# nothing is configured at the site level.
site_config = spack.config.get_config('compilers', scope='site')
if not site_config:
init_compiler_config() init_compiler_config()
elif scope == 'user':
# Check the site config and update the user config if
# nothing is configured at the site level.
site_config = spack.config.get_config('compilers', scope='site')
if not site_config:
init_compiler_config()
return config[name] if name in config else {} return config
def add_compilers_to_config(compilers, arch=None, scope=None): def add_compilers_to_config(compilers, scope=None):
"""Add compilers to the config for the specified architecture. """Add compilers to the config for the specified architecture.
Arguments: Arguments:
- compilers: a list of Compiler objects. - compilers: a list of Compiler objects.
- arch: arch to add compilers for.
- scope: configuration scope to modify. - scope: configuration scope to modify.
""" """
if arch is None: compiler_config = get_compiler_config(scope)
arch = spack.architecture.Arch()
compiler_config = get_compiler_config(arch, scope)
for compiler in compilers: for compiler in compilers:
compiler_config[str(compiler.spec)] = dict( compiler_config = _to_dict(compiler)
(c, getattr(compiler, c, "None"))
for c in _required_instance_vars + _optional_instance_vars)
update = { arch.platform.name : compiler_config } spack.config.update_config('compilers', compiler_config, scope)
spack.config.update_config('compilers', update, scope)
@_auto_compiler_spec @_auto_compiler_spec
def remove_compiler_from_config(compiler_spec, arch=None, scope=None): def remove_compiler_from_config(compiler_spec, scope=None):
"""Remove compilers from the config, by spec. """Remove compilers from the config, by spec.
Arguments: Arguments:
- compiler_specs: a list of CompilerSpec objects. - compiler_specs: a list of CompilerSpec objects.
- arch: arch to add compilers for.
- scope: configuration scope to modify. - scope: configuration scope to modify.
""" """
if arch is None: compiler_config = get_compiler_config(scope)
arch = spack.architecture.Arch() matches = [(a,c) for (a,c) in compiler_config.items() if c['spec'] == compiler_spec]
if len(matches) == 1:
del compiler_config[matches[0][0]]
else:
CompilerSpecInsufficientlySpecificError(compiler_spec)
compiler_config = get_compiler_config(arch, scope) spack.config.update_config('compilers', compiler_config, scope)
del compiler_config[str(compiler_spec)]
update = { arch : compiler_config }
spack.config.update_config('compilers', update, scope)
_cache_config_file = {} _cache_config_file = {}
def all_compilers_config(arch=None, scope=None): def all_compilers_config(scope=None):
"""Return a set of specs for all the compiler versions currently """Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec. available to build with. These are instances of CompilerSpec.
""" """
@ -161,20 +157,16 @@ def all_compilers_config(arch=None, scope=None):
global _cache_config_file #Create a cache of the config file so we don't load all the time. global _cache_config_file #Create a cache of the config file so we don't load all the time.
if not _cache_config_file: if not _cache_config_file:
arch_config = get_compiler_config(arch, scope) _cache_config_file = get_compiler_config(scope)
# Merge 'all' compilers with arch-specific ones.
# Arch-specific compilers have higher precedence.
_cache_config_file = get_compiler_config('all', scope=scope)
_cache_config_file = spack.config._merge_yaml(_cache_config_file, arch_config)
return _cache_config_file return _cache_config_file
else: else:
return _cache_config_file return _cache_config_file
def all_compilers(arch=None, scope=None): def all_compilers(scope=None):
# Return compiler specs from the merged config. # Return compiler specs from the merged config.
return [spack.spec.CompilerSpec(s) return [spack.spec.CompilerSpec(s['spec'])
for s in all_compilers_config(arch, scope)] for s in all_compilers_config(scope).values()]
def default_compiler(): def default_compiler():
@ -189,37 +181,19 @@ def default_compiler():
return sorted(versions)[-1] return sorted(versions)[-1]
def find_compilers(*path): def find_compilers():
"""Return a list of compilers found in the suppied paths. """Return a list of compilers found in the suppied paths.
This invokes the find() method for each Compiler class, This invokes the find_compilers() method for each operating
and appends the compilers detected to a list. system associated with the host platform, and appends
the compilers detected to a list.
""" """
# Make sure path elements exist, and include /bin directories # Find compilers for each operating system class
# under prefixes. oss = all_os_classes()
filtered_path = [] compiler_lists = []
for p in path: for os in oss:
# Eliminate symlinks and just take the real directories. compiler_lists.extend(os.find_compilers())
p = os.path.realpath(p)
if not os.path.isdir(p):
continue
filtered_path.append(p)
# Check for a bin directory, add it if it exists
bin = join_path(p, 'bin')
if os.path.isdir(bin):
filtered_path.append(os.path.realpath(bin))
# Once the paths are cleaned up, do a search for each type of
# compiler. We can spawn a bunch of parallel searches to reduce
# the overhead of spelunking all these directories.
types = all_compiler_types()
compiler_lists = parmap(lambda cls: cls.find(*filtered_path), types)
# ensure all the version calls we made are cached in the parent
# process, as well. This speeds up Spack a lot.
clist = reduce(lambda x,y: x+y, compiler_lists)
return clist
return compiler_lists
def supported_compilers(): def supported_compilers():
"""Return a set of names of compilers supported by Spack. """Return a set of names of compilers supported by Spack.
@ -237,47 +211,60 @@ def supported(compiler_spec):
@_auto_compiler_spec @_auto_compiler_spec
def find(compiler_spec, arch=None, scope=None): def find(compiler_spec, scope=None):
"""Return specs of available compilers that match the supplied """Return specs of available compilers that match the supplied
compiler spec. Return an list if nothing found.""" compiler spec. Return an list if nothing found."""
return [c for c in all_compilers(arch, scope) if c.satisfies(compiler_spec)] return [c for c in all_compilers(scope) if c.satisfies(compiler_spec)]
@_auto_compiler_spec @_auto_compiler_spec
def compilers_for_spec(compiler_spec, arch=None, scope=None): def compilers_for_spec(compiler_spec, scope=None):
"""This gets all compilers that satisfy the supplied CompilerSpec. """This gets all compilers that satisfy the supplied CompilerSpec.
Returns an empty list if none are found. Returns an empty list if none are found.
""" """
config = all_compilers_config(arch, scope) config = all_compilers_config(scope)
def get_compiler(cspec): def get_compilers(cspec):
items = config[str(cspec)] compilers = []
if not all(n in items for n in _required_instance_vars): for aka, cmp in config.items():
raise InvalidCompilerConfigurationError(cspec) if cmp['spec'] != str(cspec):
continue
items = cmp
alias = aka
if not ('paths' in items and all(n in items['paths'] for n in _path_instance_vars)):
raise InvalidCompilerConfigurationError(cspec)
cls = class_for_compiler_name(cspec.name) cls = class_for_compiler_name(cspec.name)
strategy = items['strategy'] compiler_paths = []
if not strategy: for c in _path_instance_vars:
raise InvalidCompilerConfigurationError(cspec) compiler_path = items['paths'][c]
if compiler_path != "None":
compiler_paths.append(compiler_path)
else:
compiler_paths.append(None)
compiler_paths = [] mods = items.get('modules')
for c in _required_instance_vars: if mods == 'None':
compiler_path = items[c] mods = []
if compiler_path != "None":
compiler_paths.append(compiler_path) if 'operating_system' in items:
operating_system = spack.architecture._operating_system_from_dict( items['operating_system'] )
else: else:
compiler_paths.append(None) operating_system = None
if 'modules' not in items: compilers.append(cls(cspec, operating_system, compiler_paths, mods, alias))
items['modules'] = None
mods = items['modules']
return cls(cspec, strategy, compiler_paths, mods) return compilers
matches = find(compiler_spec, arch, scope) matches = set(find(compiler_spec, scope))
return [get_compiler(cspec) for cspec in matches] compilers = []
for cspec in matches:
compilers.extend(get_compilers(cspec))
return compilers
# return [get_compilers(cspec) for cspec in matches]
@_auto_compiler_spec @_auto_compiler_spec
@ -285,8 +272,9 @@ def compiler_for_spec(compiler_spec, operating_system):
"""Get the compiler that satisfies compiler_spec. compiler_spec must """Get the compiler that satisfies compiler_spec. compiler_spec must
be concrete.""" be concrete."""
assert(compiler_spec.concrete) assert(compiler_spec.concrete)
compilers = [c for c in compilers_for_spec(compiler_spec) compilers = [c for c in compilers_for_spec(compiler_spec)
if c.strategy == operating_system.compiler_strategy] if c.operating_system == operating_system]
if len(compilers) < 1: if len(compilers) < 1:
raise NoCompilerForSpecError(compiler_spec, operating_system) raise NoCompilerForSpecError(compiler_spec, operating_system)
if len(compilers) > 1: if len(compilers) > 1:
@ -308,8 +296,20 @@ def class_for_compiler_name(compiler_name):
return cls return cls
def all_os_classes():
"""
Return the list of classes for all operating systems available on
this platform
"""
classes = []
platform = spack.architecture.sys_type()
for os_class in platform.operating_sys.values():
classes.append(os_class)
return classes
def all_compiler_types(): def all_compiler_types():
# return [class_for_compiler_name(c) for c in ['gcc']]
return [class_for_compiler_name(c) for c in supported_compilers()] return [class_for_compiler_name(c) for c in supported_compilers()]
@ -318,7 +318,7 @@ def __init__(self, compiler_spec):
super(InvalidCompilerConfigurationError, self).__init__( super(InvalidCompilerConfigurationError, self).__init__(
"Invalid configuration for [compiler \"%s\"]: " % compiler_spec, "Invalid configuration for [compiler \"%s\"]: " % compiler_spec,
"Compiler configuration must contain entries for all compilers: %s" "Compiler configuration must contain entries for all compilers: %s"
% _required_instance_vars) % _path_instance_vars)
class NoCompilersError(spack.error.SpackError): class NoCompilersError(spack.error.SpackError):

8
lib/spack/spack/concretize.py
View File

@ -301,15 +301,11 @@ def concretize_compiler(self, spec):
# Should think whether this can be more efficient # Should think whether this can be more efficient
def _proper_compiler_style(cspec, architecture): def _proper_compiler_style(cspec, architecture):
compilers = spack.compilers.compilers_for_spec(cspec) compilers = spack.compilers.compilers_for_spec(cspec)
filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers) filter(lambda c: c.operating_system == architecture.platform_os, compilers)
#if architecture.platform_os.compiler_strategy == 'PATH':
# filter(lambda c: not c.modules, compilers)
#if architecture.platform_os.compiler_strategy == 'MODULES':
# filter(lambda c: c.modules, compilers)
return compilers return compilers
all_compilers = spack.compilers.all_compilers(spec.architecture) all_compilers = spack.compilers.all_compilers()
if (spec.compiler and if (spec.compiler and
spec.compiler.concrete and spec.compiler.concrete and

35
lib/spack/spack/config.py
View File

@ -151,17 +151,16 @@
'default': {}, 'default': {},
'additionalProperties': False, 'additionalProperties': False,
'patternProperties': { 'patternProperties': {
r'\w[\w-]*': { # architecture r'\w[\w-]*': { # alias
'type': 'object', 'type': 'object',
'additionalProperties': False, 'additionalProperties': False,
'patternProperties': { 'required': ['paths', 'spec', 'modules', 'operating_system'],
r'\w[\w-]*@\w[\w-]*': { # compiler spec 'properties': {
'paths': {
'type': 'object', 'type': 'object',
'additionalProperties': False,
'required': ['cc', 'cxx', 'f77', 'fc'], 'required': ['cc', 'cxx', 'f77', 'fc'],
'properties': { 'additionalProperties': False,
'strategy': { 'anyOf': [ {'type' : 'string' }, 'properties': {
{'type' : 'null' }]},
'cc': { 'anyOf': [ {'type' : 'string' }, 'cc': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]}, {'type' : 'null' }]},
'cxx': { 'anyOf': [ {'type' : 'string' }, 'cxx': { 'anyOf': [ {'type' : 'string' },
@ -169,13 +168,21 @@
'f77': { 'anyOf': [ {'type' : 'string' }, 'f77': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]}, {'type' : 'null' }]},
'fc': { 'anyOf': [ {'type' : 'string' }, 'fc': { 'anyOf': [ {'type' : 'string' },
{'type' : 'null' }]}, {'type' : 'null' }]}}},
'modules': { 'anyOf': [ {'type' : 'string'}, 'spec': { 'type': 'string'},#r'\w[\w-]*@\w[\w-]*'
{'type' : 'null' }, 'operating_system': {
{'type': 'array'}, 'type': 'object',
]} 'required': ['name', 'version'],
},},},},},},},}, 'additionalProperties': False,
'properties': {
'name': {'type': 'string'},
'version': {'type': 'string'}
}},
'modules': { 'anyOf': [ {'type' : 'string'},
{'type' : 'null' },
{'type': 'array'},
]}
},},},},},},
'mirrors': { 'mirrors': {
'$schema': 'http://json-schema.org/schema#', '$schema': 'http://json-schema.org/schema#',
'title': 'Spack mirror configuration file schema', 'title': 'Spack mirror configuration file schema',

51
lib/spack/spack/operating_systems/cnl.py
View File

@ -1,4 +1,11 @@
import re
import os
from spack.architecture import OperatingSystem from spack.architecture import OperatingSystem
from spack.util.executable import *
import spack.spec
from spack.util.multiproc import parmap
import spack.compilers
class Cnl(OperatingSystem): class Cnl(OperatingSystem):
""" Compute Node Linux (CNL) is the operating system used for the Cray XC """ Compute Node Linux (CNL) is the operating system used for the Cray XC
@ -10,4 +17,46 @@ class Cnl(OperatingSystem):
def __init__(self): def __init__(self):
name = 'CNL' name = 'CNL'
version = '10' version = '10'
super(Cnl, self).__init__(name, version, "MODULES") super(Cnl, self).__init__(name, version)
def find_compilers(self, *paths):
types = spack.compilers.all_compiler_types()
compiler_lists = parmap(lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
# ensure all the version calls we made are cached in the parent
# process, as well. This speeds up Spack a lot.
clist = reduce(lambda x,y: x+y, compiler_lists)
return clist
def find_compiler(self, cmp_cls, *paths):
compilers = []
if cmp_cls.PrgEnv:
if not cmp_cls.PrgEnv_compiler:
tty.die('Must supply PrgEnv_compiler with PrgEnv')
modulecmd = which('modulecmd')
modulecmd.add_default_arg('python')
# Save the environment variable to restore later
old_modulepath = os.environ['MODULEPATH']
# if given any explicit paths, search them for module files too
if paths:
module_paths = ':' + ':'.join(p for p in paths)
os.environ['MODULEPATH'] = module_paths
output = modulecmd('avail', cmp_cls.PrgEnv_compiler, output=str, error=str)
matches = re.findall(r'(%s)/([\d\.]+[\d])' % cmp_cls.PrgEnv_compiler, output)
for name, version in matches:
v = version
comp = cmp_cls(spack.spec.CompilerSpec(name + '@' + v), self,
['cc', 'CC', 'ftn'], [cmp_cls.PrgEnv, name +'/' + v])
compilers.append(comp)
# Restore modulepath environment variable
if paths:
os.environ['MODULEPATH'] = old_modulepath
return compilers

39
lib/spack/spack/test/architecture.py
View File

@ -12,8 +12,18 @@
from spack.platforms.bgq import Bgq from spack.platforms.bgq import Bgq
from spack.platforms.darwin import Darwin from spack.platforms.darwin import Darwin
class ArchitectureTest(unittest.TestCase): from spack.test.mock_packages_test import *
#class ArchitectureTest(unittest.TestCase):
class ArchitectureTest(MockPackagesTest):
def setUp(self):
super(ArchitectureTest, self).setUp()
self.platform = sys_type()
def tearDown(self):
super(ArchitectureTest, self).tearDown()
def test_dict_functions_for_architecture(self): def test_dict_functions_for_architecture(self):
arch = Arch() arch = Arch()
arch.platform_os = arch.platform.operating_system('default_os') arch.platform_os = arch.platform.operating_system('default_os')
@ -34,13 +44,13 @@ def test_dict_functions_for_architecture(self):
self.assertTrue( isinstance(new_arch.target, Target) ) self.assertTrue( isinstance(new_arch.target, Target) )
def test_platform_class_and_compiler_strategies(self): # def test_platform_class_and_compiler_strategies(self):
a = CrayXc() # a = CrayXc()
t = a.operating_system('default_os') # t = a.operating_system('default_os')
self.assertEquals(t.compiler_strategy, 'MODULES') # self.assertEquals(t.compiler_strategy, 'MODULES')
b = Linux() # b = Linux()
s = b.operating_system('default_os') # s = b.operating_system('default_os')
self.assertEquals(s.compiler_strategy, 'PATH') # self.assertEquals(s.compiler_strategy, 'PATH')
def test_sys_type(self): def test_sys_type(self):
output_platform_class = sys_type() output_platform_class = sys_type()
@ -56,16 +66,13 @@ def test_sys_type(self):
self.assertEqual(str(output_platform_class), str(my_platform_class)) self.assertEqual(str(output_platform_class), str(my_platform_class))
def setUp(self):
self.platform = sys_type()
def test_user_front_end_input(self): def test_user_front_end_input(self):
"""Test when user inputs just frontend that both the frontend target """Test when user inputs just frontend that both the frontend target
and frontend operating system match and frontend operating system match
""" """
frontend_os = self.platform.operating_system("frontend") frontend_os = self.platform.operating_system("frontend")
frontend_target = self.platform.target("frontend") frontend_target = self.platform.target("frontend")
frontend_spec = Spec("zlib=frontend") frontend_spec = Spec("libelf=frontend")
frontend_spec.concretize() frontend_spec.concretize()
self.assertEqual(frontend_os, frontend_spec.architecture.platform_os) self.assertEqual(frontend_os, frontend_spec.architecture.platform_os)
self.assertEqual(frontend_target, frontend_spec.architecture.target) self.assertEqual(frontend_target, frontend_spec.architecture.target)
@ -76,7 +83,7 @@ def test_user_back_end_input(self):
""" """
backend_os = self.platform.operating_system("backend") backend_os = self.platform.operating_system("backend")
backend_target = self.platform.target("backend") backend_target = self.platform.target("backend")
backend_spec = Spec("zlib=backend") backend_spec = Spec("libelf=backend")
backend_spec.concretize() backend_spec.concretize()
self.assertEqual(backend_os, backend_spec.architecture.platform_os) self.assertEqual(backend_os, backend_spec.architecture.platform_os)
self.assertEqual(backend_target, backend_spec.architecture.target) self.assertEqual(backend_target, backend_spec.architecture.target)
@ -85,7 +92,7 @@ def test_user_defaults(self):
default_os = self.platform.operating_system("default_os") default_os = self.platform.operating_system("default_os")
default_target = self.platform.target("default_target") default_target = self.platform.target("default_target")
default_spec = Spec("zlib") # default is no args default_spec = Spec("libelf") # default is no args
default_spec.concretize() default_spec.concretize()
self.assertEqual(default_os, default_spec.architecture.platform_os) self.assertEqual(default_os, default_spec.architecture.platform_os)
self.assertEqual(default_target, default_spec.architecture.target) self.assertEqual(default_target, default_spec.architecture.target)
@ -103,11 +110,11 @@ def test_user_input_combination(self):
for arch in combinations: for arch in combinations:
o,t = arch o,t = arch
arch_spec = "-".join(arch) arch_spec = "-".join(arch)
spec = Spec("zlib=%s" % arch_spec) spec = Spec("libelf=%s" % arch_spec)
spec.concretize() spec.concretize()
results.append(spec.architecture.platform_os == self.platform.operating_system(o)) results.append(spec.architecture.platform_os == self.platform.operating_system(o))
results.append(spec.architecture.target == self.platform.target(t)) results.append(spec.architecture.target == self.platform.target(t))
res = all(results) res = all(results)
print res
self.assertTrue(res) self.assertTrue(res)

95
lib/spack/spack/test/config.py
View File

@ -33,43 +33,91 @@
# Some sample compiler config data # Some sample compiler config data
a_comps = { a_comps = {
"all": { 'gcc473': {
"gcc@4.7.3" : { 'paths': {
"cc" : "/gcc473", "cc" : "/gcc473",
"cxx": "/g++473", "cxx": "/g++473",
"f77": None, "f77": None,
"fc" : None }, "fc" : None
"gcc@4.5.0" : { },
'modules': None,
'spec': 'gcc@4.7.3',
'operating_system': {
'name': 'CNL',
'version': '10'
}
},
'gcc450': {
'paths': {
"cc" : "/gcc450", "cc" : "/gcc450",
"cxx": "/g++450", "cxx": "/g++450",
"f77": "/gfortran", "f77": 'gfortran',
"fc" : "/gfortran" }, "fc" : 'gfortran'
"clang@3.3" : { },
'modules': None,
'spec': 'gcc@4.5.0',
'operating_system': {
'name': 'CNL',
'version': '10'
}
},
'clang33': {
'paths': {
"cc" : "<overwritten>", "cc" : "<overwritten>",
"cxx": "<overwritten>", "cxx": "<overwritten>",
"f77": "<overwritten>", "f77": '<overwritten>',
"fc" : "<overwritten>" } "fc" : '<overwritten>' },
} 'modules': None,
'spec': 'clang@3.3',
'operating_system': {
'name': 'CNL',
'version': '10'
}
}
} }
b_comps = { b_comps = {
"all": { 'icc100': {
"icc@10.0" : { 'paths': {
"cc" : "/icc100", "cc" : "/icc100",
"cxx": "/icc100", "cxx": "/icp100",
"f77": None, "f77": None,
"fc" : None }, "fc" : None
"icc@11.1" : { },
'modules': None,
'spec': 'icc@10.0',
'operating_system': {
'name': 'CNL',
'version': '10'
}
},
'icc111': {
'paths': {
"cc" : "/icc111", "cc" : "/icc111",
"cxx": "/icp111", "cxx": "/icp111",
"f77": "/ifort", "f77": 'ifort',
"fc" : "/ifort" }, "fc" : 'ifort'
"clang@3.3" : { },
"cc" : "/clang", 'modules': None,
"cxx": "/clang++", 'spec': 'icc@11.1',
"f77": None, 'operating_system': {
"fc" : None} 'name': 'CNL',
} 'version': '10'
}
},
'clang33': {
'paths': {
"cc" : "<overwritten>",
"cxx": "<overwritten>",
"f77": '<overwritten>',
"fc" : '<overwritten>' },
'modules': None,
'spec': 'clang@3.3',
'operating_system': {
'name': 'CNL',
'version': '10'
}
}
} }
class ConfigTest(MockPackagesTest): class ConfigTest(MockPackagesTest):
@ -96,7 +144,6 @@ def check_config(self, comps, *compiler_names):
actual = config['all'][key][c] actual = config['all'][key][c]
self.assertEqual(expected, actual) self.assertEqual(expected, actual)
def test_write_key_in_memory(self): def test_write_key_in_memory(self):
# Write b_comps "on top of" a_comps. # Write b_comps "on top of" a_comps.
spack.config.update_config('compilers', a_comps, 'test_low_priority') spack.config.update_config('compilers', a_comps, 'test_low_priority')

43
lib/spack/spack/test/mock_packages_test.py
View File

@ -36,21 +36,50 @@
mock_compiler_config = """\ mock_compiler_config = """\
compilers: compilers:
all: clang3.3CNL:
clang@3.3: spec: clang@3.3
operating_system:
name: CNL
version: '10'
paths:
cc: /path/to/clang cc: /path/to/clang
cxx: /path/to/clang++ cxx: /path/to/clang++
f77: None f77: None
fc: None fc: None
strategy: PATH modules: 'None'
modules: None clang3.3RHL:
gcc@4.5.0: spec: clang@3.3
operating_system:
name: redhat
version: '6.7'
paths:
cc: /path/to/clang
cxx: /path/to/clang++
f77: None
fc: None
modules: 'None'
gcc4.5.0CNL:
paths:
cc: /path/to/gcc cc: /path/to/gcc
cxx: /path/to/g++ cxx: /path/to/g++
f77: /path/to/gfortran f77: /path/to/gfortran
fc: /path/to/gfortran fc: /path/to/gfortran
strategy: PATH operating_system:
modules: None name: CNL
version: '10'
spec: gcc@4.5.0
modules: 'None'
gcc4.5.0RHL:
paths:
cc: /path/to/gcc
cxx: /path/to/g++
f77: /path/to/gfortran
fc: /path/to/gfortran
operating_system:
name: RHL
version: '6.7'
spec: gcc@4.5.0
modules: 'None'
""" """
mock_packages_config = """\ mock_packages_config = """\

2
lib/spack/spack/test/spec_dag.py
View File

@ -243,7 +243,7 @@ def test_unsatisfiable_target(self):
if len(platform.targets) > 1: if len(platform.targets) > 1:
first = platform.targets.values()[0].name first = platform.targets.values()[0].name
second = platform.targets.values()[1].name second = platform.targets.values()[1].name
set_pkg_dep('mpileaks', 'mpich='+first) self.set_pkg_dep('mpileaks', 'mpich='+first)
spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf') spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf')
self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize) self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize)