zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merged newarch into merge

Browse Source
This commit is contained in:
Gregory Becker 2016-03-22 15:22:51 -07:00
commit 7c729d4c3c
42 changed files with 1318 additions and 183 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/__init__.py
View File

@ -39,7 +39,9 @@
lib_path = join_path(spack_root, "lib", "spack")
build_env_path = join_path(lib_path, "env")
module_path = join_path(lib_path, "spack")
platform_path = join_path(module_path, 'platforms')
compilers_path = join_path(module_path, "compilers")
operating_system_path = join_path(module_path, 'operating_systems')
test_path = join_path(module_path, "test")
hooks_path = join_path(module_path, "hooks")
var_path = join_path(spack_root, "var", "spack")

5
lib/spack/spack/abi.py
View File

@ -35,8 +35,9 @@ class ABI(object):
The current implementation is rather rough and could be improved."""
def architecture_compatible(self, parent, child):
"""Returns true iff the parent and child specs have ABI compatible architectures."""
return not parent.architecture or not child.architecture or parent.architecture == child.architecture
"""Returns true iff the parent and child specs have ABI compatible targets."""
return not parent.architecture or not child.architecture \
or parent.architecture == child.architecture
@memoized

334
lib/spack/spack/architecture.py
View File

@ -22,13 +22,17 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import re
import platform
from collections import namedtuple
import imp
import platform as py_platform
import inspect
from llnl.util.lang import memoized
from llnl.util.lang import memoized, list_modules, key_ordering
from llnl.util.filesystem import join_path
import llnl.util.tty as tty
import spack
from spack.util.naming import mod_to_class
import spack.error as serr
@ -44,46 +48,302 @@ def __init__(self):
"Could not determine sys_type for this machine.")
def get_sys_type_from_spack_globals():
"""Return the SYS_TYPE from spack globals, or None if it isn't set."""
if not hasattr(spack, "sys_type"):
@key_ordering
class Target(object):
""" Target is the processor of the host machine.
The host machine may have different front-end and back-end targets,
especially if it is a Cray machine. The target will have a name and
also the module_name (e.g craype-compiler). Targets will also
recognize which platform they came from using the set_platform method.
Targets will have compiler finding strategies
"""
def __init__(self, name, module_name=None):
self.name = name # case of cray "ivybridge" but if it's x86_64
self.module_name = module_name # craype-ivybridge
# Sets only the platform name to avoid recursiveness
def _cmp_key(self):
return (self.name, self.module_name)
def __repr__(self):
return self.__str__()
def __str__(self):
return self.name
def to_dict(self):
d = {}
d['name'] = self.name
d['module_name'] = self.module_name
return d
@key_ordering
class Platform(object):
""" Abstract class that each type of Platform will subclass.
Will return a instance of it once it
is returned
"""
priority = None # Subclass needs to set this number. This controls order in which platform is detected.
front_end = None
back_end = None
default = None # The default back end target. On cray ivybridge
front_os = None
back_os = None
default_os = None
def __init__(self, name):
self.targets = {}
self.operating_sys = {}
self.name = name
def add_target(self, name, target):
"""Used by the platform specific subclass to list available targets.
Raises an error if the platform specifies a name
that is reserved by spack as an alias.
"""
if name in ['front_end', 'fe', 'back_end', 'be', 'default']:
raise ValueError(
"%s is a spack reserved alias "
"and cannot be the name of a target" % name)
self.targets[name] = target
def target(self, name):
"""This is a getter method for the target dictionary
that handles defaulting based on the values provided by default,
front-end, and back-end. This can be overwritten
by a subclass for which we want to provide further aliasing options.
"""
if name == 'default':
name = self.default
elif name == 'front_end' or name == 'fe':
name = self.front_end
elif name == 'back_end' or name == 'be':
name = self.back_end
return self.targets[name]
def add_operating_system(self, name, os_class):
""" Add the operating_system class object into the
platform.operating_sys dictionary
"""
self.operating_sys[name] = os_class
def operating_system(self, name):
if name == 'default_os':
name = self.default_os
if name == 'front_os':
name = self.front_os
if name == 'back_os':
name = self.back_os
return self.operating_sys[name]
@classmethod
def detect(self):
""" Subclass is responsible for implementing this method.
Returns True if the Platform class detects that
it is the current platform
and False if it's not.
"""
raise NotImplementedError()
def __repr__(self):
return self.__str__()
def __str__(self):
return self.name
def _cmp_key(self):
return (self.name, (_cmp_key(t) for t in self.targets.values()),
(_cmp_key(o) for o in self.operating_sys.values()))
@key_ordering
class OperatingSystem(object):
""" Operating System will be like a class similar to platform extended
by subclasses for the specifics. Operating System will contain the
compiler finding logic. Instead of calling two separate methods to
find compilers we call find_compilers method for each operating system
"""
def __init__(self, name, version, compiler_strategy="PATH"):
self.name = name
self.version = version
self.compiler_strategy = compiler_strategy
def __str__(self):
return self.name + self.version
def __repr__(self):
return self.__str__()
def _cmp_key(self):
return (self.name, self.version, self.compiler_strategy)
def to_dict(self):
d = {}
d['name'] = self.name
d['version'] = self.version
d['compiler_strategy'] = self.compiler_strategy
return d
#NOTE: Key error caused because Architecture has no comparison method
@key_ordering
class Arch(object):
"Architecture is now a class to help with setting attributes"
def __init__(self, platform_os=None, target=None):
self.platform = sys_type()
self.platform_os = platform_os
self.target = target
def __str__(self):
return (str(self.platform) +"-"+
str(self.platform_os) + "-" + str(self.target) )
def _cmp_key(self):
platform = self.platform.name
os = self.platform_os.name if isinstance(self.platform_os, OperatingSystem) else self.platform_os
target = self.target.name if isinstance(self.target, Target) else self.target
return (platform, os, target)
def to_dict(self):
d = {}
platform = self.platform
platform_os = self.platform_os
target = self.target
d['platform'] = self.platform.name
d['platform_os'] = self.platform_os.to_dict()
d['target'] = self.target.to_dict()
return d
#def _helper_to_dict(arch_field_dict, arch_field_name, *args):
# """ General method to turn each class in architecture into a
# dictionary. Takes as argument the class dictionary, the field name
# (platform, platform_os, target) and then any attribute args
# """
# d = {}
# d[arch_field_name] = {}
# for items in args:
# d[arch_field_name][items] = arch_field_dict[items]
# return d
#
#def to_dict(arch):
# """ Convert the Arch tuple into a dictionary for yaml dumping. This
# uses the _helper_to_dict method to create the dictionary from the
# provided architecture field. Can assign the architecture
# field name (either platform, platform_os or target) and any
# attributes that make up that architecture field,
# """
# d = {}
#
# platform = arch.platform.__dict__
# platform_os = arch.platform_os.__dict__
# target = arch.target.__dict__
#
# platform_dict = _helper_to_dict(platform,'platform','name')
# os_dict = _helper_to_dict(platform_os, 'platform_os', 'name','version',
# 'compiler_strategy')
# target_dict = _helper_to_dict(target,'target', 'name',
# 'module_name','platform_name')
#
# d.update(platform_dict)
# d.update(os_dict)
# d.update(target_dict)
#
# return d
#def _platform_from_dict(platform):
# """Creates all the platform class module names into a dictionary of
# name : <class_mod> key-value pairs. From there we can construct the
# platform subclass
# """
# platform_list = all_platforms()
# platform_names = {plat.__name__.lower():plat for plat in platform_list}
# return platform_names[platform['name']]()
def _target_from_dict(target_dict):
""" Creates new instance of target and assigns all the attributes of
that target from the dictionary
"""
target = Target.__new__(Target)
target.name = target_dict['name']
#target.compiler_strategy = target_dict['compiler_strategy']
target.module_name = target_dict['module_name']
if 'platform_name' in target_dict:
target.platform_name = target_dict['platform_name']
return target
def _operating_system_from_dict(os_dict, platform_class):
""" uses platform's operating system method to grab the constructed
operating systems that are valid on the platform.
"""
# NOTE: Might need a better way to create operating system objects
name = os_dict['name']
return platform_class.operating_system(name)
def arch_from_dict(d):
""" Uses _platform_from_dict, _operating_system_from_dict, _target_from_dict
helper methods to recreate the arch tuple from the dictionary read from
a yaml file
"""
arch = Arch()
if d is None:
return None
elif hasattr(spack.sys_type, "__call__"):
return spack.sys_type()
else:
return spack.sys_type
os_dict = d['platform_os']
target_dict = d['target']
target = _target_from_dict(target_dict)
platform_os = _operating_system_from_dict(os_dict, arch.platform)
arch.target =target
arch.platform_os = platform_os
def get_sys_type_from_environment():
"""Return $SYS_TYPE or None if it's not defined."""
return os.environ.get('SYS_TYPE')
return arch
@memoized
def all_platforms():
modules = []
def get_sys_type_from_platform():
"""Return the architecture from Python's platform module."""
sys_type = platform.system() + '-' + platform.machine()
sys_type = re.sub(r'[^\w-]', '_', sys_type)
return sys_type.lower()
mod_path = spack.platform_path
mod_string = "spack.platformss"
for name in list_modules(mod_path):
mod_name = mod_string + name
path = join_path(mod_path, name) + ".py"
mod = imp.load_source(mod_name, path)
class_name = mod_to_class(name)
if not hasattr(mod, class_name):
tty.die('No class %s defined in %s' % (class_name, mod_name))
cls = getattr(mod, class_name)
if not inspect.isclass(cls):
tty.die('%s.%s is not a class' % (mod_name, class_name))
modules.append(cls)
return modules
@memoized
def sys_type():
"""Returns a SysType for the current machine."""
methods = [get_sys_type_from_spack_globals,
get_sys_type_from_environment,
get_sys_type_from_platform]
""" Gather a list of all available subclasses of platforms.
Sorts the list according to their priority looking. Priority is
an arbitrarily set number. Detects platform either using uname or
a file path (/opt/cray...)
"""
# Try to create a Platform object using the config file FIRST
platform_list = all_platforms()
platform_list.sort(key=lambda a: a.priority)
# search for a method that doesn't return None
sys_type = None
for method in methods:
sys_type = method()
if sys_type: break
# Couldn't determine the sys_type for this machine.
if sys_type is None:
return "unknown_arch"
if not isinstance(sys_type, basestring):
raise InvalidSysTypeError(sys_type)
return sys_type
for platform in platform_list:
if platform.detect():
return platform()

88
lib/spack/spack/build_environment.py
View File

@ -32,6 +32,8 @@
import shutil
import multiprocessing
import platform
import re
from llnl.util.filesystem import *
import spack
@ -57,6 +59,9 @@
SPACK_SHORT_SPEC = 'SPACK_SHORT_SPEC'
SPACK_DEBUG_LOG_DIR = 'SPACK_DEBUG_LOG_DIR'
SPACK_CRAYPE = 'SPACK_CRAYPE'
SPACK_COMP_MODULE = 'SPACK_COMP_MODULE'
class MakeExecutable(Executable):
"""Special callable executable object for make so the user can
@ -83,6 +88,68 @@ def __call__(self, *args, **kwargs):
return super(MakeExecutable, self).__call__(*args, **kwargs)
def load_module(mod):
"""Takes a module name and removes modules until it is possible to
load that module. It then loads the provided module. Depends on the
modulecmd implementation of modules used in cray and lmod.
"""
#Create an executable of the module command that will output python code
modulecmd = which('modulecmd')
modulecmd.add_default_arg('python')
# Read the module and remove any conflicting modules
# We do this without checking that they are already installed
# for ease of programming because unloading a module that is not
# loaded does nothing.
text = modulecmd('show', mod, return_oe=True).split()
for i, word in enumerate(text):
if word == 'conflict':
exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec'))
# Load the module now that there are no conflicts
load = modulecmd('load', mod, return_oe=True)
exec(compile(load, '<string>', 'exec'))
def get_path_from_module(mod):
"""Inspects a TCL module for entries that indicate the absolute path
at which the library supported by said module can be found.
"""
# Create a modulecmd executable
modulecmd = which('modulecmd')
modulecmd.add_default_arg('python')
# Read the module
text = modulecmd('show', mod, return_oe=True).split('\n')
# If it lists its package directory, return that
for line in text:
if line.find(mod.upper()+'_DIR') >= 0:
words = line.split()
return words[2]
# If it lists a -rpath instruction, use that
for line in text:
rpath = line.find('-rpath/')
if rpath >= 0:
return line[rpath+6:line.find('/lib')]
# If it lists a -L instruction, use that
for line in text:
L = line.find('-L/')
if L >= 0:
return line[L+2:line.find('/lib')]
# If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that
for line in text:
if line.find('LD_LIBRARY_PATH') >= 0:
words = line.split()
path = words[2]
return path[:path.find('/lib')]
# Unable to find module path
return None
def set_compiler_environment_variables(pkg):
assert(pkg.spec.concrete)
compiler = pkg.compiler
@ -109,6 +176,10 @@ def set_compiler_environment_variables(pkg):
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
if compiler.strategy == 'MODULES':
for mod in compiler.modules:
load_module(mod)
def set_build_environment_variables(pkg):
"""This ensures a clean install environment when we build packages.
@ -172,6 +243,8 @@ def add_env_path(path):
pkg_config_dirs.append(pcdir)
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
if pkg.spec.architecture.target.module_name:
load_module(pkg.spec.architecture.target.module_name)
def set_module_variables_for_package(pkg, m):
"""Populate the module scope of install() with some useful functions.
@ -241,10 +314,21 @@ def set_module_variables_for_package(pkg, m):
def get_rpaths(pkg):
"""Get a list of all the rpaths for a package."""
# First load all modules for external packages and update the external
# packages' paths to reflect what is found in the modules so that we can
# rpath through the modules when possible, but if not possible they are
# already loaded.
for spec in pkg.spec.traverse(root=False):
if spec.external_module:
load_module(spec.external_module)
spec.external = get_path_from_module(spec.external_module)
# Construct rpaths from the paths of each dep
rpaths = [pkg.prefix.lib, pkg.prefix.lib64]
rpaths.extend(d.prefix.lib for d in pkg.spec.dependencies.values()
rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False)
if os.path.isdir(d.prefix.lib))
rpaths.extend(d.prefix.lib64 for d in pkg.spec.dependencies.values()
rpaths.extend(d.prefix.lib64 for d in pkg.spec.traverse(root=False)
if os.path.isdir(d.prefix.lib64))
return rpaths

11
lib/spack/spack/cmd/compiler.py
View File

@ -30,6 +30,8 @@
from llnl.util.tty.colify import colify
from llnl.util.lang import index_by
import spack.architecture
import spack.compiler
import spack.compilers
import spack.spec
import spack.config
@ -39,8 +41,7 @@
description = "Manage compilers"
def setup_parser(subparser):
sp = subparser.add_subparsers(
metavar='SUBCOMMAND', dest='compiler_command')
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command')
scopes = spack.config.config_scopes
@ -71,9 +72,11 @@ def setup_parser(subparser):
def compiler_add(args):
"""Search either $PATH or a list of paths for compilers and add them
"""Search either $PATH or a list of paths OR MODULES for compilers and add them
to Spack's configuration."""
paths = args.add_paths
paths = args.add_paths # This might be a parser method. Parsing method to add_paths
if not paths:
paths = get_path('PATH')

10
lib/spack/spack/cmd/find.py
View File

@ -89,19 +89,19 @@ def display_specs(specs, **kwargs):
hashes = True
hlen = None
# Make a dict with specs keyed by architecture and compiler.
index = index_by(specs, ('architecture', 'compiler'))
# Make a dict with specs keyed by target and compiler.
index = index_by(specs, ('target', 'compiler'))
# Traverse the index and print out each package
for i, (architecture, compiler) in enumerate(sorted(index)):
for i, (target, compiler) in enumerate(sorted(index)):
if i > 0: print
header = "%s{%s} / %s{%s}" % (
spack.spec.architecture_color, architecture,
spack.spec.target_color, target,
spack.spec.compiler_color, compiler)
tty.hline(colorize(header), char='-')
specs = index[(architecture,compiler)]
specs = index[(target,compiler)]
specs.sort()
nfmt = '.' if namespace else '_'

72
lib/spack/spack/compiler.py
View File

@ -33,6 +33,7 @@
import spack.error
import spack.spec
import spack.architecture
from spack.util.multiproc import parmap
from spack.util.executable import *
from spack.util.environment import get_path
@ -98,19 +99,33 @@ class Compiler(object):
cxx11_flag = "-std=c++11"
def __init__(self, cspec, cc, cxx, f77, fc):
# Cray PrgEnv name that can be used to load this compiler
PrgEnv = None
# Name of module used to switch versions of this compiler
PrgEnv_compiler = None
def __init__(self, cspec, strategy, paths, modules=None):
def check(exe):
if exe is None:
return None
_verify_executables(exe)
return exe
self.cc = check(cc)
self.cxx = check(cxx)
self.f77 = check(f77)
self.fc = check(fc)
self.strategy = strategy
self.cc = check(paths[0])
self.cxx = check(paths[1])
if len(paths) > 2:
self.f77 = check(paths[2])
if len(paths) == 3:
self.fc = self.f77
else:
self.fc = check(paths[3])
self.spec = cspec
self.modules = modules
@property
@ -206,6 +221,18 @@ def check(key):
@classmethod
def find(cls, *path):
compilers = []
platform = spack.architecture.sys_type()
strategies = [o.compiler_strategy for o in platform.operating_sys.values()]
if 'PATH' in strategies:
compilers.extend(cls.find_in_path(*path))
if 'MODULES' in strategies:
compilers.extend(cls.find_in_modules())
return compilers
@classmethod
def find_in_path(cls, *path):
"""Try to find this type of compiler in the user's
environment. For each set of compilers found, this returns
compiler objects with the cc, cxx, f77, fc paths and the
@ -250,20 +277,47 @@ def find(cls, *path):
if newcount <= prevcount:
continue
compilers[ver] = cls(spec, *paths)
compilers[ver] = cls(spec, 'PATH', paths)
return list(compilers.values())
@classmethod
def find_in_modules(cls):
compilers = []
if cls.PrgEnv:
if not cls.PrgEnv_compiler:
tty.die('Must supply PrgEnv_compiler with PrgEnv')
modulecmd = which('modulecmd')
modulecmd.add_default_arg('python')
output = modulecmd('avail', cls.PrgEnv_compiler, return_oe=True)
matches = re.findall(r'(%s)/([\d\.]+[\d])' % cls.PrgEnv_compiler, output)
for name, version in matches:
v = version
comp = cls(spack.spec.CompilerSpec(name + '@' + v), 'MODULES',
['cc', 'CC', 'ftn'], [cls.PrgEnv, name +'/' + v])
compilers.append(comp)
return compilers
def __repr__(self):
"""Return a string representation of the compiler toolchain."""
return self.__str__()
def __str__(self):
"""Return a string representation of the compiler toolchain."""
return "%s(%s)" % (
self.name, '\n '.join((str(s) for s in (self.cc, self.cxx, self.f77, self.fc))))
"""Return a string represntation of the compiler toolchain."""
if self.strategy is 'MODULES':
return "%s(%s)" % (
self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc, self.modules))))
else:
return "%s(%s)" % (
self.name, '\n '.join((str(s) for s in (self.strategy, self.cc, self.cxx, self.f77, self.fc))))
class CompilerAccessError(spack.error.SpackError):

37
lib/spack/spack/compilers/__init__.py
View File

@ -46,7 +46,9 @@
_imported_compilers_module = 'spack.compilers'
_required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
_optional_instance_vars = ['modules']
_default_order = []
# TODO: customize order in config file
if platform.system() == 'Darwin':
_default_order = ['clang', 'gcc', 'intel']
@ -121,7 +123,7 @@ def add_compilers_to_config(compilers, arch=None, scope=None):
for compiler in compilers:
compiler_config[str(compiler.spec)] = dict(
(c, getattr(compiler, c, "None"))
for c in _required_instance_vars)
for c in _required_instance_vars + ['strategy'] + _optional_instance_vars)
update = { arch : compiler_config }
spack.config.update_config('compilers', update, scope)
@ -247,6 +249,11 @@ def get_compiler(cspec):
raise InvalidCompilerConfigurationError(cspec)
cls = class_for_compiler_name(cspec.name)
strategy = items['strategy']
if not strategy:
raise InvalidCompilerConfigurationError(cspec)
compiler_paths = []
for c in _required_instance_vars:
compiler_path = items[c]
@ -255,19 +262,28 @@ def get_compiler(cspec):
else:
compiler_paths.append(None)
return cls(cspec, *compiler_paths)
for m in _optional_instance_vars:
if m not in items:
items[m] = None
mods = items[m]
return cls(cspec, strategy, compiler_paths, mods)
matches = find(compiler_spec, arch, scope)
return [get_compiler(cspec) for cspec in matches]
@_auto_compiler_spec
def compiler_for_spec(compiler_spec):
def compiler_for_spec(compiler_spec, operating_system):
"""Get the compiler that satisfies compiler_spec. compiler_spec must
be concrete."""
assert(compiler_spec.concrete)
compilers = compilers_for_spec(compiler_spec)
assert(len(compilers) == 1)
compilers = [c for c in compilers_for_spec(compiler_spec)
if c.strategy == operating_system.compiler_strategy]
if len(compilers) < 1:
raise NoCompilerForSpecError(compiler_spec, operating_system)
if len(compilers) > 1:
raise CompilerSpecInsufficientlySpecificError(compiler_spec)
return compilers[0]
@ -286,6 +302,7 @@ def class_for_compiler_name(compiler_name):
def all_compiler_types():
# return [class_for_compiler_name(c) for c in ['gcc']]
return [class_for_compiler_name(c) for c in supported_compilers()]
@ -300,3 +317,13 @@ def __init__(self, compiler_spec):
class NoCompilersError(spack.error.SpackError):
def __init__(self):
super(NoCompilersError, self).__init__("Spack could not find any compilers!")
class NoCompilerForSpecError(spack.error.SpackError):
def __init__(self, compiler_spec, target):
super(NoCompilerForSpecError, self).__init__("No compilers for target %s satisfy spec %s" % (
target, compiler_spec))
class CompilerSpecInsufficientlySpecificError(spack.error.SpackError):
def __init__(self, compiler_spec):
super(CompilerSpecInsufficientlySpecificError, self).__init__("Multiple compilers satisfy spec %s",
compiler_spec)

2
lib/spack/spack/compilers/clang.py
View File

@ -48,7 +48,7 @@ class Clang(Compiler):
'fc' : 'f90' }
@classmethod
def default_version(self, comp):
def default_version(cls, comp):
"""The '--version' option works for clang compilers.
On most platforms, output looks like this::

57
lib/spack/spack/compilers/craype.py Normal file
View File

@ -0,0 +1,57 @@
##############################################################################}
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import llnl.util.tty as tty
#from spack.build_environment import load_module
from spack.compiler import *
#from spack.version import ver
class Craype(Compiler):
# Subclasses use possible names of C compiler
cc_names = ['cc']
# Subclasses use possible names of C++ compiler
cxx_names = ['CC']
# Subclasses use possible names of Fortran 77 compiler
f77_names = ['ftn']
# Subclasses use possible names of Fortran 90 compiler
fc_names = ['ftn']
# MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
suffixes = [r'-mp-\d\.\d']
PrgEnv = 'PrgEnv-cray'
PrgEnv_compiler = 'craype'
# @property
# def cxx11_flag(self):
# return "-hstd=c++11"
@classmethod
def default_version(cls, comp):
return get_compiler_version(comp, r'([Vv]ersion).*(\d+(\.\d+)+)')

7
lib/spack/spack/compilers/gcc.py
View File

@ -48,6 +48,9 @@ class Gcc(Compiler):
'f77' : 'gcc/gfortran',
'fc' : 'gcc/gfortran' }
PrgEnv = 'PrgEnv-gnu'
PrgEnv_compiler = 'gcc'
@property
def cxx11_flag(self):
if self.version < ver('4.3'):
@ -62,9 +65,9 @@ def fc_version(cls, fc):
return get_compiler_version(
fc, '-dumpversion',
# older gfortran versions don't have simple dumpversion output.
r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)')
r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)', module)
@classmethod
def f77_version(cls, f77):
return cls.fc_version(f77)
return cls.fc_version(f77, module)

3
lib/spack/spack/compilers/intel.py
View File

@ -43,6 +43,9 @@ class Intel(Compiler):
'f77' : 'intel/ifort',
'fc' : 'intel/ifort' }
PrgEnv = 'PrgEnv-intel'
PrgEnv_compiler = 'intel'
@property
def cxx11_flag(self):
if self.version < ver('11.1'):

3
lib/spack/spack/compilers/pgi.py
View File

@ -43,6 +43,9 @@ class Pgi(Compiler):
'f77' : 'pgi/pgfortran',
'fc' : 'pgi/pgfortran' }
PrgEnv = 'PrgEnv-pgi'
PrgEnv_compiler = 'pgi'
@classmethod
def default_version(cls, comp):
"""The '-V' option works for all the PGI compilers.

4
lib/spack/spack/compilers/xl.py
View File

@ -51,8 +51,9 @@ def cxx11_flag(self):
else:
return "-qlanglvl=extended0x"
@classmethod
def default_version(self, comp):
def default_version(cls, comp):
"""The '-qversion' is the standard option fo XL compilers.
Output looks like this::
@ -78,6 +79,7 @@ def default_version(self, comp):
return get_compiler_version(
comp, '-qversion',r'([0-9]?[0-9]\.[0-9])')
@classmethod
def fc_version(cls, fc):
"""The fortran and C/C++ versions of the XL compiler are always two units apart.

170
lib/spack/spack/concretize.py
View File

@ -33,11 +33,14 @@
TODO: make this customizable and allow users to configure
concretization policies.
"""
import collections
from llnl.util.filesystem import join_path
import spack
import spack.spec
import spack.compilers
import spack.architecture
import spack.error
from spack.util.naming import mod_to_class
from spack.version import *
from functools import partial
from spec import DependencyMap
@ -80,6 +83,11 @@ def _valid_virtuals_and_externals(self, spec):
for ext in externals:
if ext[0].satisfies(spec):
result.append(ext)
# if externals:
# sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0]))
# for external in sorted_externals:
# if external[0].satisfies(spec):
# result.append(external)
if not result:
raise NoBuildError(spec)
@ -120,8 +128,10 @@ def concretize_virtual_and_external(self, spec):
if not candidate:
# No ABI matches. Pick the top choice based on the orignal preferences.
candidate = candidates[0]
candidate_spec = candidate[0]
external_module = candidate[2]
external = candidate[1]
candidate_spec = candidate[0]
changed = False
# If we're external then trim the dependencies
@ -148,7 +158,13 @@ def fequal(candidate_field, spec_field):
changed = True
if spec._dup(candidate_spec, deps=False, cleardeps=False):
changed = True
spec.external = external
if not spec.external and external:
spec.external = external
changed = True
if not spec.external_module and external_module:
spec.external_module = external_module
changed = True
return changed
@ -173,7 +189,7 @@ def concretize_version(self, spec):
# If there are known available versions, return the most recent
# version that satisfies the spec
pkg = spec.package
pkg = spec.package # Gives error here with dynist
cmp_versions = partial(spack.pkgsort.version_compare, spec.name)
valid_versions = sorted(
[v for v in pkg.versions
@ -203,31 +219,114 @@ def concretize_version(self, spec):
return True # Things changed
def _concretize_operating_system(self, spec):
if spec.architecture.platform_os is not None:
if isinstance(spec.architecture.platform_os,spack.architecture.OperatingSystem):
return False
else:
spec.add_operating_system_from_string(spec.architecture.platform_os)
return True #changed
if spec.root.architecture and spec.root.architecture.platform_os:
if isinstance(spec.root.architecture.platform_os,spack.architecture.OperatingSystem):
spec.architecture.platform_os = spec.root.architecture.platform_os
else:
spec.add_operating_system_from_string(spec.root.architecture.platform_os)
else:
spec.architecture.platform_os = spec.architecture.platform.operating_system('default_os')
return True #changed
# """ Future method for concretizing operating system """
# if isinstance(arch.platform_os, spack.architecture.OperatingSystem):
# return False
# else:
# arch.arch_os = platform.operating_system('default_os')
# return True
def _concretize_target(self, spec):
if spec.architecture.target is not None:
if isinstance(spec.architecture.target,spack.architecture.Target):
return False
else:
spec.add_target_from_string(spec.architecture.target)
return True #changed
if spec.root.architecture and spec.root.architecture.target:
if isinstance(spec.root.architecture.target,spack.architecture.Target):
spec.architecture.target = spec.root.architecture.target
else:
spec.add_target_from_string(spec.root.architecture.target)
else:
spec.architecture.target = spec.architecture.platform.target('default')
return True #changed
# if isinstance(arch.target, spack.architecture.Target):
# return False
# else:
# arch.target = platform.target('default')
# return True
def concretize_architecture(self, spec):
"""If the spec already had an architecture, return. Otherwise if
the root of the DAG has an architecture, then use that.
Otherwise take the system's default architecture.
Intuition: Architectures won't be set a lot, and generally you
want the host system's architecture. When architectures are
mised in a spec, it is likely because the tool requries a
cross-compiled component, e.g. for tools that run on BlueGene
or Cray machines. These constraints will likely come directly
from packages, so require the user to be explicit if they want
to mess with the architecture, and revert to the default when
they're not explicit.
"""If the spec is empty provide the defaults of the platform. If the
architecture is not a basestring, then check if either the platform,
target or operating system are concretized. If any of the fields are
changed then return True. If everything is concretized (i.e the
architecture attribute is a namedtuple of classes) then return False.
If the target is a string type, then convert the string into a
concretized architecture. If it has no architecture and the root of the
DAG has an architecture, then use the root otherwise use the defaults
on the platform.
"""
if spec.architecture is not None:
return False
if spec.architecture is None:
# Set the architecture to all defaults
spec.architecture = spack.architecture.Arch()
return True
#If there is a target and it is a tuple and has both filled return
#False
# if isinstance(spec.architecture, basestring):
# spec.split_architecture_string(spec.architecture)
if spec.root.architecture:
spec.architecture = spec.root.architecture
else:
spec.architecture = spack.architecture.sys_type()
ret = any((
self._concretize_operating_system(spec),
self._concretize_target(spec)))
assert(spec.architecture is not None)
return True # changed
# Does not look pretty at all!!!
# if spec.root.architecture and \
# not isinstance(spec.root.architecture, basestring):
# bool_flag = any((
# self._concretize_platform(spec.root.architecture, platform),
# self._concretize_operating_system(spec.root.architecture,
# platform),
# self._concretize_target(spec.root.target, platform)))
# spec.architecture =spec.root.architecture
# return bool_flag
# else:
# spec.add_architecture_from_string(spec.root.architecture)
return ret
# if there is no target specified used the defaults
#if spec.target is not None:
# if isinstance(spec.target,spack.architecture.Target):
# return False
# else:
# spec.add_target_from_string(spec.target)
# return True #changed
#if spec.root.target:
# if isinstance(spec.root.target,spack.architecture.Target):
# spec.target = spec.root.target
# else:
# spec.add_target_from_string(spec.root.target)
#else:
# platform = spack.architecture.sys_type()
# spec.target = platform.target('default')
#return True #changed
def concretize_variants(self, spec):
@ -254,6 +353,24 @@ def concretize_compiler(self, spec):
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
"""
# Pass on concretizing the compiler if the target is not yet determined
if not spec.architecture.target:
#Although this usually means changed, this means awaiting other changes
return True
# Only use a matching compiler if it is of the proper style
# Takes advantage of the proper logic already existing in compiler_for_spec
# Should think whether this can be more efficient
def _proper_compiler_style(cspec, architecture):
compilers = spack.compilers.compilers_for_spec(cspec)
filter(lambda c: c.strategy == architecture.platform_os.compiler_strategy, compilers)
#if architecture.platform_os.compiler_strategy == 'PATH':
# filter(lambda c: not c.modules, compilers)
#if architecture.platform_os.compiler_strategy == 'MODULES':
# filter(lambda c: c.modules, compilers)
return compilers
all_compilers = spack.compilers.all_compilers()
if (spec.compiler and
@ -281,7 +398,12 @@ def concretize_compiler(self, spec):
raise UnavailableCompilerVersionError(other_compiler)
# copy concrete version into other_compiler
spec.compiler = matches[0].copy()
index = len(matches)-1
while not _proper_compiler_style(matches[index], spec.architecture):
index -= 1
if index == 0:
raise NoValidVersionError(spec)
spec.compiler = matches[index].copy()
assert(spec.compiler.concrete)
return True # things changed.

10
lib/spack/spack/config.py
View File

@ -136,7 +136,7 @@
# Hacked yaml for configuration files preserves line numbers.
import spack.util.spack_yaml as syaml
from spack.build_environment import get_path_from_module
"""Dict from section names -> schema for that section."""
section_schemas = {
@ -551,9 +551,13 @@ def spec_externals(spec):
for pkg,path in pkg_paths.iteritems():
if not spec.satisfies(pkg):
continue
module = allpkgs.get(pkg, {}).get('module', None)
if not path:
continue
spec_locations.append( (spack.spec.Spec(pkg), path) )
if not module:
continue
path = get_path_from_module(module)
spec_locations.append( (spack.spec.Spec(pkg), path, module) )
return spec_locations

2
lib/spack/spack/directives.py
View File

@ -259,7 +259,7 @@ def variant(pkg, name, default=False, description=""):
"""Define a variant for the package. Packager can specify a default
value (on or off) as well as a text description."""
default = bool(default)
default = default
description = str(description).strip()
if not re.match(spack.spec.identifier_re, name):

5
lib/spack/spack/directory_layout.py
View File

@ -165,7 +165,7 @@ def remove_install_directory(self, spec):
class YamlDirectoryLayout(DirectoryLayout):
"""Lays out installation directories like this::
<install root>/
<architecture>/
<target>/
<compiler>-<compiler version>/
<name>-<version>-<variants>-<hash>
@ -207,8 +207,7 @@ def relative_path_for_spec(self, spec):
spec.version,
spec.dag_hash(self.hash_len))
path = join_path(
spec.architecture,
path = join_path(spec.architecture,
"%s-%s" % (spec.compiler.name, spec.compiler.version),
dir_name)

0
lib/spack/spack/operating_systems/__init__.py Normal file
View File

13
lib/spack/spack/operating_systems/cnl.py Normal file
View File

@ -0,0 +1,13 @@
from spack.architecture import OperatingSystem
class Cnl(OperatingSystem):
""" Compute Node Linux (CNL) is the operating system used for the Cray XC
series super computers. It is a very stripped down version of GNU/Linux.
Any compilers found through this operating system will be used with
modules. If updated, user must make sure that version and name are
updated to indicate that OS has been upgraded (or downgraded)
"""
def __init__(self):
name = 'CNL'
version = '10'
super(Cnl, self).__init__(name, version, "MODULES")

15
lib/spack/spack/operating_systems/linux_distro.py Normal file
View File

@ -0,0 +1,15 @@
import platform as py_platform
from spack.architecture import OperatingSystem
class LinuxDistro(OperatingSystem):
""" This class will represent the autodetected operating system
for a Linux System. Since there are many different flavors of
Linux, this class will attempt to encompass them all through
autodetection using the python module platform and the method
platform.dist()
"""
def __init__(self):
name = py_platform.dist()[0]
version = py_platform.dist()[1]
super(LinuxDistro, self).__init__(name, version)

29
lib/spack/spack/operating_systems/mac_osx.py Normal file
View File

@ -0,0 +1,29 @@
import platform as py_platform
from spack.architecture import OperatingSystem
class MacOsx(OperatingSystem):
""" This class represents the MAC_OSX operating system. This will be auto
detected using the python platform.mac_ver. The MAC_OSX platform
will be represented using the major version operating system name, i.e
el capitan, yosemite...etc.
"""
def __init__(self):
""" Autodetects the mac version from a dictionary. Goes back as
far as 10.6 snowleopard. If the user has an older mac then
the version will just be a generic mac_os.
"""
mac_releases = {'10.6': "snowleopard",
"10.7": "lion",
"10.8": "mountainlion",
"10.9": "mavericks",
"10.10": "yosemite",
"10.11": "elcapitan"}
mac_ver = py_platform.mac_ver()[0][:-2]
try:
name = mac_releases[mac_ver]
except KeyError:
name = "mac_os"
super(MacOsx, self).__init__(name, mac_ver)

4
lib/spack/spack/package.py
View File

@ -649,11 +649,13 @@ def prefix(self):
@property
#TODO: Change this to architecture
def compiler(self):
"""Get the spack.compiler.Compiler object used to build this package."""
if not self.spec.concrete:
raise ValueError("Can only get a compiler for a concrete package.")
return spack.compilers.compiler_for_spec(self.spec.compiler)
return spack.compilers.compiler_for_spec(self.spec.compiler,
self.spec.architecture.platform_os)
def url_version(self, version):

0
lib/spack/spack/platforms/__init__.py Normal file
View File

18
lib/spack/spack/platforms/bgq.py Normal file
View File

@ -0,0 +1,18 @@
import os
from spack.architecture import Platform, Target
class Bgq(Platform):
priority = 30
front_end = 'power7'
back_end = 'powerpc'
default = 'powerpc'
def __init__(self):
super(Bgq, self).__init__('bgq')
self.add_target(self.front_end, Target(self.front_end))
self.add_target(self.back_end, Target(self.back_end,))
@classmethod
def detect(self):
return os.path.exists('/bgsys')

46
lib/spack/spack/platforms/cray_xc.py Normal file
View File

@ -0,0 +1,46 @@
import os
from spack.architecture import Platform, Target
from spack.operating_systems.linux_distro import LinuxDistro
from spack.operating_systems.cnl import Cnl
class CrayXc(Platform):
priority = 20
front_end = 'sandybridge'
back_end = 'ivybridge'
default = 'ivybridge'
front_os = "SuSE"
back_os = "CNL"
default_os = "CNL"
def __init__(self):
''' Since cori doesn't have ivybridge as a front end it's better
if we use CRAY_CPU_TARGET as the default. This will ensure
that if we're on a XC-40 or XC-30 then we can detect the target
'''
super(CrayXc, self).__init__('crayxc')
# Handle the default here so we can check for a key error
if 'CRAY_CPU_TARGET' in os.environ:
self.default = os.environ['CRAY_CPU_TARGET']
# Change the defaults to haswell if we're on an XC40
if self.default == 'haswell':
self.front_end = self.default
self.back_end = self.default
# Could switch to use modules and fe targets for front end
# Currently using compilers by path for front end.
self.add_target('sandybridge', Target('sandybridge'))
self.add_target('ivybridge',
Target('ivybridge', 'craype-ivybridge'))
self.add_target('haswell',
Target('haswell','craype-haswell'))
self.add_operating_system('SuSE', LinuxDistro())
self.add_operating_system('CNL', Cnl())
@classmethod
def detect(self):
return os.path.exists('/opt/cray/craype')

22
lib/spack/spack/platforms/darwin.py Normal file
View File

@ -0,0 +1,22 @@
import subprocess
from spack.architecture import Platform, Target
from spack.operating_systems.mac_osx import MacOsx
class Darwin(Platform):
priority = 89
front_end = 'x86_64'
back_end = 'x86_64'
default = 'x86_64'
def __init__(self):
super(Darwin, self).__init__('darwin')
self.add_target(self.default, Target(self.default))
mac_os = MacOsx()
self.default_os = mac_os.name
self.add_operating_system(mac_os.name, mac_os)
@classmethod
def detect(self):
platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE)
platform, _ = platform.communicate()
return 'darwin' in platform.strip().lower()

22
lib/spack/spack/platforms/linux.py Normal file
View File

@ -0,0 +1,22 @@
import subprocess
from spack.architecture import Platform, Target
from spack.operating_systems.linux_distro import LinuxDistro
class Linux(Platform):
priority = 90
front_end = 'x86_64'
back_end = 'x86_64'
default = 'x86_64'
def __init__(self):
super(Linux, self).__init__('linux')
self.add_target(self.default, Target(self.default))
linux_dist = LinuxDistro()
self.default_os = linux_dist.name
self.add_operating_system(linux_dist.name, linux_dist)
@classmethod
def detect(self):
platform = subprocess.Popen(['uname', '-a'], stdout = subprocess.PIPE)
platform, _ = platform.communicate()
return 'linux' in platform.strip().lower()

150
lib/spack/spack/spec.py
View File

@ -90,7 +90,9 @@
specs to avoid ambiguity. Both are provided because ~ can cause shell
expansion when it is the first character in an id typed on the command line.
"""
from collections import namedtuple
import sys
import imp
import itertools
import hashlib
import base64
@ -100,15 +102,18 @@
from yaml.error import MarkedYAMLError
import llnl.util.tty as tty
from llnl.util.filesystem import join_path
from llnl.util.lang import *
from llnl.util.tty.color import *
import spack
import spack.architecture
import spack.parse
import spack.error
import spack.compilers as compilers
from spack.version import *
from spack.util.naming import mod_to_class
from spack.util.string import *
from spack.util.prefix import Prefix
from spack.virtual import ProviderIndex
@ -119,7 +124,7 @@
# Convenient names for color formats so that other things can use them
compiler_color = '@g'
version_color = '@c'
architecture_color = '@m'
architecture_color = '@m'
enabled_variant_color = '@B'
disabled_variant_color = '@r'
dependency_color = '@.'
@ -421,6 +426,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
self._normal = kwargs.get('normal', False)
self._concrete = kwargs.get('concrete', False)
self.external = None
self.external_module = None
# This allows users to construct a spec DAG with literals.
# Note that given two specs a and b, Spec(a) copies a, but
@ -456,7 +462,13 @@ def _set_architecture(self, architecture):
"""Called by the parser to set the architecture."""
if self.architecture: raise DuplicateArchitectureError(
"Spec for '%s' cannot have two architectures." % self.name)
self.architecture = architecture
platform = spack.architecture.sys_type()
if '-' in architecture:
os, target = architecture.split('-')
else:
os = None
target = architecture
self.architecture = spack.architecture.Arch(os, target)
def _add_dependency(self, spec):
@ -571,7 +583,7 @@ def traverse(self, visited=None, d=0, **kwargs):
in the traversal.
root [=True]
If false, this won't yield the root node, just its descendents.
If False, this won't yield the root node, just its descendents.
direction [=children|parents]
If 'children', does a traversal of this spec's children. If
@ -664,7 +676,6 @@ def to_node_dict(self):
d = {
'variants' : dict(
(name,v.enabled) for name, v in self.variants.items()),
'arch' : self.architecture,
'dependencies' : dict((d, self.dependencies[d].dag_hash())
for d in sorted(self.dependencies))
}
@ -674,6 +685,13 @@ def to_node_dict(self):
if not self.concrete or self.namespace:
d['namespace'] = self.namespace
if self.architecture:
# TODO: Fix the target.to_dict to account for the tuple
# Want it to be a dict of dicts
d['architecture'] = self.architecture.to_dict()
else:
d['architecture'] = None
if self.compiler:
d.update(self.compiler.to_dict())
else:
@ -700,7 +718,8 @@ def from_node_dict(node):
spec = Spec(name)
spec.namespace = node.get('namespace', None)
spec.versions = VersionList.from_dict(node)
spec.architecture = node['arch']
# TODO: Need to fix the architecture.Target.from_dict
spec.architecture = spack.architecture.arch_from_dict(node['architecture'])
if node['compiler'] is None:
spec.compiler = None
@ -766,7 +785,6 @@ def _concretize_helper(self, presets=None, visited=None):
if self.name in presets:
changed |= self.constrain(presets[self.name])
else:
# Concretize virtual dependencies last. Because they're added
# to presets below, their constraints will all be merged, but we'll
@ -786,8 +804,9 @@ def _replace_with(self, concrete):
"""Replace this virtual spec with a concrete spec."""
assert(self.virtual)
for name, dependent in self.dependents.items():
del dependent.dependencies[self.name]
dependent._add_dependency(concrete)
if not dependent.external:
del dependent.dependencies[self.name]
dependent._add_dependency(concrete)
def _expand_virtual_packages(self):
@ -1188,9 +1207,10 @@ def constrain(self, other, deps=True):
raise UnsatisfiableVariantSpecError(self.variants[v],
other.variants[v])
# TODO: Check out the logic here
if self.architecture is not None and other.architecture is not None:
if self.architecture != other.architecture:
raise UnsatisfiableArchitectureSpecError(self.architecture,
raise UnsatisfiableTargetSpecError(self.architecture,
other.architecture)
changed = False
@ -1277,11 +1297,34 @@ def _autospec(self, spec_like):
except SpecError:
return parse_anonymous_spec(spec_like, self.name)
def _is_valid_platform(self, platform, platform_list):
if platform in platform_list:
return True
return False
def _is_valid_target(self, target, platform):
return target in platform.targets
def _is_valid_os(self, os_string, platform):
return os_string in platform.operating_sys
def add_target_from_string(self, target):
if target is None:
self.architecture.target = self.architecture.platform.target('default_target')
else:
self.architecture.target = self.architecture.platform.target(target)
def add_operating_system_from_string(self, os):
if os is None:
self.architecture.platform_os = self.architecture.platform.operating_system('default_os')
else:
self.architecture.platform_os = self.architecture.platform.operating_system(os)
def satisfies(self, other, deps=True, strict=False):
"""Determine if this spec satisfies all constraints of another.
"""determine if this spec satisfies all constraints of another.
There are two senses for satisfies:
there are two senses for satisfies:
* `loose` (default): the absence of a constraint in self
implies that it *could* be satisfied by other, so we only
@ -1293,7 +1336,7 @@ def satisfies(self, other, deps=True, strict=False):
"""
other = self._autospec(other)
# A concrete provider can satisfy a virtual dependency.
# a concrete provider can satisfy a virtual dependency.
if not self.virtual and other.virtual:
pkg = spack.repo.get(self.fullname)
if pkg.provides(other.name):
@ -1303,7 +1346,7 @@ def satisfies(self, other, deps=True, strict=False):
return True
return False
# Otherwise, first thing we care about is whether the name matches
# otherwise, first thing we care about is whether the name matches
if self.name != other.name:
return False
@ -1318,18 +1361,25 @@ def satisfies(self, other, deps=True, strict=False):
elif strict and (self.versions or other.versions):
return False
# None indicates no constraints when not strict.
# none indicates no constraints when not strict.
if self.compiler and other.compiler:
if not self.compiler.satisfies(other.compiler, strict=strict):
return False
return False
elif strict and (other.compiler and not self.compiler):
return False
if not self.variants.satisfies(other.variants, strict=strict):
return False
# Architecture satisfaction is currently just string equality.
# Target satisfaction is currently just class equality.
# If not strict, None means unconstrained.
if isinstance(self.architecture, basestring):
self.add_architecture_from_string(self.architecture)
if isinstance(other.architecture, basestring):
other.add_architecture_from_string(other.architecture)
# TODO: Need to make sure that comparisons can be made via classes
if self.architecture and other.architecture:
if self.architecture != other.architecture:
return False
@ -1399,17 +1449,19 @@ def _dup(self, other, **kwargs):
Options:
dependencies[=True]
Whether deps should be copied too. Set to false to copy a
Whether deps should be copied too. Set to False to copy a
spec but not its dependencies.
"""
# TODO: Check if comparisons for tuple are valid
# We don't count dependencies as changes here
changed = True
if hasattr(self, 'name'):
changed = (self.name != other.name and self.versions != other.versions and \
self.architecture != other.architecture and self.compiler != other.compiler and \
self.variants != other.variants and self._normal != other._normal and \
self.concrete != other.concrete and self.external != other.external)
self.concrete != other.concrete and self.external != other.external and \
self.external_module != other.external_module)
# Local node attributes get copied first.
self.name = other.name
@ -1423,6 +1475,7 @@ def _dup(self, other, **kwargs):
self.variants.spec = self
self.external = other.external
self.namespace = other.namespace
self.external_module = other.external_module
# If we copy dependencies, preserve DAG structure in the new spec
if kwargs.get('deps', True):
@ -1441,6 +1494,7 @@ def _dup(self, other, **kwargs):
self._normal = other._normal
self._concrete = other._concrete
self.external = other.external
self.external_module = other.external_module
return changed
@ -1598,7 +1652,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
${COMPILERNAME} Compiler name
${COMPILERVER} Compiler version
${OPTIONS} Options
${ARCHITECTURE} Architecture
${TARGET} Target
${SHA1} Dependencies 8-char sha1 prefix
${SPACK_ROOT} The spack root directory
@ -1611,7 +1665,7 @@ def format(self, format_string='$_$@$%@$+$=', **kwargs):
Anything else is copied verbatim into the output stream.
*Example:* ``$_$@$+`` translates to the name, version, and options
of the package, but no dependencies, arch, or compiler.
of the package, but no dependencies, architecture, or compiler.
TODO: allow, e.g., $6# to customize short hash length
TODO: allow, e.g., $## for full hash.
@ -1656,6 +1710,7 @@ def write(s, c):
elif c == '+':
if self.variants:
write(fmt % str(self.variants), c)
# TODO: Check string methods here
elif c == '=':
if self.architecture:
write(fmt % (c + str(self.architecture)), c)
@ -1707,7 +1762,7 @@ def write(s, c):
write(fmt % str(self.variants), '+')
elif named_str == 'ARCHITECTURE':
if self.architecture:
write(fmt % str(self.architecture), '=')
write(fmt % self.architecture, '=')
elif named_str == 'SHA1':
if self.dependencies:
out.write(fmt % str(self.dag_hash(7)))
@ -1734,6 +1789,40 @@ def dep_string(self):
return ''.join("^" + dep.format() for dep in self.sorted_deps())
def __cmp__(self, other):
#Package name sort order is not configurable, always goes alphabetical
if self.name != other.name:
return cmp(self.name, other.name)
#Package version is second in compare order
pkgname = self.name
if self.versions != other.versions:
return spack.pkgsort.version_compare(pkgname,
self.versions, other.versions)
#Compiler is third
if self.compiler != other.compiler:
return spack.pkgsort.compiler_compare(pkgname,
self.compiler, other.compiler)
#Variants
if self.variants != other.variants:
return spack.pkgsort.variant_compare(pkgname,
self.variants, other.variants)
#Target
if self.target != other.target:
return spack.pkgsort.target_compare(pkgname,
self.target, other.target)
#Dependency is not configurable
if self.dep_hash() != other.dep_hash():
return -1 if self.dep_hash() < other.dep_hash() else 1
#Equal specs
return 0
def __str__(self):
return self.format() + self.dep_string()
@ -1806,7 +1895,6 @@ def __init__(self):
def do_parse(self):
specs = []
try:
while self.next:
if self.accept(ID):
@ -1849,6 +1937,7 @@ def spec(self):
spec.architecture = None
spec.compiler = None
spec.external = None
spec.external_module = None
spec.dependents = DependencyMap()
spec.dependencies = DependencyMap()
spec.namespace = spec_namespace
@ -2036,9 +2125,16 @@ def __init__(self, pkg, variant):
super(UnknownVariantError, self).__init__(
"Package %s has no variant %s!" % (pkg, variant))
class UnknownArchitectureSpecError(SpecError):
""" Raised when an entry in a string field is neither a platform,
operating system or a target. """
def __init__(self, architecture_spec_entry):
super(UnknownArchitectureSpecError, self).__init__(
"Architecture spec %s is not a valid spec entry" % (
architecture_spec_entry))
class DuplicateArchitectureError(SpecError):
"""Raised when the same architecture occurs in a spec twice."""
"""Raised when the same target occurs in a spec twice."""
def __init__(self, message):
super(DuplicateArchitectureError, self).__init__(message)
@ -2119,11 +2215,11 @@ def __init__(self, provided, required):
provided, required, "variant")
class UnsatisfiableArchitectureSpecError(UnsatisfiableSpecError):
"""Raised when a spec architecture conflicts with package constraints."""
class UnsatisfiableTargetSpecError(UnsatisfiableSpecError):
"""Raised when a spec target conflicts with package constraints."""
def __init__(self, provided, required):
super(UnsatisfiableArchitectureSpecError, self).__init__(
provided, required, "architecture")
super(UnsatisfiableTargetSpecError, self).__init__(
provided, required, "target")
class UnsatisfiableProviderSpecError(UnsatisfiableSpecError):

3
lib/spack/spack/test/__init__.py
View File

@ -34,7 +34,8 @@
import spack
"""Names of tests to be included in Spack's test suite"""
test_names = ['versions',
test_names = ['architecture',
'versions',
'url_parse',
'url_substitution',
'packages',

63
lib/spack/spack/test/architecture.py Normal file
View File

@ -0,0 +1,63 @@
""" Test checks if the architecture class is created correctly and also that
the functions are looking for the correct architecture name
"""
import unittest
import os
import platform
import spack
from spack.architecture import *
import spack.spec
from spack.platforms.cray_xc import CrayXc
from spack.platforms.linux import Linux
from spack.platforms.bgq import Bgq
from spack.platforms.darwin import Darwin
class ArchitectureTest(unittest.TestCase):
def test_to_dict_function_with_architecture(self):
arch = Arch()
arch.platform_os = arch.platform.operating_system('default_os')
arch.target = arch.platform.target('default')
d = arch.to_dict()
self.assertEqual(d, {'platform' : 'crayxc',
'platform_os' : {'name': 'CNL',
'compiler_strategy' : 'MODULES',
'version':'10'},
'target' : {'name': 'haswell',
'module_name': 'craype-haswell'}})
def test_from_dict_function_with_architecture(self):
d = {'platform':'crayxc',
'platform_os' : {'name' : 'CNL', 'compiler_strategy': 'MODULES',
'version': '10'},
'target' : {'name':'haswell', 'module_name': 'craype-haswell'}}
arch = spack.architecture.arch_from_dict(d)
self.assertTrue( isinstance(arch, Arch) )
self.assertTrue( isinstance(arch.platform, Platform) )
self.assertTrue( isinstance(arch.platform_os, OperatingSystem) )
self.assertTrue( isinstance(arch.target, Target) )
def test_platform_class_and_compiler_strategies(self):
a = CrayXc()
t = a.operating_system('default_os')
self.assertEquals(t.compiler_strategy, 'MODULES')
b = Linux()
s = b.operating_system('default_os')
self.assertEquals(s.compiler_strategy, 'PATH')
def test_sys_type(self):
output_platform_class = sys_type()
my_arch_class = None
if os.path.exists('/opt/cray/craype'):
my_platform_class = CrayXc()
elif os.path.exists('/bgsys'):
my_platform_class = Bgq()
elif 'Linux' in platform.system():
my_platform_class = Linux()
elif 'Darwin' in platform.system():
my_platform_class = Darwin()
self.assertEqual(str(output_platform_class), str(my_platform_class))

4
lib/spack/spack/test/concretize.py
View File

@ -45,8 +45,8 @@ def check_spec(self, abstract, concrete):
if abstract.compiler and abstract.compiler.concrete:
self.assertEqual(abstract.compiler, concrete.compiler)
if abstract.architecture and abstract.architecture.concrete:
self.assertEqual(abstract.architecture, concrete.architecture)
if abstract.architecture and abstract.architecture.target.concrete:
self.assertEqual(abstract.target, concrete.target)
def check_concretize(self, abstract_spec):

25
lib/spack/spack/test/multimethod.py
View File

@ -88,21 +88,18 @@ def test_default_works(self):
self.assertEqual(pkg.has_a_default(), 'default')
def test_architecture_match(self):
pkg = spack.repo.get('multimethod=x86_64')
self.assertEqual(pkg.different_by_architecture(), 'x86_64')
def test_target_match(self):
platform = spack.architecture.sys_type()
targets = platform.targets.values()
for target in targets[:-1]:
pkg = spack.db.get('multimethod='+target.name)
self.assertEqual(pkg.different_by_target(), target.name)
pkg = spack.repo.get('multimethod=ppc64')
self.assertEqual(pkg.different_by_architecture(), 'ppc64')
pkg = spack.repo.get('multimethod=ppc32')
self.assertEqual(pkg.different_by_architecture(), 'ppc32')
pkg = spack.repo.get('multimethod=arm64')
self.assertEqual(pkg.different_by_architecture(), 'arm64')
pkg = spack.repo.get('multimethod=macos')
self.assertRaises(NoSuchMethodError, pkg.different_by_architecture)
pkg = spack.db.get('multimethod='+targets[-1].name)
if len(targets) == 1:
self.assertEqual(pkg.different_by_target(), targets[-1].name)
else:
self.assertRaises(NoSuchMethodError, pkg.different_by_target)
def test_dependency_match(self):

58
lib/spack/spack/test/operating_system.py Normal file
View File

@ -0,0 +1,58 @@
""" Test checks if the operating_system class is created correctly and that
the functions are using the correct operating_system. Also checks whether
the operating_system correctly uses the compiler_strategy
"""
import unittest
import os
import platform
from spack.platforms.cray_xc import CrayXc
from spack.platforms.linux import Linux
from spack.platforms.darwin import Darwin
from spack.operating_system.linux_distro import LinuxDistro
from spack.operating_system.mac_osx import MacOSX
from spack.operating_system.cnl import ComputeNodeLinux
class TestOperatingSystem(unittest.TestCase):
def setUp(self):
cray_xc = CrayXc()
linux = Linux()
darwin = Darwin()
self.cray_operating_sys = cray_xc.operating_system('front_os')
self.cray_default_os = cray_xc.operating_system('default_os')
self.cray_back_os = cray_xc.operating_system('back_os')
self.darwin_operating_sys = darwin.operating_system('default_os')
self.linux_operating_sys = linux.operating_system('default_os')
def test_cray_front_end_operating_system(self):
self.assertIsInstance(self.cray_operating_sys, LinuxDistro)
def test_cray_front_end_compiler_strategy(self):
self.assertEquals(self.cray_operating_sys.compiler_strategy, "PATH")
def test_cray_back_end_operating_system(self):
self.assertIsInstance(self.cray_back_os,ComputeNodeLinux)
def test_cray_back_end_compiler_strategy(self):
self.assertEquals(self.cray_back_os.compiler_strategy, "MODULES")
def test_linux_operating_system(self):
self.assertIsInstance(self.linux_operating_sys, LinuxDistro)
def test_linux_compiler_strategy(self):
self.assertEquals(self.linux_operating_sys.compiler_strategy, "PATH")
def test_cray_front_end_compiler_list(self):
""" Operating systems will now be in charge of finding compilers.
So, depending on which operating system you want to build for
or which operating system you are on, then you could detect
compilers in a certain way. Cray linux environment on the front
end is just a regular linux distro whereas the Cray linux compute
node is a stripped down version which modules are important
"""
self.assertEquals(True, False)

12
lib/spack/spack/test/spec_dag.py
View File

@ -238,10 +238,14 @@ def test_unsatisfiable_compiler_version(self):
self.assertRaises(spack.spec.UnsatisfiableCompilerSpecError, spec.normalize)
def test_unsatisfiable_architecture(self):
self.set_pkg_dep('mpileaks', 'mpich=bgqos_0')
spec = Spec('mpileaks ^mpich=sles_10_ppc64 ^callpath ^dyninst ^libelf ^libdwarf')
self.assertRaises(spack.spec.UnsatisfiableArchitectureSpecError, spec.normalize)
def test_unsatisfiable_target(self):
platform = spack.architecture.sys_type()
if len(platform.targets) > 1:
first = platform.targets.values()[0].name
second = platform.targets.values()[1].name
set_pkg_dep('mpileaks', 'mpich='+first)
spec = Spec('mpileaks ^mpich='+ second +' ^callpath ^dyninst ^libelf ^libdwarf')
self.assertRaises(spack.spec.UnsatisfiableTargetSpecError, spec.normalize)
def test_invalid_dep(self):

46
lib/spack/spack/test/spec_semantics.py
View File

@ -137,13 +137,14 @@ def test_satisfies_compiler_version(self):
self.check_unsatisfiable('foo %gcc@4.7', '%gcc@4.7.3')
def test_satisfies_architecture(self):
self.check_satisfies('foo=chaos_5_x86_64_ib', '=chaos_5_x86_64_ib')
self.check_satisfies('foo=bgqos_0', '=bgqos_0')
self.check_unsatisfiable('foo=bgqos_0', '=chaos_5_x86_64_ib')
self.check_unsatisfiable('foo=chaos_5_x86_64_ib', '=bgqos_0')
def test_satisfies_target(self):
platform = spack.architecture.sys_type()
targets = platform.targets.values()
for target in targets:
self.check_satisfies('foo='+target.name, '='+target.name)
for i in range(1,len(targets)):
self.check_unsatisfiable('foo='+targets[i-1].name, '='+targets[i].name)
def test_satisfies_dependencies(self):
self.check_satisfies('mpileaks^mpich', '^mpich')
@ -305,14 +306,16 @@ def test_constrain_variants(self):
self.check_constrain('libelf+debug~foo', 'libelf+debug', 'libelf+debug~foo')
def test_constrain_arch(self):
self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0')
self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0')
def test_constrain_target(self):
platform = spack.architecture.sys_type()
target = platform.target('default').name
self.check_constrain('libelf='+target, 'libelf='+target, 'libelf='+target)
self.check_constrain('libelf='+target, 'libelf', 'libelf='+target)
def test_constrain_compiler(self):
self.check_constrain('libelf=bgqos_0', 'libelf=bgqos_0', 'libelf=bgqos_0')
self.check_constrain('libelf=bgqos_0', 'libelf', 'libelf=bgqos_0')
self.check_constrain('libelf%intel', 'libelf%intel', 'libelf%intel')
self.check_constrain('libelf%intel', 'libelf', 'libelf%intel')
def test_invalid_constraint(self):
@ -322,7 +325,10 @@ def test_invalid_constraint(self):
self.check_invalid_constraint('libelf+debug', 'libelf~debug')
self.check_invalid_constraint('libelf+debug~foo', 'libelf+debug+foo')
self.check_invalid_constraint('libelf=bgqos_0', 'libelf=x86_54')
platform = spack.architecture.sys_type()
targets = platform.targets.values()
if len(targets) > 1:
self.check_invalid_constraint('libelf='+targets[0].name, 'libelf='+targets[1].name)
def test_constrain_changed(self):
@ -332,7 +338,8 @@ def test_constrain_changed(self):
self.check_constrain_changed('libelf%gcc', '%gcc@4.5')
self.check_constrain_changed('libelf', '+debug')
self.check_constrain_changed('libelf', '~debug')
self.check_constrain_changed('libelf', '=bgqos_0')
platform = spack.architecture.sys_type()
self.check_constrain_changed('libelf', '='+platform.target('default').name)
def test_constrain_not_changed(self):
@ -343,7 +350,9 @@ def test_constrain_not_changed(self):
self.check_constrain_not_changed('libelf%gcc@4.5', '%gcc@4.5')
self.check_constrain_not_changed('libelf+debug', '+debug')
self.check_constrain_not_changed('libelf~debug', '~debug')
self.check_constrain_not_changed('libelf=bgqos_0', '=bgqos_0')
platform = spack.architecture.sys_type()
default = platform.target('default').name
self.check_constrain_not_changed('libelf='+default, '='+default)
self.check_constrain_not_changed('libelf^foo', 'libelf^foo')
self.check_constrain_not_changed('libelf^foo^bar', 'libelf^foo^bar')
@ -355,7 +364,9 @@ def test_constrain_dependency_changed(self):
self.check_constrain_changed('libelf^foo%gcc', 'libelf^foo%gcc@4.5')
self.check_constrain_changed('libelf^foo', 'libelf^foo+debug')
self.check_constrain_changed('libelf^foo', 'libelf^foo~debug')
self.check_constrain_changed('libelf^foo', 'libelf^foo=bgqos_0')
platform = spack.architecture.sys_type()
default = platform.target('default').name
self.check_constrain_changed('libelf^foo', 'libelf^foo='+default)
def test_constrain_dependency_not_changed(self):
@ -365,4 +376,7 @@ def test_constrain_dependency_not_changed(self):
self.check_constrain_not_changed('libelf^foo%gcc@4.5', 'libelf^foo%gcc@4.5')
self.check_constrain_not_changed('libelf^foo+debug', 'libelf^foo+debug')
self.check_constrain_not_changed('libelf^foo~debug', 'libelf^foo~debug')
self.check_constrain_not_changed('libelf^foo=bgqos_0', 'libelf^foo=bgqos_0')
platform = spack.architecture.sys_type()
default = platform.target('default').name
self.check_constrain_not_changed('libelf^foo='+default, 'libelf^foo='+default)

1
lib/spack/spack/util/executable.py
View File

@ -157,6 +157,7 @@ def streamify(arg, mode):
raise ProcessError("Command exited with status %d:"
% proc.returncode, cmd_line)
if output is str or error is str:
result = ''
if output is str: result += out

40
var/spack/mock_configs/site_spackconfig/compilers.yaml Normal file
View File

@ -0,0 +1,40 @@
compilers:
all:
clang@3.3:
cc: /path/to/clang
cxx: /path/to/clang++
f77: None
fc: None
modules: None
strategy: PATH
gcc@4.5.0:
cc: /path/to/gcc
cxx: /path/to/g++
f77: /path/to/gfortran
fc: /path/to/gfortran
modules: None
strategy: PATH
gcc@5.2.0:
cc: cc
cxx: CC
f77: ftn
fc: ftn
modules:
- PrgEnv-gnu
- gcc/5.2.0
strategy: MODULES
intel@15.0.1:
cc: cc
ccx: CC
f77: ftn
fc: ftn
modules:
- PrgEnv-intel
- intel/15.0.1
strategy: MODULES
intel@15.1.2:
cc: /path/to/icc
cxx: /path/to/ic++
f77: /path/to/ifort
fc: /path/to/ifort
strategy: PATH

38
var/spack/packages/adios/package.py Normal file
View File

@ -0,0 +1,38 @@
import os
from spack import *
class Adios(Package):
"""The Adaptable IO System (ADIOS) provides a simple,
flexible way for scientists to describe the
data in their code that may need to be written,
read, or processed outside of the running simulation
"""
homepage = "http://www.olcf.ornl.gov/center-projects/adios/"
url = "http://users.nccs.gov/~pnorbert/adios-1.9.0.tar.gz"
version('1.9.0', 'dbf5cb10e32add2f04c9b4052b7ffa76')
# Lots of setting up here for this package
# module swap PrgEnv-intel PrgEnv-$COMP
# module load cray-netcdf/4.3.3.1
# module load cray-hdf5/1.8.14
# module load python/2.7.10
depends_on('hdf5')
depends_on('mxml')
def install(self, spec, prefix):
configure_args = ["--prefix=%s" % prefix,
"--with-mxml=%s" % spec['mxml'].prefix,
"--with-hdf5=%s" % spec['hdf5'].prefix,
"--with-netcdf=%s" % os.environ["NETCDF_DIR"],
"--with-infiniband=no",
"MPICC=cc","MPICXX=CC","MPIFC=ftn",
"CPPFLAGS=-DMPICH_IGNORE_CXX_SEEK"]
if spec.satisfies('%gcc'):
configure_args.extend(["CC=gcc", "CXX=g++", "FC=gfortran"])
configure(*configure_args)
make()
make("install")

26
var/spack/packages/mxml/package.py Normal file
View File

@ -0,0 +1,26 @@
import os
from spack import *
class Mxml(Package):
"""Mini-XML is a small XML library that you can use to read and write XML
and XML-like data files in your application without requiring large
non-standard libraries
"""
homepage = "http://www.msweet.org"
url = "http://www.msweet.org/files/project3/mxml-2.9.tar.gz"
version('2.9', 'e21cad0f7aacd18f942aa0568a8dee19')
version('2.8', 'd85ee6d30de053581242c4a86e79a5d2')
version('2.7', '76f2ae49bf0f5745d5cb5d9507774dc9')
version('2.6', '68977789ae64985dddbd1a1a1652642e')
version('2.5', 'f706377fba630b39fa02fd63642b17e5')
# module swap PrgEnv-intel PrgEnv-$COMP (Can use whatever compiler you want to use)
# Case statement to change CC and CXX flags
def install(self, spec, prefix):
configure('--prefix=%s' % prefix, "--disable-shared", 'CFLAGS=-static')
make()
make("install")

42
var/spack/repos/builtin.mock/packages/multimethod/package.py
View File

@ -22,8 +22,11 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import imp
from llnl.util.filesystem import join_path
from spack.util.naming import mod_to_class
from spack import *
import spack.architecture
class Multimethod(Package):
"""This package is designed for use with Spack's multimethod test.
@ -101,25 +104,26 @@ def has_a_default(self):
#
# Make sure we can switch methods on different architectures
# Make sure we can switch methods on different target
#
@when('=x86_64')
def different_by_architecture(self):
return 'x86_64'
@when('=ppc64')
def different_by_architecture(self):
return 'ppc64'
@when('=ppc32')
def different_by_architecture(self):
return 'ppc32'
@when('=arm64')
def different_by_architecture(self):
return 'arm64'
# for platform_name in ['cray_xc', 'darwin', 'linux']:
# file_path = join_path(spack.platform_path, platform_name)
# platform_mod = imp.load_source('spack.platforms', file_path + '.py')
# cls = getattr(platform_mod, mod_to_class(platform_name))
# platform = cls()
platform = spack.architecture.sys_type()
targets = platform.targets.values()
if len(targets) > 1:
targets = targets[:-1]
for target in targets:
@when('='+target.name)
def different_by_target(self):
if isinstance(self.spec.architecture.target,basestring):
return self.spec.architecture.target
else:
return self.spec.architecture.target.name
#
# Make sure we can switch methods on different dependencies
#

2
var/spack/repos/builtin/packages/py-h5py/package.py
View File

@ -13,6 +13,8 @@ class PyH5py(Package):
depends_on('hdf5')
depends_on('py-numpy')
depends_on('py-cython')
depends_on('py-six')
depends_on('py-pkgconfig')
def install(self, spec, prefix):
python('setup.py', 'configure', '--hdf5=%s' % spec['hdf5'].prefix)