Merge branch 'features/spackathon' of github.com:NERSC/spack into crayport

Conflicts:
	lib/spack/spack/compiler.py
This commit is contained in:
Gregory Becker
2015-11-11 10:35:59 -08:00
290 changed files with 813 additions and 737 deletions

View File

@@ -37,6 +37,7 @@
lib_path = join_path(prefix, "lib", "spack")
build_env_path = join_path(lib_path, "env")
module_path = join_path(lib_path, "spack")
arch_path = join_path(module_path, 'architectures')
compilers_path = join_path(module_path, "compilers")
test_path = join_path(module_path, "test")
hooks_path = join_path(module_path, "hooks")

View File

@@ -23,70 +23,190 @@
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
import os
import imp
import platform as py_platform
import inspect
from llnl.util.lang import memoized
from llnl.util.lang import memoized, list_modules
from llnl.util.filesystem import join_path
import llnl.util.tty as tty
import spack
from spack.util.naming import mod_to_class
import spack.error as serr
from spack.version import Version
from external import yaml
class InvalidSysTypeError(serr.SpackError):
def __init__(self, sys_type):
super(InvalidSysTypeError, self).__init__(
"Invalid sys_type value for Spack: " + sys_type)
super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type)
class NoSysTypeError(serr.SpackError):
def __init__(self):
super(NoSysTypeError, self).__init__(
"Could not determine sys_type for this machine.")
super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.")
class Target(object):
""" Target is the processor of the host machine. The host machine may have different front-end
and back-end targets, especially if it is a Cray machine. The target will have a name and
also the module_name (e.g craype-compiler). Targets will also recognize which architecture
they came from using the set_architecture method. Targets will have compiler finding strategies
"""
default_strategy = None # Can probably add a compiler path here
def __init__(self,name, module_name=None):
self.name = name # case of cray "ivybridge" but if it's x86_64
self.module_name = module_name # craype-ivybridge
def set_architecture(self, architecture): # Target should get the architecture class.
self.architecture = architecture
@property
def compiler_strategy(self):
if default_strategy:
return default_strategy
elif self.module_name: # If there is a module_name given then use MODULES
return "MODULES"
else:
return "PATH"
class Architecture(object):
""" Abstract class that each type of Architecture will subclass. Will return a instance of it once it
is returned
"""
priority = None # Subclass needs to set this number. This controls order in which arch is detected.
front_end = None
back_end = None
default = None # The default back end target. On cray ivybridge
def __init__(self, name):
self.targets = {}
self.name = name
def add_target(self, name, target):
"""Used by the architecture specific subclass to list available targets. Raises an error
if the architecture specifies a name that is reserved by spack as an alias.
"""
if name in ['front_end', 'fe', 'back_end', 'be', 'default']:
raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name)
target.set_architecture(self)
self.targets[name] = target
def target(self, name):
"""This is a getter method for the target dictionary that handles defaulting based
on the values provided by default, front-end, and back-end. This can be overwritten
by a subclass for which we want to provide further aliasing options.
"""
if name == 'default':
name = self.default
elif name == 'front_end' or name == 'fe':
name = self.front_end
elif name == 'back_end' or name == 'be':
name = self.back_end
return self.targets[name]
@classmethod
def detect(self):
""" Subclass is responsible for implementing this method.
Returns True if the architecture detects if it is the current architecture
and False if it's not.
"""
raise NotImplementedError()
def __str__(self):
return self.name
def get_sys_type_from_spack_globals():
"""Return the SYS_TYPE from spack globals, or None if it isn't set."""
if not hasattr(spack, "sys_type"):
return None
return None
elif hasattr(spack.sys_type, "__call__"):
return spack.sys_type()
return spack.sys_type() #If in __init__.py there is a sys_type() then call that
else:
return spack.sys_type
return spack.sys_type # Else use the attributed which defaults to None
def get_sys_type_from_environment():
"""Return $SYS_TYPE or None if it's not defined."""
return os.environ.get('SYS_TYPE')
# This is livermore dependent. Hard coded for livermore
#def get_sys_type_from_environment():
# """Return $SYS_TYPE or None if it's not defined."""
# return os.environ.get('SYS_TYPE')
def get_mac_sys_type():
"""Return a Mac OS SYS_TYPE or None if this isn't a mac."""
"""Return a Mac OS SYS_TYPE or None if this isn't a mac.
Front-end config
"""
mac_ver = py_platform.mac_ver()[0]
if not mac_ver:
return None
return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())
return "macosx_%s_%s" % (
Version(mac_ver).up_to(2), py_platform.machine())
def get_sys_type_from_uname():
""" Returns a sys_type from the uname argument
Front-end config
"""
try:
arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
arch, _ = arch_proc.communicate()
return arch.strip()
except:
return None
def get_sys_type_from_config_file():
spack_home_dir = os.environ["HOME"] + "/.spack"
yaml_file = os.path.join(spack_home_dir, 'architecture.yaml')
try:
config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load()
arch = config_dict['architecture']
front = arch['front']
back = arch['back']
return Architecture(front,back)
except:
print "No architecture.yaml config file found"
return None
@memoized
def all_architectures():
modules = []
for name in list_modules(spack.arch_path):
mod_name = 'spack.architectures' + name
path = join_path(spack.arch_path, name) + ".py"
mod = imp.load_source(mod_name, path)
class_name = mod_to_class(name)
if not hasattr(mod, class_name):
tty.die('No class %s defined in %s' % (class_name, mod_name))
cls = getattr(mod, class_name)
if not inspect.isclass(cls):
tty.die('%s.%s is not a class' % (mod_name, class_name))
modules.append(cls)
return modules
@memoized
def sys_type():
"""Returns a SysType for the current machine."""
methods = [get_sys_type_from_spack_globals,
get_sys_type_from_environment,
get_mac_sys_type]
"""Priority of gathering sys-type.
1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30
2. UNAME
3. GLOBALS
4. MAC OSX
Yaml should be a priority here because we want the user to be able to specify the type of architecture to use.
If there is no yaml present then it should move on to the next function and stop immediately once it gets a
arch name
"""
# Try to create an architecture object using the config file FIRST
architecture_list = all_architectures()
architecture_list.sort(key = lambda a: a.priority)
for arch in architecture_list:
if arch.detect():
return arch()
# search for a method that doesn't return None
sys_type = None
for method in methods:
sys_type = method()
if sys_type: break
# Couldn't determine the sys_type for this machine.
if sys_type is None:
return "unknown_arch"
if not isinstance(sys_type, basestring):
raise InvalidSysTypeError(sys_type)
return sys_type

View File

@@ -0,0 +1,19 @@
import os
from spack.architecture import Architecture, Target
class Bgq(Architecture):
priority = 30
front_end = 'power7'
back_end = 'powerpc'
default = 'powerpc'
def __init__(self):
super(Bgq, self).__init__('cray')
self.add_target('power7', Target('power7'))
self.add_target('powerpc', Target('powerpc'))
@classmethod
def detect(self):
return os.path.exists('/bgsys')

View File

@@ -0,0 +1,22 @@
import os
from spack.architecture import Architecture, Target
class Cray(Architecture):
priority = 20
front_end = 'sandybridge'
back_end = 'ivybridge'
default = 'ivybridge'
def __init__(self):
super(Cray, self).__init__('cray')
# Back End compiler needs the proper target module loaded.
self.add_target('ivybridge', Target('ivybridge','craype-ivybridge'))
# Could switch to use modules and fe targets for front end
# Currently using compilers by path for front end.
self.add_target('sandybridge', Target('sandybridge'))
@classmethod
def detect(self):
return os.path.exists('/opt/cray/craype')

View File

@@ -0,0 +1,17 @@
import subprocess
from spack.architecture import Architecture
class Linux(Architecture):
priority = 60
front_end = "x86_64"
back_end = "x86_64"
default = "x86_64"
def __init__(self):
super(Linux, self).__init__('linux')
@classmethod
def detect(self):
arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
arch, _ = arch.communicate()
return 'x86_64' in arch.strip()

View File

@@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs):
return super(MakeExecutable, self).__call__(*args, **kwargs)
def load_module(mod):
"""Takes a module name and removes modules until it is possible to
load that module. It then loads the provided module. Depends on the
modulecmd implementation of modules used in cray and lmod.
"""
#Create an executable of the module command that will output python code
modulecmd = which('modulecmd')
modulecmd.add_default_arg('python')
# Read the module and remove any conflicting modules
# We do this without checking that they are already installed
# for ease of programming because unloading a module that is not
# loaded does nothing.
text = modulecmd('show', mod, return_oe=True).split()
for i, word in enumerate(text):
if word == 'conflict':
exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec'))
# Load the module now that there are no conflicts
load = modulecmd('load', mod, return_oe=True)
exec(compile(load, '<string>', 'exec'))
def set_compiler_environment_variables(pkg):
assert(pkg.spec.concrete)
compiler = pkg.compiler
@@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg):
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
if compiler.PrgEnv:
os.environ['SPACK_CRAYPE'] = compiler.PrgEnv
os.environ['SPACK_COMP_MODULE'] = compiler.module
if compiler.modules:
for mod in compiler.modules:
load_module(mod)
def set_build_environment_variables(pkg):
@@ -163,8 +183,10 @@ def set_build_environment_variables(pkg):
pcdir = join_path(p, libdir, 'pkgconfig')
if os.path.isdir(pcdir):
pkg_config_dirs.append(pcdir)
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
path_put_first("PKG_CONFIG_PATH", pkg_config_dirs)
if pkg.spec.architecture.compiler_strategy.lower() == 'module':
load_module(pkg.spec.architecture.module_name)
def set_module_variables_for_package(pkg):
"""Populate the module scope of install() with some useful functions.
@@ -239,8 +261,8 @@ def get_rpaths(pkg):
def setup_package(pkg):
"""Execute all environment setup routines."""
set_compiler_environment_variables(pkg)
set_build_environment_variables(pkg)
set_compiler_environment_variables(pkg)
set_module_variables_for_package(pkg)
# Allow dependencies to set up environment as well.

View File

@@ -29,6 +29,8 @@
from llnl.util.tty.colify import colify
from llnl.util.lang import index_by
import spack.architecture
import spack.compiler
import spack.compilers
import spack.spec
import spack.config
@@ -38,11 +40,9 @@
description = "Manage compilers"
def setup_parser(subparser):
sp = subparser.add_subparsers(
metavar='SUBCOMMAND', dest='compiler_command')
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command')
update_parser = sp.add_parser(
'add', help='Add compilers to the Spack configuration.')
update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.')
update_parser.add_argument('add_paths', nargs=argparse.REMAINDER)
remove_parser = sp.add_parser('remove', help='remove compiler')
@@ -55,14 +55,17 @@ def setup_parser(subparser):
def compiler_add(args):
"""Search either $PATH or a list of paths for compilers and add them
"""Search either $PATH or a list of paths OR MODULES for compilers and add them
to Spack's configuration."""
paths = args.add_paths
paths = args.add_paths # This might be a parser method. Parsing method to add_paths
if not paths:
paths = get_path('PATH')
compilers = [c for c in spack.compilers.find_compilers(*args.add_paths)
if c.spec not in spack.compilers.all_compilers()]
if c.spec not in spack.compilers.all_compilers()]
if compilers:
spack.compilers.add_compilers_to_config('user', *compilers)

View File

@@ -282,6 +282,11 @@ def find_in_modules(cls):
modulecmd
matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output)
loaded_modules = os.environ["LOADEDMODULES"].split(":")
#output = _shell('module avail %s' % cls.PrgEnv_compiler)
for module in loaded_modules:
match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module)
for name, version in matches:
v = version + '-craype'
comp = cls(spack.spec.CompilerSpec(name + '@' + v),

View File

@@ -122,6 +122,34 @@ def concretize_architecture(self, spec):
return True # changed
def new_concretize_architecture(self, spec):
"""If the spec already has an architecture and it is a an architecture type,
return. Otherwise, if it has an architecture that is a string type, generate an
architecture based on that type. If it has no architecture and the root of the
DAG has an architecture, then use that. Otherwise, take the system's default
architecture.
"""
if spec.architecture is not None:
if isinstance(spec.architecture,spack.architecture.Target):
return False
else:
arch = spack.architecture.sys_type()
spec.architecture = arch.target(spec.architecture)
return True #changed
if spec.root.architecture:
if isinstance(spec.root.architecture,spack.architecture.Target):
spec.architecture = spec.root.architecture
else:
arch = spack.architecture.sys_type()
spec.architecture = arch.target(spec.root.architecture)
else:
arch = spack.architecture.sys_type()
spec.architecture = arch.target('default')
return True #changed
def concretize_variants(self, spec):
"""If the spec already has variants filled in, return. Otherwise, add
the default variants from the package specification.

View File

@@ -31,7 +31,8 @@
import spack
"""Names of tests to be included in Spack's test suite"""
test_names = ['versions',
"""test_names = ['architecture',
'versions',
'url_parse',
'url_substitution',
'packages',
@@ -57,7 +58,8 @@
'optional_deps',
'make_executable',
'configure_guess']
"""
test_names = ['architecture']
def list_tests():
"""Return names of all tests that can be run for Spack."""

View File

@@ -0,0 +1,19 @@
""" Test checks if the architecture class is created correctly and also that
the functions are looking for the correct architecture name
"""
import unittest
import spack
from spack.architecture import *
class ArchitectureTest(unittest.TestCase):
def test_Architecture_class(self):
a = Architecture('Cray-XC40')
a.add_arch_strategy()
self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'})
def test_get_sys_type_from_config_file(self):
output_arch_class = get_sys_type_from_config_file()
my_arch_class = Architecture('Linux x86_64','Cray-xc40')
self.assertEqual(output_arch_class, my_arch_class)

View File

@@ -56,7 +56,11 @@ def command(self):
def __call__(self, *args, **kwargs):
"""Run the executable with subprocess.check_output, return output."""
return_output = kwargs.get("return_output", False)
# Return oe returns a combined stream, setting both output and error
# without setting return oe returns them concatenated by a double line break
return_oe = kwargs.get("return_oe", False)
return_output = True if return_oe else kwargs.get("return_output", False)
return_error = True if return_oe else kwargs.get("return_error", False)
fail_on_error = kwargs.get("fail_on_error", True)
ignore_errors = kwargs.get("ignore_errors", ())
@@ -95,8 +99,8 @@ def streamify(arg, mode):
proc = subprocess.Popen(
cmd,
stdin=input,
stderr=error,
stdout=subprocess.PIPE if return_output else output)
stdout=subprocess.PIPE if return_output else output,
stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error))
out, err = proc.communicate()
self.returncode = proc.returncode
@@ -104,8 +108,15 @@ def streamify(arg, mode):
if fail_on_error and rc != 0 and (rc not in ignore_errors):
raise ProcessError("Command exited with status %d:"
% proc.returncode, cmd_line)
if return_output:
return out
# Return out or error if specified. Return combined stream if requested,
# otherwise return them concatenated by double line break if both requested.
if return_output or return_error:
if return_oe or not return_error:
return out
elif return_output:
return out+'\n\n'+err
else:
return err
except OSError, e:
raise ProcessError(