Merge branch 'features/spackathon' of github.com:NERSC/spack into crayport
Conflicts: lib/spack/spack/compiler.py
This commit is contained in:
@@ -37,6 +37,7 @@
|
||||
lib_path = join_path(prefix, "lib", "spack")
|
||||
build_env_path = join_path(lib_path, "env")
|
||||
module_path = join_path(lib_path, "spack")
|
||||
arch_path = join_path(module_path, 'architectures')
|
||||
compilers_path = join_path(module_path, "compilers")
|
||||
test_path = join_path(module_path, "test")
|
||||
hooks_path = join_path(module_path, "hooks")
|
||||
|
@@ -23,70 +23,190 @@
|
||||
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
import os
|
||||
import imp
|
||||
import platform as py_platform
|
||||
import inspect
|
||||
|
||||
from llnl.util.lang import memoized
|
||||
from llnl.util.lang import memoized, list_modules
|
||||
from llnl.util.filesystem import join_path
|
||||
import llnl.util.tty as tty
|
||||
|
||||
import spack
|
||||
from spack.util.naming import mod_to_class
|
||||
import spack.error as serr
|
||||
from spack.version import Version
|
||||
|
||||
from external import yaml
|
||||
|
||||
class InvalidSysTypeError(serr.SpackError):
|
||||
def __init__(self, sys_type):
|
||||
super(InvalidSysTypeError, self).__init__(
|
||||
"Invalid sys_type value for Spack: " + sys_type)
|
||||
super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type)
|
||||
|
||||
|
||||
class NoSysTypeError(serr.SpackError):
|
||||
def __init__(self):
|
||||
super(NoSysTypeError, self).__init__(
|
||||
"Could not determine sys_type for this machine.")
|
||||
super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.")
|
||||
|
||||
|
||||
class Target(object):
|
||||
""" Target is the processor of the host machine. The host machine may have different front-end
|
||||
and back-end targets, especially if it is a Cray machine. The target will have a name and
|
||||
also the module_name (e.g craype-compiler). Targets will also recognize which architecture
|
||||
they came from using the set_architecture method. Targets will have compiler finding strategies
|
||||
"""
|
||||
default_strategy = None # Can probably add a compiler path here
|
||||
|
||||
def __init__(self,name, module_name=None):
|
||||
self.name = name # case of cray "ivybridge" but if it's x86_64
|
||||
self.module_name = module_name # craype-ivybridge
|
||||
|
||||
def set_architecture(self, architecture): # Target should get the architecture class.
|
||||
self.architecture = architecture
|
||||
|
||||
@property
|
||||
def compiler_strategy(self):
|
||||
if default_strategy:
|
||||
return default_strategy
|
||||
elif self.module_name: # If there is a module_name given then use MODULES
|
||||
return "MODULES"
|
||||
else:
|
||||
return "PATH"
|
||||
|
||||
class Architecture(object):
|
||||
""" Abstract class that each type of Architecture will subclass. Will return a instance of it once it
|
||||
is returned
|
||||
"""
|
||||
|
||||
priority = None # Subclass needs to set this number. This controls order in which arch is detected.
|
||||
front_end = None
|
||||
back_end = None
|
||||
default = None # The default back end target. On cray ivybridge
|
||||
|
||||
def __init__(self, name):
|
||||
self.targets = {}
|
||||
self.name = name
|
||||
|
||||
def add_target(self, name, target):
|
||||
"""Used by the architecture specific subclass to list available targets. Raises an error
|
||||
if the architecture specifies a name that is reserved by spack as an alias.
|
||||
"""
|
||||
if name in ['front_end', 'fe', 'back_end', 'be', 'default']:
|
||||
raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name)
|
||||
target.set_architecture(self)
|
||||
self.targets[name] = target
|
||||
|
||||
def target(self, name):
|
||||
"""This is a getter method for the target dictionary that handles defaulting based
|
||||
on the values provided by default, front-end, and back-end. This can be overwritten
|
||||
by a subclass for which we want to provide further aliasing options.
|
||||
"""
|
||||
if name == 'default':
|
||||
name = self.default
|
||||
elif name == 'front_end' or name == 'fe':
|
||||
name = self.front_end
|
||||
elif name == 'back_end' or name == 'be':
|
||||
name = self.back_end
|
||||
|
||||
return self.targets[name]
|
||||
|
||||
@classmethod
|
||||
def detect(self):
|
||||
""" Subclass is responsible for implementing this method.
|
||||
Returns True if the architecture detects if it is the current architecture
|
||||
and False if it's not.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def __str__(self):
|
||||
return self.name
|
||||
|
||||
|
||||
def get_sys_type_from_spack_globals():
|
||||
"""Return the SYS_TYPE from spack globals, or None if it isn't set."""
|
||||
if not hasattr(spack, "sys_type"):
|
||||
return None
|
||||
return None
|
||||
elif hasattr(spack.sys_type, "__call__"):
|
||||
return spack.sys_type()
|
||||
return spack.sys_type() #If in __init__.py there is a sys_type() then call that
|
||||
else:
|
||||
return spack.sys_type
|
||||
return spack.sys_type # Else use the attributed which defaults to None
|
||||
|
||||
|
||||
def get_sys_type_from_environment():
|
||||
"""Return $SYS_TYPE or None if it's not defined."""
|
||||
return os.environ.get('SYS_TYPE')
|
||||
# This is livermore dependent. Hard coded for livermore
|
||||
#def get_sys_type_from_environment():
|
||||
# """Return $SYS_TYPE or None if it's not defined."""
|
||||
# return os.environ.get('SYS_TYPE')
|
||||
|
||||
|
||||
def get_mac_sys_type():
|
||||
"""Return a Mac OS SYS_TYPE or None if this isn't a mac."""
|
||||
"""Return a Mac OS SYS_TYPE or None if this isn't a mac.
|
||||
Front-end config
|
||||
"""
|
||||
mac_ver = py_platform.mac_ver()[0]
|
||||
if not mac_ver:
|
||||
return None
|
||||
return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())
|
||||
|
||||
return "macosx_%s_%s" % (
|
||||
Version(mac_ver).up_to(2), py_platform.machine())
|
||||
|
||||
def get_sys_type_from_uname():
|
||||
""" Returns a sys_type from the uname argument
|
||||
Front-end config
|
||||
"""
|
||||
try:
|
||||
arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
|
||||
arch, _ = arch_proc.communicate()
|
||||
return arch.strip()
|
||||
except:
|
||||
return None
|
||||
|
||||
def get_sys_type_from_config_file():
|
||||
|
||||
spack_home_dir = os.environ["HOME"] + "/.spack"
|
||||
yaml_file = os.path.join(spack_home_dir, 'architecture.yaml')
|
||||
try:
|
||||
config_dict = yaml.load(open(yaml_file)) # Fix this to have yaml.load()
|
||||
arch = config_dict['architecture']
|
||||
front = arch['front']
|
||||
back = arch['back']
|
||||
return Architecture(front,back)
|
||||
|
||||
except:
|
||||
print "No architecture.yaml config file found"
|
||||
return None
|
||||
|
||||
|
||||
@memoized
|
||||
def all_architectures():
|
||||
modules = []
|
||||
for name in list_modules(spack.arch_path):
|
||||
mod_name = 'spack.architectures' + name
|
||||
path = join_path(spack.arch_path, name) + ".py"
|
||||
mod = imp.load_source(mod_name, path)
|
||||
class_name = mod_to_class(name)
|
||||
if not hasattr(mod, class_name):
|
||||
tty.die('No class %s defined in %s' % (class_name, mod_name))
|
||||
cls = getattr(mod, class_name)
|
||||
if not inspect.isclass(cls):
|
||||
tty.die('%s.%s is not a class' % (mod_name, class_name))
|
||||
|
||||
modules.append(cls)
|
||||
|
||||
return modules
|
||||
|
||||
@memoized
|
||||
def sys_type():
|
||||
"""Returns a SysType for the current machine."""
|
||||
methods = [get_sys_type_from_spack_globals,
|
||||
get_sys_type_from_environment,
|
||||
get_mac_sys_type]
|
||||
"""Priority of gathering sys-type.
|
||||
1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30
|
||||
2. UNAME
|
||||
3. GLOBALS
|
||||
4. MAC OSX
|
||||
Yaml should be a priority here because we want the user to be able to specify the type of architecture to use.
|
||||
If there is no yaml present then it should move on to the next function and stop immediately once it gets a
|
||||
arch name
|
||||
"""
|
||||
# Try to create an architecture object using the config file FIRST
|
||||
architecture_list = all_architectures()
|
||||
architecture_list.sort(key = lambda a: a.priority)
|
||||
|
||||
for arch in architecture_list:
|
||||
if arch.detect():
|
||||
return arch()
|
||||
|
||||
# search for a method that doesn't return None
|
||||
sys_type = None
|
||||
for method in methods:
|
||||
sys_type = method()
|
||||
if sys_type: break
|
||||
|
||||
# Couldn't determine the sys_type for this machine.
|
||||
if sys_type is None:
|
||||
return "unknown_arch"
|
||||
|
||||
if not isinstance(sys_type, basestring):
|
||||
raise InvalidSysTypeError(sys_type)
|
||||
|
||||
return sys_type
|
||||
|
0
lib/spack/spack/architectures/__init__.py
Normal file
0
lib/spack/spack/architectures/__init__.py
Normal file
19
lib/spack/spack/architectures/bgq.py
Normal file
19
lib/spack/spack/architectures/bgq.py
Normal file
@@ -0,0 +1,19 @@
|
||||
import os
|
||||
|
||||
from spack.architecture import Architecture, Target
|
||||
|
||||
class Bgq(Architecture):
|
||||
priority = 30
|
||||
front_end = 'power7'
|
||||
back_end = 'powerpc'
|
||||
default = 'powerpc'
|
||||
|
||||
def __init__(self):
|
||||
super(Bgq, self).__init__('cray')
|
||||
self.add_target('power7', Target('power7'))
|
||||
self.add_target('powerpc', Target('powerpc'))
|
||||
|
||||
@classmethod
|
||||
def detect(self):
|
||||
return os.path.exists('/bgsys')
|
||||
|
22
lib/spack/spack/architectures/cray.py
Normal file
22
lib/spack/spack/architectures/cray.py
Normal file
@@ -0,0 +1,22 @@
|
||||
import os
|
||||
|
||||
from spack.architecture import Architecture, Target
|
||||
|
||||
class Cray(Architecture):
|
||||
priority = 20
|
||||
front_end = 'sandybridge'
|
||||
back_end = 'ivybridge'
|
||||
default = 'ivybridge'
|
||||
|
||||
def __init__(self):
|
||||
super(Cray, self).__init__('cray')
|
||||
# Back End compiler needs the proper target module loaded.
|
||||
self.add_target('ivybridge', Target('ivybridge','craype-ivybridge'))
|
||||
# Could switch to use modules and fe targets for front end
|
||||
# Currently using compilers by path for front end.
|
||||
self.add_target('sandybridge', Target('sandybridge'))
|
||||
|
||||
@classmethod
|
||||
def detect(self):
|
||||
return os.path.exists('/opt/cray/craype')
|
||||
|
17
lib/spack/spack/architectures/linux.py
Normal file
17
lib/spack/spack/architectures/linux.py
Normal file
@@ -0,0 +1,17 @@
|
||||
import subprocess
|
||||
from spack.architecture import Architecture
|
||||
|
||||
class Linux(Architecture):
|
||||
priority = 60
|
||||
front_end = "x86_64"
|
||||
back_end = "x86_64"
|
||||
default = "x86_64"
|
||||
|
||||
def __init__(self):
|
||||
super(Linux, self).__init__('linux')
|
||||
|
||||
@classmethod
|
||||
def detect(self):
|
||||
arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
|
||||
arch, _ = arch.communicate()
|
||||
return 'x86_64' in arch.strip()
|
@@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs):
|
||||
return super(MakeExecutable, self).__call__(*args, **kwargs)
|
||||
|
||||
|
||||
def load_module(mod):
|
||||
"""Takes a module name and removes modules until it is possible to
|
||||
load that module. It then loads the provided module. Depends on the
|
||||
modulecmd implementation of modules used in cray and lmod.
|
||||
"""
|
||||
#Create an executable of the module command that will output python code
|
||||
modulecmd = which('modulecmd')
|
||||
modulecmd.add_default_arg('python')
|
||||
|
||||
# Read the module and remove any conflicting modules
|
||||
# We do this without checking that they are already installed
|
||||
# for ease of programming because unloading a module that is not
|
||||
# loaded does nothing.
|
||||
text = modulecmd('show', mod, return_oe=True).split()
|
||||
for i, word in enumerate(text):
|
||||
if word == 'conflict':
|
||||
exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec'))
|
||||
# Load the module now that there are no conflicts
|
||||
load = modulecmd('load', mod, return_oe=True)
|
||||
exec(compile(load, '<string>', 'exec'))
|
||||
|
||||
|
||||
def set_compiler_environment_variables(pkg):
|
||||
assert(pkg.spec.concrete)
|
||||
compiler = pkg.compiler
|
||||
@@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg):
|
||||
|
||||
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
|
||||
|
||||
if compiler.PrgEnv:
|
||||
os.environ['SPACK_CRAYPE'] = compiler.PrgEnv
|
||||
os.environ['SPACK_COMP_MODULE'] = compiler.module
|
||||
|
||||
|
||||
if compiler.modules:
|
||||
for mod in compiler.modules:
|
||||
load_module(mod)
|
||||
|
||||
|
||||
def set_build_environment_variables(pkg):
|
||||
@@ -163,8 +183,10 @@ def set_build_environment_variables(pkg):
|
||||
pcdir = join_path(p, libdir, 'pkgconfig')
|
||||
if os.path.isdir(pcdir):
|
||||
pkg_config_dirs.append(pcdir)
|
||||
path_set("PKG_CONFIG_PATH", pkg_config_dirs)
|
||||
path_put_first("PKG_CONFIG_PATH", pkg_config_dirs)
|
||||
|
||||
if pkg.spec.architecture.compiler_strategy.lower() == 'module':
|
||||
load_module(pkg.spec.architecture.module_name)
|
||||
|
||||
def set_module_variables_for_package(pkg):
|
||||
"""Populate the module scope of install() with some useful functions.
|
||||
@@ -239,8 +261,8 @@ def get_rpaths(pkg):
|
||||
|
||||
def setup_package(pkg):
|
||||
"""Execute all environment setup routines."""
|
||||
set_compiler_environment_variables(pkg)
|
||||
set_build_environment_variables(pkg)
|
||||
set_compiler_environment_variables(pkg)
|
||||
set_module_variables_for_package(pkg)
|
||||
|
||||
# Allow dependencies to set up environment as well.
|
||||
|
@@ -29,6 +29,8 @@
|
||||
from llnl.util.tty.colify import colify
|
||||
from llnl.util.lang import index_by
|
||||
|
||||
import spack.architecture
|
||||
import spack.compiler
|
||||
import spack.compilers
|
||||
import spack.spec
|
||||
import spack.config
|
||||
@@ -38,11 +40,9 @@
|
||||
description = "Manage compilers"
|
||||
|
||||
def setup_parser(subparser):
|
||||
sp = subparser.add_subparsers(
|
||||
metavar='SUBCOMMAND', dest='compiler_command')
|
||||
sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command')
|
||||
|
||||
update_parser = sp.add_parser(
|
||||
'add', help='Add compilers to the Spack configuration.')
|
||||
update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.')
|
||||
update_parser.add_argument('add_paths', nargs=argparse.REMAINDER)
|
||||
|
||||
remove_parser = sp.add_parser('remove', help='remove compiler')
|
||||
@@ -55,14 +55,17 @@ def setup_parser(subparser):
|
||||
|
||||
|
||||
def compiler_add(args):
|
||||
"""Search either $PATH or a list of paths for compilers and add them
|
||||
"""Search either $PATH or a list of paths OR MODULES for compilers and add them
|
||||
to Spack's configuration."""
|
||||
paths = args.add_paths
|
||||
|
||||
|
||||
paths = args.add_paths # This might be a parser method. Parsing method to add_paths
|
||||
if not paths:
|
||||
paths = get_path('PATH')
|
||||
|
||||
|
||||
compilers = [c for c in spack.compilers.find_compilers(*args.add_paths)
|
||||
if c.spec not in spack.compilers.all_compilers()]
|
||||
if c.spec not in spack.compilers.all_compilers()]
|
||||
|
||||
|
||||
if compilers:
|
||||
spack.compilers.add_compilers_to_config('user', *compilers)
|
||||
|
@@ -282,6 +282,11 @@ def find_in_modules(cls):
|
||||
modulecmd
|
||||
matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output)
|
||||
|
||||
loaded_modules = os.environ["LOADEDMODULES"].split(":")
|
||||
#output = _shell('module avail %s' % cls.PrgEnv_compiler)
|
||||
for module in loaded_modules:
|
||||
match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module)
|
||||
|
||||
for name, version in matches:
|
||||
v = version + '-craype'
|
||||
comp = cls(spack.spec.CompilerSpec(name + '@' + v),
|
||||
|
@@ -122,6 +122,34 @@ def concretize_architecture(self, spec):
|
||||
return True # changed
|
||||
|
||||
|
||||
def new_concretize_architecture(self, spec):
|
||||
"""If the spec already has an architecture and it is a an architecture type,
|
||||
return. Otherwise, if it has an architecture that is a string type, generate an
|
||||
architecture based on that type. If it has no architecture and the root of the
|
||||
DAG has an architecture, then use that. Otherwise, take the system's default
|
||||
architecture.
|
||||
"""
|
||||
if spec.architecture is not None:
|
||||
if isinstance(spec.architecture,spack.architecture.Target):
|
||||
return False
|
||||
else:
|
||||
arch = spack.architecture.sys_type()
|
||||
spec.architecture = arch.target(spec.architecture)
|
||||
return True #changed
|
||||
|
||||
if spec.root.architecture:
|
||||
if isinstance(spec.root.architecture,spack.architecture.Target):
|
||||
spec.architecture = spec.root.architecture
|
||||
else:
|
||||
arch = spack.architecture.sys_type()
|
||||
spec.architecture = arch.target(spec.root.architecture)
|
||||
else:
|
||||
arch = spack.architecture.sys_type()
|
||||
spec.architecture = arch.target('default')
|
||||
|
||||
return True #changed
|
||||
|
||||
|
||||
def concretize_variants(self, spec):
|
||||
"""If the spec already has variants filled in, return. Otherwise, add
|
||||
the default variants from the package specification.
|
||||
|
@@ -31,7 +31,8 @@
|
||||
import spack
|
||||
|
||||
"""Names of tests to be included in Spack's test suite"""
|
||||
test_names = ['versions',
|
||||
"""test_names = ['architecture',
|
||||
'versions',
|
||||
'url_parse',
|
||||
'url_substitution',
|
||||
'packages',
|
||||
@@ -57,7 +58,8 @@
|
||||
'optional_deps',
|
||||
'make_executable',
|
||||
'configure_guess']
|
||||
|
||||
"""
|
||||
test_names = ['architecture']
|
||||
|
||||
def list_tests():
|
||||
"""Return names of all tests that can be run for Spack."""
|
||||
|
19
lib/spack/spack/test/architecture.py
Normal file
19
lib/spack/spack/test/architecture.py
Normal file
@@ -0,0 +1,19 @@
|
||||
""" Test checks if the architecture class is created correctly and also that
|
||||
the functions are looking for the correct architecture name
|
||||
"""
|
||||
import unittest
|
||||
import spack
|
||||
from spack.architecture import *
|
||||
|
||||
class ArchitectureTest(unittest.TestCase):
|
||||
|
||||
def test_Architecture_class(self):
|
||||
a = Architecture('Cray-XC40')
|
||||
a.add_arch_strategy()
|
||||
self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'})
|
||||
|
||||
def test_get_sys_type_from_config_file(self):
|
||||
output_arch_class = get_sys_type_from_config_file()
|
||||
my_arch_class = Architecture('Linux x86_64','Cray-xc40')
|
||||
|
||||
self.assertEqual(output_arch_class, my_arch_class)
|
@@ -56,7 +56,11 @@ def command(self):
|
||||
|
||||
def __call__(self, *args, **kwargs):
|
||||
"""Run the executable with subprocess.check_output, return output."""
|
||||
return_output = kwargs.get("return_output", False)
|
||||
# Return oe returns a combined stream, setting both output and error
|
||||
# without setting return oe returns them concatenated by a double line break
|
||||
return_oe = kwargs.get("return_oe", False)
|
||||
return_output = True if return_oe else kwargs.get("return_output", False)
|
||||
return_error = True if return_oe else kwargs.get("return_error", False)
|
||||
fail_on_error = kwargs.get("fail_on_error", True)
|
||||
ignore_errors = kwargs.get("ignore_errors", ())
|
||||
|
||||
@@ -95,8 +99,8 @@ def streamify(arg, mode):
|
||||
proc = subprocess.Popen(
|
||||
cmd,
|
||||
stdin=input,
|
||||
stderr=error,
|
||||
stdout=subprocess.PIPE if return_output else output)
|
||||
stdout=subprocess.PIPE if return_output else output,
|
||||
stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error))
|
||||
out, err = proc.communicate()
|
||||
self.returncode = proc.returncode
|
||||
|
||||
@@ -104,8 +108,15 @@ def streamify(arg, mode):
|
||||
if fail_on_error and rc != 0 and (rc not in ignore_errors):
|
||||
raise ProcessError("Command exited with status %d:"
|
||||
% proc.returncode, cmd_line)
|
||||
if return_output:
|
||||
return out
|
||||
# Return out or error if specified. Return combined stream if requested,
|
||||
# otherwise return them concatenated by double line break if both requested.
|
||||
if return_output or return_error:
|
||||
if return_oe or not return_error:
|
||||
return out
|
||||
elif return_output:
|
||||
return out+'\n\n'+err
|
||||
else:
|
||||
return err
|
||||
|
||||
except OSError, e:
|
||||
raise ProcessError(
|
||||
|
1
lib/spack/spack/util/python_recipe_parser
Submodule
1
lib/spack/spack/util/python_recipe_parser
Submodule
Submodule lib/spack/spack/util/python_recipe_parser added at 437a62abb3
Reference in New Issue
Block a user