Merge branch 'features/spackathon' of github.com:NERSC/spack into crayport

Conflicts:
	lib/spack/spack/compiler.py

lib/spack/spack/__init__.py

@@ -37,6 +37,7 @@
 lib_path       = join_path(prefix, "lib", "spack")
 build_env_path = join_path(lib_path, "env")
 module_path    = join_path(lib_path, "spack")
+arch_path      = join_path(module_path, 'architectures')
 compilers_path = join_path(module_path, "compilers")
 test_path      = join_path(module_path, "test")
 hooks_path     = join_path(module_path, "hooks")

lib/spack/spack/architecture.py

@@ -23,70 +23,190 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
+import imp
 import platform as py_platform
+import inspect
 
-from llnl.util.lang import memoized
+from llnl.util.lang import memoized, list_modules
+from llnl.util.filesystem import join_path
+import llnl.util.tty as tty
 
 import spack
+from spack.util.naming import mod_to_class
 import spack.error as serr
 from spack.version import Version
-
+from external import yaml
 
 class InvalidSysTypeError(serr.SpackError):
     def __init__(self, sys_type):
-        super(InvalidSysTypeError, self).__init__(
-            "Invalid sys_type value for Spack: " + sys_type)
+        super(InvalidSysTypeError, self).__init__("Invalid sys_type value for Spack: " + sys_type)
 
 
 class NoSysTypeError(serr.SpackError):
     def __init__(self):
-        super(NoSysTypeError, self).__init__(
-            "Could not determine sys_type for this machine.")
+        super(NoSysTypeError, self).__init__("Could not determine sys_type for this machine.")
+
+
+class Target(object):
+    """ Target is the processor of the host machine. The host machine may have different front-end
+        and back-end targets, especially if it is a Cray machine. The target will have a name and
+        also the module_name (e.g craype-compiler). Targets will also recognize which architecture
+        they came from using the set_architecture method. Targets will have compiler finding strategies
+        """
+    default_strategy = None # Can probably add a compiler path here
+
+    def __init__(self,name, module_name=None):
+        self.name = name # case of cray "ivybridge" but if it's x86_64
+        self.module_name = module_name # craype-ivybridge
+
+    def set_architecture(self, architecture): # Target should get the architecture class.
+        self.architecture = architecture
+        
+    @property
+    def compiler_strategy(self):
+        if default_strategy:
+            return default_strategy
+        elif self.module_name: # If there is a module_name given then use MODULES
+            return "MODULES"
+        else:
+            return "PATH"
+
+class Architecture(object):
+    """ Abstract class that each type of Architecture will subclass. Will return a instance of it once it 
+        is returned
+    """
+
+    priority        = None # Subclass needs to set this number. This controls order in which arch is detected.
+    front_end       = None
+    back_end        = None
+    default         = None # The default back end target. On cray ivybridge
+
+    def __init__(self, name):
+        self.targets = {}
+        self.name = name
+
+    def add_target(self, name, target):
+        """Used by the architecture specific subclass to list available targets. Raises an error
+        if the architecture specifies a name that is reserved by spack as an alias.
+        """
+        if name in ['front_end', 'fe', 'back_end', 'be', 'default']:
+            raise ValueError("%s is a spack reserved alias and cannot be the name of a target" % name)
+        target.set_architecture(self)
+        self.targets[name] = target
+
+    def target(self, name):
+        """This is a getter method for the target dictionary that handles defaulting based
+        on the values provided by default, front-end, and back-end. This can be overwritten
+        by a subclass for which we want to provide further aliasing options.
+        """
+        if name == 'default':
+            name = self.default
+        elif name == 'front_end' or name == 'fe':
+            name = self.front_end
+        elif name == 'back_end' or name == 'be':
+            name = self.back_end
+
+        return self.targets[name]
+
+    @classmethod
+    def detect(self):
+        """ Subclass is responsible for implementing this method. 
+            Returns True if the architecture detects if it is the current architecture
+            and False if it's not.
+        """
+        raise NotImplementedError()
+    
+    def __str__(self):
+        return self.name
 
 
 def get_sys_type_from_spack_globals():
     """Return the SYS_TYPE from spack globals, or None if it isn't set."""
     if not hasattr(spack, "sys_type"):
-        return None
+        return None 
     elif hasattr(spack.sys_type, "__call__"):
-        return spack.sys_type()
+        return spack.sys_type() #If in __init__.py there is a sys_type() then call that
     else:
-        return spack.sys_type
+        return spack.sys_type # Else use the attributed which defaults to None
 
 
-def get_sys_type_from_environment():
-    """Return $SYS_TYPE or None if it's not defined."""
-    return os.environ.get('SYS_TYPE')
+# This is livermore dependent. Hard coded for livermore
+#def get_sys_type_from_environment():
+#    """Return $SYS_TYPE or None if it's not defined."""
+#    return os.environ.get('SYS_TYPE')
 
 
 def get_mac_sys_type():
-    """Return a Mac OS SYS_TYPE or None if this isn't a mac."""
+    """Return a Mac OS SYS_TYPE or None if this isn't a mac.
+       Front-end config
+    """
     mac_ver = py_platform.mac_ver()[0]
     if not mac_ver:
         return None
+    return "macosx_%s_%s" % (Version(mac_ver).up_to(2), py_platform.machine())
 
-    return "macosx_%s_%s" % (
-        Version(mac_ver).up_to(2), py_platform.machine())
+
+def get_sys_type_from_uname():
+    """ Returns a sys_type from the uname argument 
+        Front-end config
+    """
+    try:
+        arch_proc = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
+        arch, _ = arch_proc.communicate()
+        return arch.strip()
+    except:
+        return None
+
+def get_sys_type_from_config_file():
+     
+    spack_home_dir = os.environ["HOME"] + "/.spack" 
+    yaml_file = os.path.join(spack_home_dir, 'architecture.yaml')
+    try:
+        config_dict = yaml.load(open(yaml_file))  # Fix this to have yaml.load()
+        arch = config_dict['architecture']
+        front = arch['front']
+        back = arch['back']
+        return Architecture(front,back)
+    
+    except:
+        print "No architecture.yaml config file found"
+        return None
 
 
+@memoized
+def all_architectures():
+    modules = []
+    for name in list_modules(spack.arch_path):
+        mod_name = 'spack.architectures' + name
+        path = join_path(spack.arch_path, name) + ".py"
+        mod = imp.load_source(mod_name, path)
+        class_name = mod_to_class(name)
+        if not hasattr(mod, class_name):
+            tty.die('No class %s defined in %s' % (class_name, mod_name)) 
+        cls = getattr(mod, class_name)
+        if not inspect.isclass(cls):
+            tty.die('%s.%s is not a class' % (mod_name, class_name))
+
+        modules.append(cls)
+
+    return modules
+
 @memoized
 def sys_type():
-    """Returns a SysType for the current machine."""
-    methods = [get_sys_type_from_spack_globals,
-               get_sys_type_from_environment,
-               get_mac_sys_type]
+    """Priority of gathering sys-type.
+       1. YAML file that the user specifies the name of the architecture. e.g Cray-XC40 or Cray-XC30
+       2. UNAME
+       3. GLOBALS
+       4. MAC OSX 
+       Yaml should be a priority here because we want the user to be able to specify the type of architecture to use.
+       If there is no yaml present then it should move on to the next function and stop immediately once it gets a 
+       arch name
+    """
+    # Try to create an architecture object using the config file FIRST
+    architecture_list = all_architectures() 
+    architecture_list.sort(key = lambda a: a.priority) 
+    
+    for arch in architecture_list:
+        if arch.detect():
+            return arch()
 
-    # search for a method that doesn't return None
-    sys_type = None
-    for method in methods:
-        sys_type = method()
-        if sys_type: break
-
-    # Couldn't determine the sys_type for this machine.
-    if sys_type is None:
-        return "unknown_arch"
-
-    if not isinstance(sys_type, basestring):
-        raise InvalidSysTypeError(sys_type)
-
-    return sys_type

lib/spack/spack/architectures/bgq.py

@@ -0,0 +1,19 @@
+import os
+
+from spack.architecture import Architecture, Target
+
+class Bgq(Architecture):
+    priority    = 30
+    front_end   = 'power7'
+    back_end    = 'powerpc'
+    default     = 'powerpc'
+
+    def __init__(self):
+        super(Bgq, self).__init__('cray')
+        self.add_target('power7', Target('power7'))
+        self.add_target('powerpc', Target('powerpc'))
+
+    @classmethod
+    def detect(self):
+        return os.path.exists('/bgsys')
+    

lib/spack/spack/architectures/cray.py

@@ -0,0 +1,22 @@
+import os
+
+from spack.architecture import Architecture, Target
+
+class Cray(Architecture):
+    priority    = 20
+    front_end   = 'sandybridge' 
+    back_end    = 'ivybridge'
+    default     = 'ivybridge'
+
+    def __init__(self):
+        super(Cray, self).__init__('cray')
+        # Back End compiler needs the proper target module loaded.
+        self.add_target('ivybridge', Target('ivybridge','craype-ivybridge'))
+        # Could switch to use modules and fe targets for front end
+        # Currently using compilers by path for front end.
+        self.add_target('sandybridge', Target('sandybridge'))
+
+    @classmethod
+    def detect(self):
+        return os.path.exists('/opt/cray/craype')
+    

lib/spack/spack/architectures/linux.py

@@ -0,0 +1,17 @@
+import subprocess 
+from spack.architecture import Architecture
+
+class Linux(Architecture):
+    priority    = 60 
+    front_end   = "x86_64"
+    back_end    = "x86_64"
+    default     = "x86_64"
+
+    def __init__(self):
+        super(Linux, self).__init__('linux')
+
+    @classmethod
+    def detect(self):
+        arch = subprocess.Popen(['uname', '-i'], stdout = subprocess.PIPE)
+        arch, _ = arch.communicate()
+        return 'x86_64' in arch.strip()

lib/spack/spack/build_environment.py

@@ -86,6 +86,28 @@ def __call__(self, *args, **kwargs):
         return super(MakeExecutable, self).__call__(*args, **kwargs)
 
 
+def load_module(mod):
+    """Takes a module name and removes modules until it is possible to
+    load that module. It then loads the provided module. Depends on the 
+    modulecmd implementation of modules used in cray and lmod.
+    """
+    #Create an executable of the module command that will output python code
+    modulecmd = which('modulecmd')
+    modulecmd.add_default_arg('python')
+    
+    # Read the module and remove any conflicting modules
+    # We do this without checking that they are already installed
+    # for ease of programming because unloading a module that is not
+    # loaded does nothing.
+    text = modulecmd('show', mod, return_oe=True).split()
+    for i, word in enumerate(text):
+        if word == 'conflict':
+            exec(compile(modulecmd('unload', text[i+1], return_oe=True), '<string>', 'exec'))    
+    # Load the module now that there are no conflicts
+    load = modulecmd('load', mod, return_oe=True)
+    exec(compile(load, '<string>', 'exec'))
+
+
 def set_compiler_environment_variables(pkg):
     assert(pkg.spec.concrete)
     compiler = pkg.compiler
@@ -108,11 +130,9 @@ def set_compiler_environment_variables(pkg):
 
     os.environ['SPACK_COMPILER_SPEC']  = str(pkg.spec.compiler)
 
-    if compiler.PrgEnv:
-        os.environ['SPACK_CRAYPE']       = compiler.PrgEnv
-        os.environ['SPACK_COMP_MODULE']  = compiler.module
-
-
+    if compiler.modules:
+        for mod in compiler.modules:
+            load_module(mod)
 
 
 def set_build_environment_variables(pkg):
@@ -163,8 +183,10 @@ def set_build_environment_variables(pkg):
             pcdir = join_path(p, libdir, 'pkgconfig')
             if os.path.isdir(pcdir):
                 pkg_config_dirs.append(pcdir)
-    path_set("PKG_CONFIG_PATH", pkg_config_dirs)
+    path_put_first("PKG_CONFIG_PATH", pkg_config_dirs)
 
+    if pkg.spec.architecture.compiler_strategy.lower() == 'module':
+        load_module(pkg.spec.architecture.module_name)
 
 def set_module_variables_for_package(pkg):
     """Populate the module scope of install() with some useful functions.
@@ -239,8 +261,8 @@ def get_rpaths(pkg):
 
 def setup_package(pkg):
     """Execute all environment setup routines."""
-    set_compiler_environment_variables(pkg)
     set_build_environment_variables(pkg)
+    set_compiler_environment_variables(pkg)
     set_module_variables_for_package(pkg)
 
     # Allow dependencies to set up environment as well.

lib/spack/spack/cmd/compiler.py

@@ -29,6 +29,8 @@
 from llnl.util.tty.colify import colify
 from llnl.util.lang import index_by
 
+import spack.architecture
+import spack.compiler
 import spack.compilers
 import spack.spec
 import spack.config
@@ -38,11 +40,9 @@
 description = "Manage compilers"
 
 def setup_parser(subparser):
-    sp = subparser.add_subparsers(
-        metavar='SUBCOMMAND', dest='compiler_command')
+    sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='compiler_command')
 
-    update_parser = sp.add_parser(
-        'add', help='Add compilers to the Spack configuration.')
+    update_parser = sp.add_parser('add', help='Add compilers to the Spack configuration.')
     update_parser.add_argument('add_paths', nargs=argparse.REMAINDER)
 
     remove_parser = sp.add_parser('remove', help='remove compiler')
@@ -55,14 +55,17 @@ def setup_parser(subparser):
 
 
 def compiler_add(args):
-    """Search either $PATH or a list of paths for compilers and add them
+    """Search either $PATH or a list of paths OR MODULES for compilers and add them
        to Spack's configuration."""
-    paths = args.add_paths
+
+    
+    paths = args.add_paths # This might be a parser method. Parsing method to add_paths 
     if not paths:
         paths = get_path('PATH')
-
+    
     compilers = [c for c in spack.compilers.find_compilers(*args.add_paths)
-                 if c.spec not in spack.compilers.all_compilers()]
+                if c.spec not in spack.compilers.all_compilers()]
+
 
     if compilers:
         spack.compilers.add_compilers_to_config('user', *compilers)

lib/spack/spack/compiler.py

@@ -282,6 +282,11 @@ def find_in_modules(cls):
             modulecmd
             matches = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, output)
 
+            loaded_modules = os.environ["LOADEDMODULES"].split(":")
+            #output = _shell('module avail %s' % cls.PrgEnv_compiler)
+            for module in loaded_modules:
+                match = re.findall(r'(%s)/([^\s(]*)' % cls.PrgEnv_compiler, module)
+
             for name, version in matches:
                 v = version + '-craype'
                 comp = cls(spack.spec.CompilerSpec(name + '@' + v),

lib/spack/spack/concretize.py

@@ -122,6 +122,34 @@ def concretize_architecture(self, spec):
         return True   # changed
 
 
+    def new_concretize_architecture(self, spec):
+        """If the spec already has an architecture and it is a an architecture type,
+        return. Otherwise, if it has an architecture that is a string type, generate an
+        architecture based on that type. If it has no architecture and the root of the
+        DAG has an architecture, then use that. Otherwise, take the system's default
+        architecture.
+        """
+        if spec.architecture is not None:
+            if isinstance(spec.architecture,spack.architecture.Target):
+                return False
+            else:
+                arch = spack.architecture.sys_type()
+                spec.architecture = arch.target(spec.architecture)
+                return True #changed
+
+        if spec.root.architecture:
+            if isinstance(spec.root.architecture,spack.architecture.Target):
+                spec.architecture = spec.root.architecture
+            else:
+                arch = spack.architecture.sys_type()
+                spec.architecture = arch.target(spec.root.architecture)
+        else:
+            arch = spack.architecture.sys_type()
+            spec.architecture = arch.target('default')
+        
+        return True #changed
+
+
     def concretize_variants(self, spec):
         """If the spec already has variants filled in, return.  Otherwise, add
            the default variants from the package specification.

lib/spack/spack/test/__init__.py

@@ -31,7 +31,8 @@
 import spack
 
 """Names of tests to be included in Spack's test suite"""
-test_names = ['versions',
+"""test_names = ['architecture',
+              'versions',
               'url_parse',
               'url_substitution',
               'packages',
@@ -57,7 +58,8 @@
               'optional_deps',
               'make_executable',
               'configure_guess']
-
+"""
+test_names = ['architecture']
 
 def list_tests():
     """Return names of all tests that can be run for Spack."""

lib/spack/spack/test/architecture.py

@@ -0,0 +1,19 @@
+""" Test checks if the architecture class is created correctly and also that
+    the functions are looking for the correct architecture name
+"""
+import unittest
+import spack
+from spack.architecture import *
+
+class ArchitectureTest(unittest.TestCase):
+
+    def test_Architecture_class(self):
+        a = Architecture('Cray-XC40')
+        a.add_arch_strategy()
+        self.assertEquals(a.get_arch_dict(), {'Cray-XC40': 'MODULES'})
+
+    def test_get_sys_type_from_config_file(self):
+        output_arch_class = get_sys_type_from_config_file()
+        my_arch_class = Architecture('Linux x86_64','Cray-xc40')
+
+        self.assertEqual(output_arch_class, my_arch_class)

lib/spack/spack/util/executable.py

@@ -56,7 +56,11 @@ def command(self):
 
     def __call__(self, *args, **kwargs):
         """Run the executable with subprocess.check_output, return output."""
-        return_output = kwargs.get("return_output", False)
+        # Return oe returns a combined stream, setting both output and error 
+        # without setting return oe returns them concatenated by a double line break
+        return_oe    = kwargs.get("return_oe", False)
+        return_output = True if return_oe else kwargs.get("return_output", False)
+        return_error  = True if return_oe else kwargs.get("return_error", False)
         fail_on_error = kwargs.get("fail_on_error", True)
         ignore_errors = kwargs.get("ignore_errors", ())
 
@@ -95,8 +99,8 @@ def streamify(arg, mode):
             proc = subprocess.Popen(
                 cmd,
                 stdin=input,
-                stderr=error,
-                stdout=subprocess.PIPE if return_output else output)
+                stdout=subprocess.PIPE if return_output else output,
+                stderr=subprocess.STDOUT if return_oe else (subprocess.PIPE if return_error else error))
             out, err = proc.communicate()
             self.returncode = proc.returncode
 
@@ -104,8 +108,15 @@ def streamify(arg, mode):
             if fail_on_error and rc != 0 and (rc not in ignore_errors):
                 raise ProcessError("Command exited with status %d:"
                                    % proc.returncode, cmd_line)
-            if return_output:
-                return out
+            # Return out or error if specified. Return combined stream if requested,
+            # otherwise return them concatenated by double line break if both requested.
+            if return_output or return_error:
+                if return_oe or not return_error:
+                    return out
+                elif return_output:
+                    return out+'\n\n'+err
+                else:
+                    return err
 
         except OSError, e:
             raise ProcessError(