Support for cray external dependencies implemented in modules

lib/spack/spack/build_environment.py

@@ -32,10 +32,11 @@
 import shutil
 import multiprocessing
 import platform
+import re
+
 from llnl.util.filesystem import *
 
 import spack
-import spack.compilers as compilers
 from spack.util.executable import Executable, which
 from spack.util.environment import *
 
@@ -108,6 +109,46 @@ def load_module(mod):
     exec(compile(load, '<string>', 'exec'))
 
 
+def get_path_from_module(mod):
+    """Inspects a TCL module for entries that indicate the absolute path
+    at which the library supported by said module can be found.
+    """
+    # Create a modulecmd executable
+    modulecmd = which('modulecmd')
+    modulecmd.add_default_arg('python')
+
+    # Read the module
+    text = modulecmd('show', mod, return_oe=True).split('\n')
+
+    # If it lists its package directory, return that
+    for line in text:
+        if line.find(mod.upper()+'_DIR') >= 0:
+            words = line.split()
+            return words[2]
+
+    # If it lists a -rpath instruction, use that
+    for line in text:
+        rpath = line.find('-rpath/')
+        if rpath >= 0:
+            return line[rpath+6:line.find('/lib')]
+
+    # If it lists a -L instruction, use that
+    for line in text:
+        L = line.find('-L/')
+        if L >= 0:
+            return line[L+2:line.find('/lib')]
+
+    # If it sets the LD_LIBRARY_PATH or CRAY_LD_LIBRARY_PATH, use that
+    for line in text:
+        if line.find('LD_LIBRARY_PATH') >= 0:
+            words = line.split()
+            path = words[2]
+            return path[:path.find('/lib')]
+
+    # Unable to find module path
+    return None
+
+
 def set_compiler_environment_variables(pkg):
     assert(pkg.spec.concrete)
     compiler = pkg.compiler
@@ -251,6 +292,17 @@ def set_module_variables_for_package(pkg):
 
 def get_rpaths(pkg):
     """Get a list of all the rpaths for a package."""
+
+    # First load all modules for external packages and update the external
+    # packages' paths to reflect what is found in the modules so that we can
+    # rpath through the modules when possible, but if not possible they are 
+    # already loaded.
+    for spec in pkg.spec.traverse(root=False):
+        if spec.external_module:
+            load_module(spec.external_module)
+            spec.external = get_path_from_module(spec.external_module)
+
+    # Construct rpaths from the paths of each dep
     rpaths = [pkg.prefix.lib, pkg.prefix.lib64]
     rpaths.extend(d.prefix.lib for d in pkg.spec.traverse(root=False)
                   if os.path.isdir(d.prefix.lib))

lib/spack/spack/concretize.py

@@ -97,7 +97,7 @@ def _valid_virtuals_and_externals(self, spec):
             externals = spec_externals(pkg)
             buildable = not is_spec_nobuild(pkg)
             if buildable:
-                result.append((pkg, None))
+                result.append((pkg, None, None))
             if externals:
                 sorted_externals = sorted(externals, cmp=lambda a,b: a[0].__cmp__(b[0]))
                 for external in sorted_externals:
@@ -131,6 +131,7 @@ def concretize_virtual_and_external(self, spec):
         if not candidate:
             #No ABI matches. Pick the top choice based on the orignal preferences.
             candidate = candidates[0]
+        external_module = candidate[2]
         external = candidate[1]
         candidate_spec = candidate[0]
         
@@ -144,6 +145,9 @@ def concretize_virtual_and_external(self, spec):
         if not spec.external and external:
             spec.external = external
             changed = True
+        if not spec.external_module and external_module:
+            spec.external_module = external_module
+            changed = True
 
         #If we're external then trim the dependencies
         if external and spec.dependencies:

lib/spack/spack/config.py

@@ -103,6 +103,8 @@
 from llnl.util.filesystem import mkdirp
 import copy
 
+from spack.build_environment import get_path_from_module
+
 _config_sections = {}
 class _ConfigCategory:
     name = None
@@ -255,7 +257,8 @@ def get_packages_config():
         package_name = spack.spec.Spec(p.keys()[0]).name
         if package_name not in indexed_packages:
             indexed_packages[package_name] = []
-        indexed_packages[package_name].append({ spack.spec.Spec(key) : val for key, val in p.iteritems() })
+        pkg_dict = dict([ (spack.spec.Spec(key), val) for key, val in p.iteritems()])
+        indexed_packages[package_name].append( pkg_dict )
     return indexed_packages
 
 
@@ -286,9 +289,13 @@ def spec_externals(spec):
             if not pkg.satisfies(spec):
                 continue
             path = conf.get('path', None)
+            module = conf.get('module', None)
             if not path:
-                continue
+                if not module:
-            spec_locations.append( (pkg, path) )
+                    continue
+                else:
+                    path = get_path_from_module(module)
+            spec_locations.append( (pkg, path, module) )
     return spec_locations
         
     

lib/spack/spack/spec.py

@@ -420,6 +420,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
         self._normal = kwargs.get('normal', False)
         self._concrete = kwargs.get('concrete', False)
         self.external = None
+        self.external_module = None
 
         # This allows users to construct a spec DAG with literals.
         # Note that given two specs a and b, Spec(a) copies a, but
@@ -1352,8 +1353,9 @@ def _dup(self, other, **kwargs):
             changed = (self.name != other.name and self.versions != other.versions and \
                        self.architecture != other.architecture and self.compiler != other.compiler and \
                        self.variants != other.variants and self._normal != other._normal and \
-                       self.concrete != other.concrete and self.external != other.external)
+                       self.concrete != other.concrete and self.external != other.external and \
-                                       
+                       self.external_module != other.external_module)
+
         # Local node attributes get copied first.
         self.name = other.name
         self.versions = other.versions.copy()
@@ -1365,6 +1367,7 @@ def _dup(self, other, **kwargs):
         self.variants = other.variants.copy()
         self.variants.spec = self
         self.external = other.external
+        self.external_module = other.external_module
 
         # If we copy dependencies, preserve DAG structure in the new spec
         if kwargs.get('deps', True):
@@ -1383,6 +1386,7 @@ def _dup(self, other, **kwargs):
         self._normal = other._normal
         self._concrete = other._concrete
         self.external = other.external
+        self.external_module = other.external_module
         return changed
 
 
@@ -1809,6 +1813,7 @@ def spec(self):
         spec.architecture = None
         spec.compiler = None
         spec.external = None
+        spec.external_module = None
         spec.dependents   = DependencyMap()
         spec.dependencies = DependencyMap()