Added feature: package extensions

- packages can be "extended" by others
- allows extension to be symlinked into extendee's prefix.
- used for python modules.
  - first module: py-setuptools

lib/spack/llnl/util/filesystem.py

@@ -24,7 +24,8 @@
 ##############################################################################
 __all__ = ['set_install_permissions', 'install', 'expand_user', 'working_dir',
            'touch', 'mkdirp', 'force_remove', 'join_path', 'ancestor',
-           'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe']
+           'can_access', 'filter_file', 'change_sed_delimiter', 'is_exe',
+           'check_link_tree', 'merge_link_tree', 'unmerge_link_tree']
 
 import os
 import sys
@@ -222,3 +223,82 @@ def ancestor(dir, n=1):
 def can_access(file_name):
     """True if we have read/write access to the file."""
     return os.access(file_name, os.R_OK|os.W_OK)
+
+
+def traverse_link_tree(src_root, dest_root, follow_nonexisting=True, **kwargs):
+    # Yield directories before or after their contents.
+    order  = kwargs.get('order', 'pre')
+    if order not in ('pre', 'post'):
+        raise ValueError("Order must be 'pre' or 'post'.")
+
+    # List of relative paths to ignore under the src root.
+    ignore = kwargs.get('ignore', None)
+    if isinstance(ignore, basestring):
+        ignore = (ignore,)
+
+    for dirpath, dirnames, filenames in os.walk(src_root):
+        rel_path  = dirpath[len(src_root):]
+        rel_path = rel_path.lstrip(os.path.sep)
+        dest_dirpath = os.path.join(dest_root, rel_path)
+
+        # Don't descend into ignored directories
+        if ignore and dest_dirpath in ignore:
+            return
+
+        # Don't descend into dirs in dest that do not exist in src.
+        if not follow_nonexisting:
+            dirnames[:] = [
+                d for d in dirnames
+                if os.path.exists(os.path.join(dest_dirpath, d))]
+
+        # preorder yields directories before children
+        if order == 'pre':
+            yield (dirpath, dest_dirpath)
+
+        for name in filenames:
+            src_file  = os.path.join(dirpath, name)
+            dest_file = os.path.join(dest_dirpath, name)
+
+            # Ignore particular paths inside the install root.
+            src_relpath = src_file[len(src_root):]
+            src_relpath = src_relpath.lstrip(os.path.sep)
+            if ignore and src_relpath in ignore:
+                continue
+
+            yield (src_file, dest_file)
+
+        # postorder yields directories after children
+        if order == 'post':
+            yield (dirpath, dest_dirpath)
+
+
+
+def check_link_tree(src_root, dest_root, **kwargs):
+    for src, dest in traverse_link_tree(src_root, dest_root, False, **kwargs):
+        if os.path.exists(dest) and not os.path.isdir(dest):
+            return dest
+    return None
+
+
+def merge_link_tree(src_root, dest_root, **kwargs):
+    kwargs['order'] = 'pre'
+    for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
+        if os.path.isdir(src):
+            mkdirp(dest)
+        else:
+            assert(not os.path.exists(dest))
+            os.symlink(src, dest)
+
+
+def unmerge_link_tree(src_root, dest_root, **kwargs):
+    kwargs['order'] = 'post'
+    for src, dest in traverse_link_tree(src_root, dest_root, **kwargs):
+        if os.path.isdir(dest):
+            if not os.listdir(dest):
+                # TODO: what if empty directories were present pre-merge?
+                shutil.rmtree(dest, ignore_errors=True)
+
+        elif os.path.exists(dest):
+            if not os.path.islink(dest):
+                raise ValueError("%s is not a link tree!" % dest)
+            os.remove(dest)

lib/spack/spack/__init__.py

@@ -138,7 +138,7 @@
 #       should live.  This file is overloaded for spack core vs. for packages.
 #
 __all__ = ['Package', 'Version', 'when', 'ver']
-from spack.package import Package
+from spack.package import Package, ExtensionConflictError
 from spack.version import Version, ver
 from spack.multimethod import when
 

lib/spack/spack/directory_layout.py

@@ -53,6 +53,19 @@ def __init__(self, root):
         self.root = root
 
 
+    @property
+    def hidden_file_paths(self):
+        """Return a list of hidden files used by the directory layout.
+
+        Paths are relative to the root of an install directory.
+
+        If the directory layout uses no hidden files to maintain
+        state, this should return an empty container, e.g. [] or (,).
+
+        """
+        raise NotImplementedError()
+
+
     def all_specs(self):
         """To be implemented by subclasses to traverse all specs for which there is
            a directory within the root.
@@ -156,6 +169,11 @@ def __init__(self, root, **kwargs):
         self.extension_file_name = extension_file_name
 
 
+    @property
+    def hidden_file_paths(self):
+        return ('.spec', '.extensions')
+
+
     def relative_path_for_spec(self, spec):
         _check_concrete(spec)
         dir_name = spec.format('$_$@$+$#')
@@ -249,28 +267,32 @@ def extension_file_path(self, spec):
 
 
     def get_extensions(self, spec):
-        path = self.extension_file_path(spec)
+        _check_concrete(spec)
 
+        path = self.extension_file_path(spec)
         extensions = set()
         if os.path.exists(path):
-            with closing(open(path)) as spec_file:
+            with closing(open(path)) as ext_file:
-                for line in spec_file:
+                for line in ext_file:
                     try:
-                        extensions.add(Spec(line))
+                        extensions.add(Spec(line.strip()))
-                    except SpecError, e:
+                    except spack.error.SpackError, e:
                         raise InvalidExtensionSpecError(str(e))
         return extensions
 
 
-    def write_extensions(self, extensions):
+    def write_extensions(self, spec, extensions):
         path = self.extension_file_path(spec)
         with closing(open(path, 'w')) as spec_file:
             for extension in sorted(extensions):
-                spec_file.write("%s\n" % extensions)
+                spec_file.write("%s\n" % extension)
 
 
     def add_extension(self, spec, extension_spec):
-        exts = get_extensions(spec)
+        _check_concrete(spec)
+        _check_concrete(extension_spec)
+
+        exts = self.get_extensions(spec)
         if extension_spec in exts:
             raise ExtensionAlreadyInstalledError(spec, extension_spec)
         else:
@@ -279,16 +301,19 @@ def add_extension(self, spec, extension_spec):
                     raise ExtensionConflictError(spec, extension_spec, already_installed)
 
         exts.add(extension_spec)
-        self.write_extensions(exts)
+        self.write_extensions(spec, exts)
 
 
     def remove_extension(self, spec, extension_spec):
-        exts = get_extensions(spec)
+        _check_concrete(spec)
+        _check_concrete(extension_spec)
+
+        exts = self.get_extensions(spec)
         if not extension_spec in exts:
             raise NoSuchExtensionError(spec, extension_spec)
 
         exts.remove(extension_spec)
-        self.write_extensions(exts)
+        self.write_extensions(spec, exts)
 
 
 class DirectoryLayoutError(SpackError):
@@ -328,7 +353,7 @@ class ExtensionAlreadyInstalledError(DirectoryLayoutError):
     """Raised when an extension is added to a package that already has it."""
     def __init__(self, spec, extension_spec):
         super(ExtensionAlreadyInstalledError, self).__init__(
-            "%s is already installed in %s" % (extension_spec, spec))
+            "%s is already installed in %s" % (extension_spec.short_spec, spec.short_spec))
 
 
 class ExtensionConflictError(DirectoryLayoutError):
@@ -336,12 +361,12 @@ class ExtensionConflictError(DirectoryLayoutError):
     def __init__(self, spec, extension_spec, conflict):
         super(ExtensionConflictError, self).__init__(
             "%s cannot be installed in %s because it conflicts with %s."% (
-                extension_spec, spec, conflict))
+                extension_spec.short_spec, spec.short_spec, conflict.short_spec))
 
 
 class NoSuchExtensionError(DirectoryLayoutError):
     """Raised when an extension isn't there on remove."""
     def __init__(self, spec, extension_spec):
         super(NoSuchExtensionError, self).__init__(
-            "%s cannot be removed from %s beacuse it's not installed."% (
+            "%s cannot be removed from %s because it's not installed."% (
-                extension_spec, spec, conflict))
+                extension_spec.short_spec, spec.short_spec))

lib/spack/spack/hooks/extensions.py

@@ -0,0 +1,49 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import spack
+
+
+def post_install(pkg):
+    assert(pkg.spec.concrete)
+    for name, spec in pkg.extendees.items():
+        ext = pkg.spec[name]
+        epkg = ext.package
+        if epkg.installed:
+            epkg.do_activate(pkg)
+
+
+def pre_uninstall(pkg):
+    assert(pkg.spec.concrete)
+
+    # Need to do this b/c uninstall does not automatically do it.
+    # TODO: store full graph info in stored .spec file.
+    pkg.spec.normalize()
+
+    for name, spec in pkg.extendees.items():
+        ext = pkg.spec[name]
+        epkg = ext.package
+        if epkg.installed:
+            epkg.do_deactivate(pkg)

lib/spack/spack/package.py

@@ -329,6 +329,9 @@ class SomePackage(Package):
     """By default we build in parallel.  Subclasses can override this."""
     parallel = True
 
+    """Most packages are NOT extendable.  Set to True if you want extensions."""
+    extendable = False
+
 
     def __init__(self, spec):
         # this determines how the package should be built.
@@ -398,6 +401,9 @@ def ensure_has_dict(attr_name):
         self._fetch_time = 0.0
         self._total_time = 0.0
 
+        for name, spec in self.extendees.items():
+            spack.db.get(spec)._check_extendable()
+
 
     @property
     def version(self):
@@ -877,6 +883,79 @@ def do_uninstall(self, **kwargs):
         spack.hooks.post_uninstall(self)
 
 
+    def _check_extendable(self):
+        if not self.extendable:
+            raise ValueError("Package %s is not extendable!" % self.name)
+
+
+    def _sanity_check_extension(self, extension):
+        self._check_extendable()
+        if not self.installed:
+            raise ValueError("Can only (de)activate extensions for installed packages.")
+        if not extension.installed:
+            raise ValueError("Extensions must first be installed.")
+        if not self.name in extension.extendees:
+            raise ValueError("%s does not extend %s!" % (extension.name, self.name))
+        if not self.spec.satisfies(extension.extendees[self.name]):
+            raise ValueError("%s does not satisfy %s!" % (self.spec, extension.spec))
+
+
+    def do_activate(self, extension):
+        self._sanity_check_extension(extension)
+
+        self.activate(extension)
+        spack.install_layout.add_extension(self.spec, extension.spec)
+        tty.msg("Activated extension %s for %s."
+                % (extension.spec.short_spec, self.spec.short_spec))
+
+
+    def activate(self, extension):
+        """Symlinks all files from the extension into extendee's install dir.
+
+        Package authors can override this method to support other
+        extension mechanisms.  Spack internals (commands, hooks, etc.)
+        should call do_activate() method so that proper checks are
+        always executed.
+
+        """
+        conflict = check_link_tree(
+            extension.prefix, self.prefix,
+            ignore=spack.install_layout.hidden_file_paths)
+
+        if conflict:
+            raise ExtensionConflictError(conflict)
+
+        merge_link_tree(extension.prefix, self.prefix,
+                        ignore=spack.install_layout.hidden_file_paths)
+
+
+    def do_deactivate(self, extension):
+        self._sanity_check_extension(extension)
+        self.deactivate(extension)
+
+        ext = extension.spec
+        if ext in spack.install_layout.get_extensions(self.spec):
+            spack.install_layout.remove_extension(self.spec, ext)
+
+        tty.msg("Deactivated extension %s for %s."
+                % (extension.spec.short_spec, self.spec.short_spec))
+
+
+    def deactivate(self, extension):
+        """Unlinks all files from extension out of extendee's install dir.
+
+        Package authors can override this method to support other
+        extension mechanisms.  Spack internals (commands, hooks, etc.)
+        should call do_deactivate() method so that proper checks are
+        always executed.
+
+        """
+        unmerge_link_tree(extension.prefix, self.prefix,
+                          ignore=spack.install_layout.hidden_file_paths)
+        tty.msg("Deactivated %s as extension of %s."
+                % (extension.spec.short_spec, self.spec.short_spec))
+
+
     def do_clean(self):
         if self.stage.expanded_archive_path:
             self.stage.chdir_to_source()
@@ -1068,3 +1147,9 @@ class NoURLError(PackageError):
     def __init__(self, cls):
         super(NoURLError, self).__init__(
             "Package %s has no version with a URL." % cls.__name__)
+
+
+class ExtensionConflictError(PackageError):
+    def __init__(self, path):
+        super(ExtensionConflictError, self).__init__(
+            "Extension blocked by file: %s" % path)

lib/spack/spack/relations.py

@@ -68,7 +68,7 @@ class Mpileaks(Package):
         spack install mpileaks ^mvapich
         spack install mpileaks ^mpich
 """
-__all__ = [ 'depends_on', 'provides', 'patch', 'version' ]
+__all__ = [ 'depends_on', 'extends', 'provides', 'patch', 'version' ]
 
 import re
 import inspect
@@ -135,7 +135,7 @@ def extends(*specs):
     for string in specs:
         for spec in spack.spec.parse(string):
             if pkg == spec.name:
-                raise CircularReferenceError('depends_on', pkg)
+                raise CircularReferenceError('extends', pkg)
             dependencies[spec.name] = spec
             extendees[spec.name] = spec
 

var/spack/packages/python/package.py

@@ -1,10 +1,13 @@
 from spack import *
 
+
 class Python(Package):
     """The Python programming language."""
     homepage = "http://www.python.org"
     url      = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tar.xz"
 
+    extendable = True
+
     version('2.7.8', 'd235bdfa75b8396942e360a70487ee00')
 
     depends_on("openssl")