Add expand=False option for URL downloads.

- Allows skipping the expand step for downloads.
  - Fixed stage so that it knows expansion didn't fail when there is a
    no-expand URLFetchStrategy.

- Updated docs to reflect new option, and provided an example.

lib/spack/docs/packaging_guide.rst

@@ -401,6 +401,35 @@ construct the new one for ``8.2.1``.
 When you supply a custom URL for a version, Spack uses that URL
 *verbatim* and does not perform extrapolation.
 
+Skipping the expand step
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Spack normally expands archives automatically after downloading
+them. If you want to skip this step (e.g., for self-extracting
+executables and other custom archive types), you can add
+``expand=False`` to a ``version`` directive.
+
+.. code-block:: python
+
+   version('8.2.1', '4136d7b4c04df68b686570afa26988ac',
+           url='http://example.com/foo-8.2.1-special-version.tar.gz', 'expand=False')
+
+When ``expand`` is set to ``False``, Spack sets the current working
+directory to the directory containing the downloaded archive before it
+calls your ``install`` method.  Within ``install``, the path to the
+downloaded archive is available as ``self.stage.archive_file``.
+
+Here is an example snippet for packages distribuetd as self-extracting
+archives.  The example sets permissions on the downloaded file to make
+it executable, then runs it with some arguments.
+
+.. code-block:: python
+
+   def install(self, spec, prefix):
+       set_executable(self.stage.archive_file)
+       installer = Executable(self.stage.archive_file)
+       installer('--prefix=%s' % prefix, 'arg1', 'arg2', 'etc.')
+
 Checksums
 ~~~~~~~~~~~~~~~~~
 

lib/spack/llnl/util/filesystem.py

@@ -25,7 +25,8 @@
 __all__ = ['set_install_permissions', 'install', 'install_tree', 'traverse_tree',
            'expand_user', 'working_dir', 'touch', 'touchp', 'mkdirp',
            'force_remove', 'join_path', 'ancestor', 'can_access', 'filter_file',
-           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink', 'remove_dead_links', 'remove_linked_tree']
+           'FileFilter', 'change_sed_delimiter', 'is_exe', 'force_symlink',
+           'set_executable', 'remove_dead_links', 'remove_linked_tree']
 
 import os
 import sys
@@ -345,6 +346,12 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
     if order == 'post':
         yield (source_path, dest_path)
 
+
+def set_executable(path):
+    st = os.stat(path)
+    os.chmod(path, st.st_mode | stat.S_IEXEC)
+
+
 def remove_dead_links(root):
     """
     Removes any dead link that is present in root

lib/spack/spack/fetch_strategy.py

@@ -82,7 +82,6 @@ class FetchStrategy(object):
 
     class __metaclass__(type):
         """This metaclass registers all fetch strategies in a list."""
-
         def __init__(cls, name, bases, dict):
             type.__init__(cls, name, bases, dict)
             if cls.enabled: all_strategies.append(cls)
@@ -145,6 +144,8 @@ def __init__(self, url=None, digest=None, **kwargs):
         self.digest = kwargs.get('md5', None)
         if not self.digest: self.digest = digest
 
+        self.expand_archive = kwargs.get('expand', True)
+
         if not self.url:
             raise ValueError("URLFetchStrategy requires a url for fetching.")
 
@@ -218,6 +219,10 @@ def archive_file(self):
 
     @_needs_stage
     def expand(self):
+        if not self.expand_archive:
+            tty.msg("Skipping expand step for %s" % self.archive_file)
+            return
+
         tty.msg("Staging archive: %s" % self.archive_file)
 
         self.stage.chdir()

lib/spack/spack/mirror.py

@@ -51,13 +51,20 @@ def mirror_archive_filename(spec, fetcher):
         raise ValueError("mirror.path requires spec with concrete version.")
 
     if isinstance(fetcher, fs.URLFetchStrategy):
-        # If we fetch this version with a URLFetchStrategy, use URL's archive type
-        ext = url.downloaded_file_extension(fetcher.url)
+        if fetcher.expand_archive:
+            # If we fetch this version with a URLFetchStrategy, use URL's archive type
+            ext = url.downloaded_file_extension(fetcher.url)
+        else:
+            # If the archive shouldn't be expanded, don't check for its extension.
+            ext = None
     else:
         # Otherwise we'll make a .tar.gz ourselves
         ext = 'tar.gz'
 
-    return "%s-%s.%s" % (spec.package.name, spec.version, ext)
+    filename = "%s-%s" % (spec.package.name, spec.version)
+    if ext:
+        filename += ".%s" % ext
+    return filename
 
 
 def mirror_archive_path(spec, fetcher):

lib/spack/spack/stage.py

@@ -229,13 +229,22 @@ def archive_file(self):
 
     @property
     def source_path(self):
-        """Returns the path to the expanded/checked out source code
-           within this fetch strategy's path.
+        """Returns the path to the expanded/checked out source code.
 
-           This assumes nothing else is going ot be put in the
-           FetchStrategy's path.  It searches for the first
-           subdirectory of the path it can find, then returns that.
+        To find the source code, this method searches for the first
+        subdirectory of the stage that it can find, and returns it.
+        This assumes nothing besides the archive file will be in the
+        stage path, but it has the advantage that we don't need to
+        know the name of the archive or its contents.
+
+        If the fetch strategy is not supposed to expand the downloaded
+        file, it will just return the stage path. If the archive needs
+        to be expanded, it will return None when no archive is found.
         """
+        if isinstance(self.fetcher, fs.URLFetchStrategy):
+            if not self.fetcher.expand_archive:
+                return self.path
+
         for p in [os.path.join(self.path, f) for f in os.listdir(self.path)]:
             if os.path.isdir(p):
                 return p
@@ -416,21 +425,15 @@ def expand_archive(self):
                 shutil.move(source_path, destination_path)
 
 
-@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive', 'restage', 'destroy'])
+@pattern.composite(method_list=['fetch', 'create', 'check', 'expand_archive',  'restage', 'destroy'])
 class StageComposite:
     """
     Composite for Stage type objects. The first item in this composite is considered to be the root package, and
     operations that return a value are forwarded to it.
     """
-
-    @property
-    def source_path(self):
-        return self[0].source_path
-
-    @property
-    def path(self):
-        return self[0].path
-
+    #
+    # __enter__ and __exit__ delegate to all stages in the composite.
+    #
     def __enter__(self):
         for item in self:
             item.__enter__()
@@ -440,9 +443,24 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         for item in reversed(self):
             item.__exit__(exc_type, exc_val, exc_tb)
 
+    #
+    # Below functions act only on the *first* stage in the composite.
+    #
+    @property
+    def source_path(self):
+        return self[0].source_path
+
+    @property
+    def path(self):
+        return self[0].path
+
     def chdir_to_source(self):
         return self[0].chdir_to_source()
 
+    @property
+    def archive_file(self):
+        return self[0].archive_file
+
 
 class DIYStage(object):
     """Simple class that allows any directory to be a spack stage."""