Add a mirror module that handles new fetch strategies.

- Uses new fetchers to get source
- Add archive() method to fetch strategies to support this.
- Updated mirror command to use new mirror module

lib/spack/spack/cmd/mirror.py

@@ -23,23 +23,19 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 import os
-import shutil
+import sys
 from datetime import datetime
-from contextlib import closing
 
 from external import argparse
 import llnl.util.tty as tty
 from llnl.util.tty.colify import colify
-from llnl.util.filesystem import mkdirp, join_path
 
 import spack
 import spack.cmd
 import spack.config
+import spack.mirror
 from spack.spec import Spec
 from spack.error import SpackError
-from spack.stage import Stage
-from spack.util.compression import extension
-
 
 description = "Manage mirrors."
 
@@ -105,26 +101,33 @@ def mirror_list(args):
         print fmt % (name, val)
 
 
+def _read_specs_from_file(filename):
+    with closing(open(filename, "r")) as stream:
+        for i, string in enumerate(stream):
+            try:
+                s = Spec(string)
+                s.package
+                args.specs.append(s)
+            except SpackError, e:
+                tty.die("Parse error in %s, line %d:" % (args.file, i+1),
+                        ">>> " + string, str(e))
+
+
 def mirror_create(args):
     """Create a directory to be used as a spack mirror, and fill it with
        package archives."""
     # try to parse specs from the command line first.
-    args.specs = spack.cmd.parse_specs(args.specs)
+    specs = spack.cmd.parse_specs(args.specs)
 
     # If there is a file, parse each line as a spec and add it to the list.
     if args.file:
-        with closing(open(args.file, "r")) as stream:
+        if specs:
-            for i, string in enumerate(stream):
+            tty.die("Cannot pass specs on the command line with --file.")
-                try:
+        specs = _read_specs_from_file(args.file)
-                    s = Spec(string)
-                    s.package
-                    args.specs.append(s)
-                except SpackError, e:
-                    tty.die("Parse error in %s, line %d:" % (args.file, i+1),
-                            ">>> " + string, str(e))
 
-    if not args.specs:
+    # If nothing is passed, use all packages.
-        args.specs = [Spec(n) for n in spack.db.all_package_names()]
+    if not specs:
+        specs = [Spec(n) for n in spack.db.all_package_names()]
 
     # Default name for directory is spack-mirror-<DATESTAMP>
     if not args.directory:
@@ -132,85 +135,23 @@ def mirror_create(args):
         args.directory = 'spack-mirror-' + timestamp
 
     # Make sure nothing is in the way.
+    existed = False
     if os.path.isfile(args.directory):
         tty.error("%s already exists and is a file." % args.directory)
+    elif os.path.isdir(args.directory):
+        existed = True
 
-    # Create a directory if none exists
+    # Actually do the work to create the mirror
-    if not os.path.isdir(args.directory):
+    present, mirrored, error = spack.mirror.create(args.directory, specs)
-        mkdirp(args.directory)
+    p, m, e = len(present), len(mirrored), len(error)
-        tty.msg("Created new mirror in %s" % args.directory)
-    else:
-        tty.msg("Adding to existing mirror in %s" % args.directory)
 
-    # Things to keep track of while parsing specs.
+    verb = "updated" if existed else "created"
-    working_dir = os.getcwd()
+    tty.msg(
-    num_mirrored = 0
+        "Successfully %s mirror in %s." % (verb, args.directory),
-    num_error = 0
+        "Archive stats:",
-
+        "  %-4d already present"  % p,
-    # Iterate through packages and download all the safe tarballs for each of them
+        "  %-4d added"            % m,
-    for spec in args.specs:
+        "  %-4d failed to fetch." % e)
-        pkg = spec.package
-
-        # Skip any package that has no checksummed versions.
-        if not pkg.versions:
-            tty.msg("No safe (checksummed) versions for package %s."
-                    % pkg.name)
-            continue
-
-        # create a subdir for the current package.
-        pkg_path = join_path(args.directory, pkg.name)
-        mkdirp(pkg_path)
-
-        # Download all the tarballs using Stages, then move them into place
-        for version in pkg.versions:
-            # Skip versions that don't match the spec
-            vspec = Spec('%s@%s' % (pkg.name, version))
-            if not vspec.satisfies(spec):
-                continue
-
-            mirror_path = "%s/%s-%s.%s" % (
-                pkg.name, pkg.name, version, extension(pkg.url))
-
-            os.chdir(working_dir)
-            mirror_file = join_path(args.directory, mirror_path)
-            if os.path.exists(mirror_file):
-                tty.msg("Already fetched %s." % mirror_file)
-                num_mirrored += 1
-                continue
-
-            # Get the URL for the version and set up a stage to download it.
-            url = pkg.url_for_version(version)
-            stage = Stage(url)
-            try:
-                # fetch changes directory into the stage
-                stage.fetch()
-
-                if not args.no_checksum and version in pkg.versions:
-                    digest = pkg.versions[version]
-                    stage.check(digest)
-                    tty.msg("Checksum passed for %s@%s" % (pkg.name, version))
-
-                # change back and move the new archive into place.
-                os.chdir(working_dir)
-                shutil.move(stage.archive_file, mirror_file)
-                tty.msg("Added %s to mirror" % mirror_file)
-                num_mirrored += 1
-
-            except Exception, e:
-                 tty.warn("Error while fetching %s." % url, e.message)
-                 num_error += 1
-
-            finally:
-                stage.destroy()
-
-    # If nothing happened, try to say why.
-    if not num_mirrored:
-        if num_error:
-            tty.error("No packages added to mirror.",
-                      "All packages failed to fetch.")
-        else:
-            tty.error("No packages added to mirror. No versions matched specs:")
-            colify(args.specs, indent=4)
 
 
 def mirror(parser, args):
@@ -218,4 +159,5 @@ def mirror(parser, args):
                'add'    : mirror_add,
                'remove' : mirror_remove,
                'list'   : mirror_list }
+
     action[args.mirror_command](args)

lib/spack/spack/fetch_strategy.py

@@ -37,6 +37,8 @@
         Restore original state of downloaded code.  Used by clean commands.
         This may just remove the expanded source and re-expand an archive,
         or it may run something like git reset --hard.
+    * archive()
+        Archive a source directory, e.g. for creating a mirror.
 """
 import os
 import re
@@ -91,6 +93,9 @@ def archive(self, destination): pass  # Used to create tarball for mirror.
     def __str__(self):       # Should be human readable URL.
         return "FetchStrategy.__str___"
 
+    @property
+    def unique_name(self): pass
+
     # This method is used to match fetch strategies to version()
     # arguments in packages.
     @classmethod
@@ -189,7 +194,7 @@ def expand(self):
 
 
     def archive(self, destination):
-        """This archive"""
+        """Just moves this archive to the destination."""
         if not self.archive_file:
             raise NoArchiveFileError("Cannot call archive() before fetching.")
         assert(extension(destination) == extension(self.archive_file))
@@ -231,6 +236,10 @@ def __str__(self):
         else:
             return "URLFetchStrategy<no url>"
 
+    @property
+    def unique_name(self):
+        return "spack-fetch-url:%s" % self
+
 
 class VCSFetchStrategy(FetchStrategy):
     def __init__(self, name, *rev_types, **kwargs):
@@ -384,6 +393,17 @@ def reset(self):
         self.git('clean', '-f')
 
 
+    @property
+    def unique_name(self):
+        name = "spack-fetch-git:%s" % self.url
+        if self.commit:
+            name += "@" + self.commit
+        elif self.branch:
+            name += "@" + self.branch
+        elif self.tag:
+            name += "@" + self.tag
+
+
 class SvnFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a subversion repository.
        Use like this in a package:
@@ -457,6 +477,14 @@ def reset(self):
         self.svn('revert', '.', '-R')
 
 
+    @property
+    def unique_name(self):
+        name = "spack-fetch-svn:%s" % self.url
+        if self.revision:
+            name += "@" + self.revision
+
+
+
 class HgFetchStrategy(VCSFetchStrategy):
     """Fetch strategy that gets source code from a Mercurial repository.
        Use like this in a package:
@@ -532,6 +560,14 @@ def reset(self):
         self.stage.chdir_to_source()
 
 
+    @property
+    def unique_name(self):
+        name = "spack-fetch-hg:%s" % self.url
+        if self.revision:
+            name += "@" + self.revision
+
+
+
 def from_url(url):
     """Given a URL, find an appropriate fetch strategy for it.
        Currently just gives you a URLFetchStrategy that uses curl.
@@ -546,9 +582,18 @@ def args_are_for(args, fetcher):
     fetcher.matches(args)
 
 
-def from_args(args, pkg):
+def for_package_version(pkg, version):
     """Determine a fetch strategy based on the arguments supplied to
        version() in the package description."""
+    # If it's not a known version, extrapolate one.
+    if not version in pkg.versions:
+        url = pkg.url_for_verison(version)
+        if not url:
+            raise InvalidArgsError(pkg, version)
+        return URLFetchStrategy()
+
+    # Grab a dict of args out of the package version dict
+    args = pkg.versions[version]
 
     # Test all strategies against per-version arguments.
     for fetcher in all_strategies:
@@ -564,9 +609,7 @@ def from_args(args, pkg):
         if fetcher.matches(attrs):
             return fetcher(**attrs)
 
-    raise InvalidArgsError(
+    raise InvalidArgsError(pkg, version)
-        "Could not construct fetch strategy for package %s",
-        pkg.spec.format("%_%@"))
 
 
 class FetchStrategyError(spack.error.SpackError):
@@ -593,5 +636,7 @@ def __init__(self, msg, long_msg):
 
 
 class InvalidArgsError(FetchStrategyError):
-    def __init__(self, msg, long_msg):
+    def __init__(self, pkg, version):
-        super(InvalidArgsError, self).__init__(msg, long_msg)
+        msg = "Could not construct a fetch strategy for package %s at version %s"
+        msg %= (pkg.name, version)
+        super(InvalidArgsError, self).__init__(msg)

lib/spack/spack/mirror.py

@@ -0,0 +1,171 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+"""
+This file contains code for creating spack mirror directories.  A
+mirror is an organized hierarchy containing specially named archive
+files.  This enabled spack to know where to find files in a mirror if
+the main server for a particualr package is down.  Or, if the computer
+where spack is run is not connected to the internet, it allows spack
+to download packages directly from a mirror (e.g., on an intranet).
+"""
+import sys
+import os
+import llnl.util.tty as tty
+from llnl.util.filesystem import *
+
+import spack
+import spack.error
+import spack.fetch_strategy as fs
+from spack.spec import Spec
+from spack.stage import Stage
+from spack.version import *
+from spack.util.compression import extension
+
+
+def mirror_archive_filename(spec):
+    """Get the path that this spec will live at within a mirror."""
+    if not spec.version.concrete:
+        raise ValueError("mirror.path requires spec with concrete version.")
+
+    url = spec.package.default_url
+    if url is None:
+        ext = 'tar.gz'
+    else:
+        ext = extension(url)
+
+    return "%s-%s.%s" % (spec.package.name, spec.version, ext)
+
+
+def get_matching_versions(specs):
+    """Get a spec for EACH known version matching any spec in the list."""
+    matching = []
+    for spec in specs:
+        pkg = spec.package
+
+        # Skip any package that has no known versions.
+        if not pkg.versions:
+            tty.msg("No safe (checksummed) versions for package %s." % pkg.name)
+            continue
+
+        for v in reversed(sorted(pkg.versions)):
+            if v.satisfies(spec.versions):
+                s = Spec(pkg.name)
+                s.versions = VersionList([v])
+                matching.append(s)
+    return matching
+
+
+def create(path, specs, **kwargs):
+    """Create a directory to be used as a spack mirror, and fill it with
+       package archives.
+
+       Arguments:
+         path    Path to create a mirror directory hierarchy in.
+         specs   Any package versions matching these specs will be added
+                 to the mirror.
+
+       Return Value:
+         Returns a tuple of lists: (present, mirrored, error)
+         * present:  Package specs that were already prsent.
+         * mirrored: Package specs that were successfully mirrored.
+         * error:    Package specs that failed to mirror due to some error.
+
+       This routine iterates through all known package versions, and
+       it creates specs for those versions.  If the version satisfies any spec
+       in the specs list, it is downloaded and added to the mirror.
+    """
+    # Make sure nothing is in the way.
+    if os.path.isfile(path):
+        raise MirrorError("%s already exists and is a file." % path)
+
+    # automatically spec-ify anything in the specs array.
+    specs = [s if isinstance(s, Spec) else Spec(s) for s in specs]
+
+    # Get concrete specs for each matching version of these specs.
+    version_specs = get_matching_versions(specs)
+    for s in version_specs:
+        s.concretize()
+
+    # Create a directory if none exists
+    if not os.path.isdir(path):
+        mkdirp(path)
+
+    # Things to keep track of while parsing specs.
+    present  = []
+    mirrored = []
+    error    = []
+
+    # Iterate through packages and download all the safe tarballs for each of them
+    for spec in version_specs:
+        pkg = spec.package
+
+        stage = None
+        try:
+            # create a subdirectory for the current package@version
+            realpath = os.path.realpath(path)
+            subdir = join_path(realpath, pkg.name)
+            mkdirp(subdir)
+
+            archive_file = mirror_archive_filename(spec)
+            archive_path = join_path(subdir, archive_file)
+            if os.path.exists(archive_path):
+                present.append(spec)
+                continue
+
+            # Set up a stage and a fetcher for the download
+            fetcher = fs.for_package_version(pkg, pkg.version)
+            stage = Stage(fetcher, name=fetcher.unique_name)
+            fetcher.set_stage(stage)
+
+            # Do the fetch and checksum if necessary
+            fetcher.fetch()
+            if not kwargs.get('no_checksum', False):
+                fetcher.check()
+                tty.msg("Checksum passed for %s@%s" % (pkg.name, pkg.version))
+
+            # Fetchers have to know how to archive their files.  Use
+            # that to move/copy/create an archive in the mirror.
+            fetcher.archive(archive_path)
+            tty.msg("Added %s to mirror" % archive_path)
+            mirrored.append(spec)
+
+        except Exception, e:
+            if spack.debug:
+                sys.excepthook(*sys.exc_info())
+            else:
+                tty.warn("Error while fetching %s." % spec.format('$_$@'), e.message)
+            error.append(spec)
+
+        finally:
+            if stage:
+                stage.destroy()
+
+    return (present, mirrored, error)
+
+
+class MirrorError(spack.error.SpackError):
+    """Superclass of all mirror-creation related errors."""
+    def __init__(self, msg, long_msg=None):
+        super(MirrorError, self).__init__(msg, long_msg)

lib/spack/spack/package.py

@@ -385,8 +385,8 @@ def ensure_has_dict(attr_name):
 
     @property
     def version(self):
-        if not self.spec.concrete:
+        if not self.spec.versions.concrete:
-            raise ValueError("Can only get version of concrete package.")
+            raise ValueError("Can only get of package with concrete version.")
         return self.spec.versions[0]
 
 
@@ -451,18 +451,20 @@ def stage(self):
             raise ValueError("Can only get a stage for a concrete package.")
 
         if self._stage is None:
-            self._stage = Stage(
+            self._stage = Stage(self.fetcher,
-                self.fetcher, mirror_path=self.mirror_path(), name=self.spec.short_spec)
+                                mirror_path=self.mirror_path(),
+                                name=self.spec.short_spec)
         return self._stage
 
 
     @property
     def fetcher(self):
-        if not self.spec.concrete:
+        if not self.spec.versions.concrete:
-            raise ValueError("Can only get a fetcher for a concrete package.")
+            raise ValueError(
+                "Can only get a fetcher for a package with concrete versions.")
 
         if not self._fetcher:
-            self._fetcher = fs.from_args(self.versions[self.version], self)
+            self._fetcher = fs.for_package_version(self, self.version)
         return self._fetcher
 
 
@@ -598,13 +600,14 @@ def url_version(self, version):
 
     @property
     def default_url(self):
-        if self.spec.version.concrete:
+        if self.spec.versions.concrete:
             return self.url_for_version(self.version)
         else:
             url = getattr(self, 'url', None)
             if url:
                 return url
 
+        return None
 
 
     def remove_prefix(self):