Ability to list versions from web page with spack list -v PACKAGE

Experimental feature automatically parses versions out of web pages and prints what
it thinks are avaialble versions of a package, e.g.:

    $ spack list -v libunwind
    1.1    1.0   0.98.6  0.98.4  0.98.2  0.98  0.96  0.93  0.91  0.1
    1.0.1  0.99  0.98.5  0.98.3  0.98.1  0.97  0.95  0.92  0.9   0.0

lib/spack/spack/cmd/list.py

@@ -1,10 +1,19 @@
+import os
+import re
+
 import spack
 import spack.packages as packages
+from spack.version import ver
 from spack.colify import colify
+import spack.url as url
+import spack.tty as tty
+
 
 description ="List spack packages"
 
 def setup_parser(subparser):
+    subparser.add_argument('-v', '--versions', metavar="PACKAGE", dest='version_package',
+                           help='List available versions of a package (experimental).')
     subparser.add_argument('-i', '--installed', action='store_true', dest='installed',
                            help='List installed packages for each platform along with versions.')
 
@@ -16,8 +25,33 @@ def list(parser, args):
             print "%s:" % sys_type
             package_vers = []
             for pkg in pkgs[sys_type]:
-                pv = [pkg.name + "/" + v for v in pkg.installed_versions]
+                pv = [pkg.name + "@" + v for v in pkg.installed_versions]
                 package_vers.extend(pv)
             colify(sorted(package_vers), indent=4)
+
+    elif args.version_package:
+        pkg = packages.get(args.version_package)
+
+        try:
+            # Run curl but grab the mime type from the http headers
+            listing = spack.curl('-s', '-L', pkg.list_url, return_output=True)
+            url_regex = os.path.basename(url.wildcard_version(pkg.url))
+            strings = re.findall(url_regex, listing)
+
+            versions = []
+            wildcard = pkg.version.wildcard()
+            for s in strings:
+                match = re.search(wildcard, s)
+                if match:
+                    versions.append(ver(match.group(0)))
+
+            colify(str(v) for v in reversed(sorted(set(versions))))
+
+        except:
+            tty.die("Listing versions for %s failed" % pkg.name,
+                    "Listing versions is experimental.  You may need to add the list_url",
+                    "attribute to the package to tell Spack where to look for versions.")
+            raise
+
     else:
         colify(packages.all_package_names())

lib/spack/spack/colify.py

@@ -1,5 +1,3 @@
-#!/usr/bin/env python
-#
 # colify
 # By Todd Gamblin, tgamblin@llnl.gov
 #
@@ -100,6 +98,11 @@ def colify(elts, **options):
     if not type(elts) == list:
         elts = list(elts)
 
+    if not output.isatty():
+        for elt in elts:
+            output.write("%s\n" % elt)
+        return
+
     console_cols = options.get("cols", None)
     if not console_cols:
         console_cols, console_rows = get_terminal_size()

lib/spack/spack/package.py

@@ -25,6 +25,8 @@
 import url
 import arch
 
+from spec import Compiler
+from version import Version
 from multi_function import platform
 from stage import Stage
 from dependency import *
@@ -241,6 +243,11 @@ class SomePackage(Package):
     """Controls whether install and uninstall check deps before running."""
     ignore_dependencies = False
 
+    # TODO: multi-compiler support
+    """Default compiler for this package"""
+    compiler = Compiler('gcc')
+
+
     def __init__(self, sys_type = arch.sys_type()):
         # Check for attributes that derived classes must set.
         attr.required(self, 'homepage')
@@ -261,10 +268,14 @@ def __init__(self, sys_type = arch.sys_type()):
         validate.url(self.url)
 
         # Set up version
-        attr.setdefault(self, 'version', url.parse_version(self.url))
+        if not hasattr(self, 'version'):
-        if not self.version:
+            try:
-            tty.die("Couldn't extract version from %s. " +
+                self.version = url.parse_version(self.url)
-                    "You must specify it explicitly for this URL." % self.url)
+            except UndetectableVersionError:
+                tty.die("Couldn't extract a default version from %s. You " +
+                        "must specify it explicitly in the package." % self.url)
+        elif type(self.version) == string:
+            self.version = Version(self.version)
 
         # This adds a bunch of convenience commands to the package's module scope.
         self.add_commands_to_module()
@@ -275,6 +286,10 @@ def __init__(self, sys_type = arch.sys_type()):
         # stage used to build this package.
         self.stage = Stage(self.stage_name, self.url)
 
+        # Set a default list URL (place to find lots of versions)
+        if not hasattr(self, 'list_url'):
+            self.list_url = os.path.dirname(self.url)
+
 
     def add_commands_to_module(self):
         """Populate the module scope of install() with some useful functions.
@@ -395,6 +410,16 @@ def prefix(self):
         return new_path(self.package_path, self.version)
 
 
+    def url_version(self, version):
+        """Given a version, this returns a string that should be substituted into the
+           package's URL to download that version.
+           By default, this just returns the version string. Subclasses may need to
+           override this, e.g. for boost versions where you need to ensure that there
+           are _'s in the download URL.
+        """
+        return version.string
+
+
     def remove_prefix(self):
         """Removes the prefix for a package along with any empty parent directories."""
         if self.dirty:

lib/spack/spack/packages/__init__.py

@@ -6,11 +6,10 @@
 import glob
 
 import spack
+import spack.error
 from spack.utils import *
 import spack.arch as arch
-import spack.version as version
+
-import spack.attr as attr
-import spack.error as serr
 
 # Valid package names -- can contain - but can't start with it.
 valid_package = r'^\w[\w-]*$'
@@ -20,7 +19,16 @@
 
 instances = {}
 
-class InvalidPackageNameError(serr.SpackError):
+
+def get(pkg, arch=arch.sys_type()):
+    key = (pkg, arch)
+    if not key in instances:
+        package_class = get_class(pkg)
+        instances[key] = package_class(arch)
+    return instances[key]
+
+
+class InvalidPackageNameError(spack.error.SpackError):
     """Raised when we encounter a bad package name."""
     def __init__(self, name):
         super(InvalidPackageNameError, self).__init__(
@@ -34,7 +42,7 @@ def valid_name(pkg):
 
 def validate_name(pkg):
     if not valid_name(pkg):
-        raise spack.InvalidPackageNameError(pkg)
+        raise InvalidPackageNameError(pkg)
 
 
 def filename_for(pkg):
@@ -45,8 +53,6 @@ def filename_for(pkg):
 
 def installed_packages(**kwargs):
     """Returns a dict from systype strings to lists of Package objects."""
-    list_installed = kwargs.get('installed', False)
-
     pkgs = {}
     if not os.path.isdir(spack.install_path):
         return pkgs
@@ -108,14 +114,6 @@ def get_class(pkg):
     return klass
 
 
-def get(pkg, arch=arch.sys_type()):
-    key = (pkg, arch)
-    if not key in instances:
-        package_class = get_class(pkg)
-        instances[key] = package_class(arch)
-    return instances[key]
-
-
 def compute_dependents():
     """Reads in all package files and sets dependence information on
        Package objects in memory.

lib/spack/spack/packages/libdwarf.py

@@ -9,6 +9,8 @@ class Libdwarf(Package):
     url      = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz"
     md5      = "64b42692e947d5180e162e46c689dfbf"
 
+    list_url = "http://reality.sgiweb.org/davea/dwarf.html"
+
     depends_on("libelf")
 
 

lib/spack/spack/url.py

@@ -163,9 +163,19 @@ def parse_name_and_version(path):
     return (name, ver)
 
 
-def version_format(path):
+def substitute_version(path, new_version):
-    """Given a URL or archive name, find the version and create a format string
+    """Given a URL or archive name, find the version in the path and substitute
-       that will allow another version to be substituted.
+       the new version for it.
     """
     ver, start, end = parse_version_string_with_indices(path)
-    return path[:start] + '%s' + path[end:]
+    return path[:start] + new_version + path[end:]
+
+
+def wildcard_version(path):
+    """Find the version in the supplied path, and return a regular expression
+       that will match this path with any version in its place.
+    """
+    ver, start, end = parse_version_string_with_indices(path)
+    v = Version(ver)
+
+    return re.escape(path[:start]) + v.wildcard() + re.escape(path[end:])

lib/spack/spack/version.py

@@ -41,6 +41,10 @@ def __init__(self, string):
         segments = re.findall(segment_regex, string)
         self.version = tuple(int_if_int(seg) for seg in segments)
 
+        # Store the separators from the original version string as well.
+        # last element of separators is ''
+        self.separators = tuple(re.split(segment_regex, string)[1:-1])
+
 
     def up_to(self, index):
         """Return a version string up to the specified component, exclusive.
@@ -48,6 +52,29 @@ def up_to(self, index):
         """
         return '.'.join(str(x) for x in self[:index])
 
+    def wildcard(self):
+        """Create a regex that will match variants of this version string."""
+        def a_or_n(seg):
+            if type(seg) == int:
+                return r'[0-9]+'
+            else:
+                return r'[a-zA-Z]+'
+
+        version = self.version
+        separators = ('',) + self.separators
+
+        version += (version[-1],) * 2
+        separators += (separators[-1],) * 2
+
+        sep_res = [re.escape(sep) for sep in separators]
+        seg_res = [a_or_n(seg) for seg in version]
+
+        wc = seg_res[0]
+        for i in xrange(1, len(sep_res)):
+            wc += '(?:' + sep_res[i] + seg_res[i]
+        wc += ')?' * (len(seg_res) - 1)
+        return wc
+
     def __iter__(self):
         for v in self.version:
             yield v
@@ -96,13 +123,16 @@ def __lt__(self, other):
 
     def __eq__(self, other):
         """Implemented to match __lt__.  See __lt__."""
-        if type(other) == VersionRange:
+        if type(other) != Version:
             return False
         return self.version == other.version
 
     def __ne__(self, other):
         return not (self == other)
 
+    def __hash__(self):
+        return hash(self.version)
+
 
 @total_ordering
 class VersionRange(object):