zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ability to list versions from web page with spack list -v PACKAGE

Browse Source 
Experimental feature automatically parses versions out of web pages and prints what
it thinks are avaialble versions of a package, e.g.:

    $ spack list -v libunwind
    1.1    1.0   0.98.6  0.98.4  0.98.2  0.98  0.96  0.93  0.91  0.1
    1.0.1  0.99  0.98.5  0.98.3  0.98.1  0.97  0.95  0.92  0.9   0.0
This commit is contained in:
Todd Gamblin
2013-05-17 16:25:00 -07:00
parent 6e557798e8
commit 57ef3b8a80
7 changed files with 129 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
lib/spack/spack/cmd/list.py
View File

@@ -1,10 +1,19 @@
import os
import re
import spack
import spack.packages as packages
from spack.version import ver
from spack.colify import colify
import spack.url as url
import spack.tty as tty
description ="List spack packages"
def setup_parser(subparser):
subparser.add_argument('-v', '--versions', metavar="PACKAGE", dest='version_package',
help='List available versions of a package (experimental).')
subparser.add_argument('-i', '--installed', action='store_true', dest='installed',
help='List installed packages for each platform along with versions.')
@@ -16,8 +25,33 @@ def list(parser, args):
print "%s:" % sys_type
package_vers = []
for pkg in pkgs[sys_type]:
pv = [pkg.name + "/" + v for v in pkg.installed_versions]
pv = [pkg.name + "@" + v for v in pkg.installed_versions]
package_vers.extend(pv)
colify(sorted(package_vers), indent=4)
elif args.version_package:
pkg = packages.get(args.version_package)
try:
# Run curl but grab the mime type from the http headers
listing = spack.curl('-s', '-L', pkg.list_url, return_output=True)
url_regex = os.path.basename(url.wildcard_version(pkg.url))
strings = re.findall(url_regex, listing)
versions = []
wildcard = pkg.version.wildcard()
for s in strings:
match = re.search(wildcard, s)
if match:
versions.append(ver(match.group(0)))
colify(str(v) for v in reversed(sorted(set(versions))))
except:
tty.die("Listing versions for %s failed" % pkg.name,
"Listing versions is experimental. You may need to add the list_url",
"attribute to the package to tell Spack where to look for versions.")
raise
else:
colify(packages.all_package_names())

7
lib/spack/spack/colify.py
View File

@@ -1,5 +1,3 @@
#!/usr/bin/env python
#
# colify
# By Todd Gamblin, tgamblin@llnl.gov
#
@@ -100,6 +98,11 @@ def colify(elts, **options):
if not type(elts) == list:
elts = list(elts)
if not output.isatty():
for elt in elts:
output.write("%s\n" % elt)
return
console_cols = options.get("cols", None)
if not console_cols:
console_cols, console_rows = get_terminal_size()

33
lib/spack/spack/package.py
View File

@@ -25,6 +25,8 @@
import url
import arch
from spec import Compiler
from version import Version
from multi_function import platform
from stage import Stage
from dependency import *
@@ -241,6 +243,11 @@ class SomePackage(Package):
"""Controls whether install and uninstall check deps before running."""
ignore_dependencies = False
# TODO: multi-compiler support
"""Default compiler for this package"""
compiler = Compiler('gcc')
def __init__(self, sys_type = arch.sys_type()):
# Check for attributes that derived classes must set.
attr.required(self, 'homepage')
@@ -261,10 +268,14 @@ def __init__(self, sys_type = arch.sys_type()):
validate.url(self.url)
# Set up version
attr.setdefault(self, 'version', url.parse_version(self.url))
if not self.version:
tty.die("Couldn't extract version from %s. " +
"You must specify it explicitly for this URL." % self.url)
if not hasattr(self, 'version'):
try:
self.version = url.parse_version(self.url)
except UndetectableVersionError:
tty.die("Couldn't extract a default version from %s. You " +
"must specify it explicitly in the package." % self.url)
elif type(self.version) == string:
self.version = Version(self.version)
# This adds a bunch of convenience commands to the package's module scope.
self.add_commands_to_module()
@@ -275,6 +286,10 @@ def __init__(self, sys_type = arch.sys_type()):
# stage used to build this package.
self.stage = Stage(self.stage_name, self.url)
# Set a default list URL (place to find lots of versions)
if not hasattr(self, 'list_url'):
self.list_url = os.path.dirname(self.url)
def add_commands_to_module(self):
"""Populate the module scope of install() with some useful functions.
@@ -395,6 +410,16 @@ def prefix(self):
return new_path(self.package_path, self.version)
def url_version(self, version):
"""Given a version, this returns a string that should be substituted into the
package's URL to download that version.
By default, this just returns the version string. Subclasses may need to
override this, e.g. for boost versions where you need to ensure that there
are _'s in the download URL.
"""
return version.string
def remove_prefix(self):
"""Removes the prefix for a package along with any empty parent directories."""
if self.dirty:

28
lib/spack/spack/packages/__init__.py
View File

@@ -6,11 +6,10 @@
import glob
import spack
import spack.error
from spack.utils import *
import spack.arch as arch
import spack.version as version
import spack.attr as attr
import spack.error as serr
# Valid package names -- can contain - but can't start with it.
valid_package = r'^\w[\w-]*$'
@@ -20,7 +19,16 @@
instances = {}
class InvalidPackageNameError(serr.SpackError):
def get(pkg, arch=arch.sys_type()):
key = (pkg, arch)
if not key in instances:
package_class = get_class(pkg)
instances[key] = package_class(arch)
return instances[key]
class InvalidPackageNameError(spack.error.SpackError):
"""Raised when we encounter a bad package name."""
def __init__(self, name):
super(InvalidPackageNameError, self).__init__(
@@ -34,7 +42,7 @@ def valid_name(pkg):
def validate_name(pkg):
if not valid_name(pkg):
raise spack.InvalidPackageNameError(pkg)
raise InvalidPackageNameError(pkg)
def filename_for(pkg):
@@ -45,8 +53,6 @@ def filename_for(pkg):
def installed_packages(**kwargs):
"""Returns a dict from systype strings to lists of Package objects."""
list_installed = kwargs.get('installed', False)
pkgs = {}
if not os.path.isdir(spack.install_path):
return pkgs
@@ -108,14 +114,6 @@ def get_class(pkg):
return klass
def get(pkg, arch=arch.sys_type()):
key = (pkg, arch)
if not key in instances:
package_class = get_class(pkg)
instances[key] = package_class(arch)
return instances[key]
def compute_dependents():
"""Reads in all package files and sets dependence information on
Package objects in memory.

2
lib/spack/spack/packages/libdwarf.py
View File

@@ -9,6 +9,8 @@ class Libdwarf(Package):
url = "http://reality.sgiweb.org/davea/libdwarf-20130207.tar.gz"
md5 = "64b42692e947d5180e162e46c689dfbf"
list_url = "http://reality.sgiweb.org/davea/dwarf.html"
depends_on("libelf")

18
lib/spack/spack/url.py
View File

@@ -163,9 +163,19 @@ def parse_name_and_version(path):
return (name, ver)
def version_format(path):
"""Given a URL or archive name, find the version and create a format string
that will allow another version to be substituted.
def substitute_version(path, new_version):
"""Given a URL or archive name, find the version in the path and substitute
the new version for it.
"""
ver, start, end = parse_version_string_with_indices(path)
return path[:start] + '%s' + path[end:]
return path[:start] + new_version + path[end:]
def wildcard_version(path):
"""Find the version in the supplied path, and return a regular expression
that will match this path with any version in its place.
"""
ver, start, end = parse_version_string_with_indices(path)
v = Version(ver)
return re.escape(path[:start]) + v.wildcard() + re.escape(path[end:])

32
lib/spack/spack/version.py
View File

@@ -41,6 +41,10 @@ def __init__(self, string):
segments = re.findall(segment_regex, string)
self.version = tuple(int_if_int(seg) for seg in segments)
# Store the separators from the original version string as well.
# last element of separators is ''
self.separators = tuple(re.split(segment_regex, string)[1:-1])
def up_to(self, index):
"""Return a version string up to the specified component, exclusive.
@@ -48,6 +52,29 @@ def up_to(self, index):
"""
return '.'.join(str(x) for x in self[:index])
def wildcard(self):
"""Create a regex that will match variants of this version string."""
def a_or_n(seg):
if type(seg) == int:
return r'[0-9]+'
else:
return r'[a-zA-Z]+'
version = self.version
separators = ('',) + self.separators
version += (version[-1],) * 2
separators += (separators[-1],) * 2
sep_res = [re.escape(sep) for sep in separators]
seg_res = [a_or_n(seg) for seg in version]
wc = seg_res[0]
for i in xrange(1, len(sep_res)):
wc += '(?:' + sep_res[i] + seg_res[i]
wc += ')?' * (len(seg_res) - 1)
return wc
def __iter__(self):
for v in self.version:
yield v
@@ -96,13 +123,16 @@ def __lt__(self, other):
def __eq__(self, other):
"""Implemented to match __lt__. See __lt__."""
if type(other) == VersionRange:
if type(other) != Version:
return False
return self.version == other.version
def __ne__(self, other):
return not (self == other)
def __hash__(self):
return hash(self.version)
@total_ordering
class VersionRange(object):