zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Better version substitution and wildcard URLs.

Browse Source 
- Previously, URLs like this wouldn't work with spack create:

	http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2

The '5' in hdf5 would interfere with version wildcard substitution beacuse
the wildcard regex would subsume it.

We now take the name of the package OUT of the URL before splitting it up
and adding version wildcards.  This prevents names with numbers from breaking
url.wildcard_version.

Also added a package sanity check test that ensures all builtin packages
work with wildcard_version.
This commit is contained in:
Todd Gamblin 2014-05-16 17:14:37 -07:00
parent e0c7a63a5a
commit b32cbd6b13
6 changed files with 156 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
lib/spack/spack/cmd/create.py
View File

@ -123,6 +123,18 @@ def make_version_dict(ver_hash_tuples):
for v, h in ver_hash_tuples) + ' }'
def get_name():
"""Prompt user to input a package name."""
name = ""
while not name:
new_name = raw_input("Name: ")
if spack.db.valid_name(name):
name = new_name
else:
print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'"
return name
def create(parser, args):
url = args.url
@ -130,18 +142,15 @@ def create(parser, args):
name, version = spack.url.parse_name_and_version(url)
if not name:
tty.msg("Couldn't guess a name for this package.")
while not name:
new_name = raw_input("Name: ")
if spack.db.valid_name(name):
name = new_name
else:
print "Package name can only contain A-Z, a-z, 0-9, '_' and '-'"
name = get_name()
if not version:
tty.die("Couldn't guess a version string from %s." % url)
tty.msg("This looks like a URL for %s version %s." % (name, version))
tty.msg("Creating template for package %s" % name)
# Create a directory for the new package.
pkg_path = spack.db.filename_for_package_name(name)
if os.path.exists(pkg_path) and not args.force:
tty.die("%s already exists." % pkg_path)

4
lib/spack/spack/packages.py
View File

@ -171,7 +171,7 @@ def all_package_names(self):
def all_packages(self):
for name in self.all_package_names():
yield get(name)
yield self.get(name)
def exists(self, pkg_name):
@ -228,7 +228,7 @@ def compute_dependents(self):
pkg._dependents = []
for name, dep in pkg.dependencies.iteritems():
dpkg = get(name)
dpkg = self.get(name)
if dpkg._dependents is None:
dpkg._dependents = []
dpkg._dependents.append(pkg.name)

3
lib/spack/spack/test/__init__.py
View File

@ -43,7 +43,8 @@
'spec_dag',
'concretize',
'multimethod',
'install']
'install',
'package_sanity']
def list_tests():

40
lib/spack/spack/test/package_sanity.py Normal file
View File

@ -0,0 +1,40 @@
##############################################################################
# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://scalability-llnl.github.io/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License (as published by
# the Free Software Foundation) version 2.1 dated February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""\
This test does sanity checks on Spack's builtin package database.
"""
import unittest
import spack
import spack.url as url
class PackageSanityTest(unittest.TestCase):
def test_url_versions(self):
"""Ensure that url_for_version does the right thing for at least the
default version of each package.
"""
for pkg in spack.db.all_packages():
v = url.parse_version(pkg.url)
self.assertEqual(pkg.url, pkg.url_for_version(v))

114
lib/spack/spack/test/url_parse.py
View File

@ -36,21 +36,33 @@ def assert_not_detected(self, string):
self.assertRaises(
url.UndetectableVersionError, url.parse_name_and_version, string)
def assert_detected(self, name, v, string):
def check(self, name, v, string, **kwargs):
# Make sure correct name and version are extracted.
parsed_name, parsed_v = url.parse_name_and_version(string)
self.assertEqual(parsed_name, name)
self.assertEqual(parsed_v, url.Version(v))
# Some URLs (like boost) are special and need to override the
# built-in functionality.
if kwargs.get('no_check_url', False):
return
# Make sure Spack formulates the right URL when we try to
# build one with a specific version.
self.assertEqual(string, url.substitute_version(string, v))
def test_wwwoffle_version(self):
self.assert_detected(
self.check(
'wwwoffle', '2.9h',
'http://www.gedanken.demon.co.uk/download-wwwoffle/wwwoffle-2.9h.tgz')
def test_version_sourceforge_download(self):
self.assert_detected(
self.check(
'foo_bar', '1.21',
'http://sourceforge.net/foo_bar-1.21.tar.gz/download')
self.assert_detected(
self.check(
'foo_bar', '1.21',
'http://sf.net/foo_bar-1.21.tar.gz/download')
@ -59,216 +71,222 @@ def test_no_version(self):
self.assert_not_detected('foo')
def test_version_all_dots(self):
self.assert_detected(
self.check(
'foo.bar.la', '1.14','http://example.com/foo.bar.la.1.14.zip')
def test_version_underscore_separator(self):
self.assert_detected(
self.check(
'grc', '1.1',
'http://example.com/grc_1.1.tar.gz')
def test_boost_version_style(self):
self.assert_detected(
self.check(
'boost', '1.39.0',
'http://example.com/boost_1_39_0.tar.bz2')
'http://example.com/boost_1_39_0.tar.bz2',
no_check_url=True)
def test_erlang_version_style(self):
self.assert_detected(
self.check(
'otp', 'R13B',
'http://erlang.org/download/otp_src_R13B.tar.gz')
def test_another_erlang_version_style(self):
self.assert_detected(
self.check(
'otp', 'R15B01',
'https://github.com/erlang/otp/tarball/OTP_R15B01')
def test_yet_another_erlang_version_style(self):
self.assert_detected(
self.check(
'otp', 'R15B03-1',
'https://github.com/erlang/otp/tarball/OTP_R15B03-1')
def test_p7zip_version_style(self):
self.assert_detected(
self.check(
'p7zip', '9.04',
'http://kent.dl.sourceforge.net/sourceforge/p7zip/p7zip_9.04_src_all.tar.bz2')
def test_new_github_style(self):
self.assert_detected(
self.check(
'libnet', '1.1.4',
'https://github.com/sam-github/libnet/tarball/libnet-1.1.4')
def test_gloox_beta_style(self):
self.assert_detected(
self.check(
'gloox', '1.0-beta7',
'http://camaya.net/download/gloox-1.0-beta7.tar.bz2')
def test_sphinx_beta_style(self):
self.assert_detected(
self.check(
'sphinx', '1.10-beta',
'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz')
def test_astyle_verson_style(self):
self.assert_detected(
self.check(
'astyle', '1.23',
'http://kent.dl.sourceforge.net/sourceforge/astyle/astyle_1.23_macosx.tar.gz')
def test_version_dos2unix(self):
self.assert_detected(
self.check(
'dos2unix', '3.1',
'http://www.sfr-fresh.com/linux/misc/dos2unix-3.1.tar.gz')
def test_version_internal_dash(self):
self.assert_detected(
self.check(
'foo-arse', '1.1-2',
'http://example.com/foo-arse-1.1-2.tar.gz')
def test_version_single_digit(self):
self.assert_detected(
self.check(
'foo_bar', '45',
'http://example.com/foo_bar.45.tar.gz')
def test_noseparator_single_digit(self):
self.assert_detected(
self.check(
'foo_bar', '45',
'http://example.com/foo_bar45.tar.gz')
def test_version_developer_that_hates_us_format(self):
self.assert_detected(
self.check(
'foo-bar-la', '1.2.3',
'http://example.com/foo-bar-la.1.2.3.tar.gz')
def test_version_regular(self):
self.assert_detected(
self.check(
'foo_bar', '1.21',
'http://example.com/foo_bar-1.21.tar.gz')
def test_version_github(self):
self.assert_detected(
self.check(
'yajl', '1.0.5',
'http://github.com/lloyd/yajl/tarball/1.0.5')
def test_version_github_with_high_patch_number(self):
self.assert_detected(
self.check(
'yajl', '1.2.34',
'http://github.com/lloyd/yajl/tarball/v1.2.34')
def test_yet_another_version(self):
self.assert_detected(
self.check(
'mad', '0.15.1b',
'http://example.com/mad-0.15.1b.tar.gz')
def test_lame_version_style(self):
self.assert_detected(
self.check(
'lame', '398-2',
'http://kent.dl.sourceforge.net/sourceforge/lame/lame-398-2.tar.gz')
def test_ruby_version_style(self):
self.assert_detected(
self.check(
'ruby', '1.9.1-p243',
'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz')
def test_omega_version_style(self):
self.assert_detected(
self.check(
'omega', '0.80.2',
'http://www.alcyone.com/binaries/omega/omega-0.80.2-src.tar.gz')
def test_rc_style(self):
self.assert_detected(
self.check(
'libvorbis', '1.2.2rc1',
'http://downloads.xiph.org/releases/vorbis/libvorbis-1.2.2rc1.tar.bz2')
def test_dash_rc_style(self):
self.assert_detected(
self.check(
'js', '1.8.0-rc1',
'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz')
def test_angband_version_style(self):
self.assert_detected(
self.check(
'angband', '3.0.9b',
'http://rephial.org/downloads/3.0/angband-3.0.9b-src.tar.gz')
def test_stable_suffix(self):
self.assert_detected(
self.check(
'libevent', '1.4.14b',
'http://www.monkey.org/~provos/libevent-1.4.14b-stable.tar.gz')
def test_debian_style_1(self):
self.assert_detected(
self.check(
'sl', '3.03',
'http://ftp.de.debian.org/debian/pool/main/s/sl/sl_3.03.orig.tar.gz')
def test_debian_style_2(self):
self.assert_detected(
self.check(
'mmv', '1.01b',
'http://ftp.de.debian.org/debian/pool/main/m/mmv/mmv_1.01b.orig.tar.gz')
def test_imagemagick_style(self):
self.assert_detected(
self.check(
'ImageMagick', '6.7.5-7',
'http://downloads.sf.net/project/machomebrew/mirror/ImageMagick-6.7.5-7.tar.bz2')
def test_dash_version_dash_style(self):
self.assert_detected(
self.check(
'antlr', '3.4',
'http://www.antlr.org/download/antlr-3.4-complete.jar')
def test_apache_version_style(self):
self.assert_detected(
self.check(
'apache-cassandra', '1.2.0-rc2',
'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz')
def test_jpeg_style(self):
self.assert_detected(
self.check(
'jpegsrc', '8d',
'http://www.ijg.org/files/jpegsrc.v8d.tar.gz')
def test_pypy_version(self):
self.assert_detected(
self.check(
'pypy', '1.4.1',
'http://pypy.org/download/pypy-1.4.1-osx.tar.bz2')
def test_openssl_version(self):
self.assert_detected(
self.check(
'openssl', '0.9.8s',
'http://www.openssl.org/source/openssl-0.9.8s.tar.gz')
def test_xaw3d_version(self):
self.assert_detected(
self.check(
'Xaw3d', '1.5E',
'ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz')
def test_fann_version(self):
self.assert_detected(
self.check(
'fann', '2.1.0beta',
'http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip')
def test_iges_version(self):
self.assert_detected(
self.check(
'grads', '2.0.1',
'ftp://iges.org/grads/2.0/grads-2.0.1-bin-darwin9.8-intel.tar.gz')
def test_haxe_version(self):
self.assert_detected(
self.check(
'haxe', '2.08',
'http://haxe.org/file/haxe-2.08-osx.tar.gz')
def test_imap_version(self):
self.assert_detected(
self.check(
'imap', '2007f',
'ftp://ftp.cac.washington.edu/imap/imap-2007f.tar.gz')
def test_suite3270_version(self):
self.assert_detected(
self.check(
'suite3270', '3.3.12ga7',
'http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz')
def test_synergy_version(self):
self.assert_detected(
self.check(
'synergy', '1.3.6p2',
'http://synergy.googlecode.com/files/synergy-1.3.6p2-MacOSX-Universal.zip')
def test_mvapich2_version(self):
self.assert_detected(
self.check(
'mvapich2', '1.9',
'http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz')
def test_hdf5_version(self):
self.check(
'hdf5', '1.8.13',
'http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2')

37
lib/spack/spack/url.py
View File

@ -191,6 +191,16 @@ def parse_name_and_version(path):
return (name, ver)
def insensitize(string):
"""Chagne upper and lowercase letters to be case insensitive in
the provided string. e.g., 'a' because '[Aa]', 'B' becomes
'[bB]', etc. Use for building regexes."""
def to_ins(match):
char = match.group(1)
return '[%s%s]' % (char.lower(), char.upper())
return re.sub(r'([a-zA-Z])', to_ins, string)
def substitute_version(path, new_version):
"""Given a URL or archive name, find the version in the path and substitute
the new version for it.
@ -203,11 +213,26 @@ def wildcard_version(path):
"""Find the version in the supplied path, and return a regular expression
that will match this path with any version in its place.
"""
ver, start, end = parse_version_string_with_indices(path)
# Get name and version, so we can treat them specially
name, v = parse_name_and_version(path)
v = Version(ver)
parts = [re.escape(p) for p in re.split(v.wildcard(), path)]
# Construct a case-insensitive regular expression for the package name.
name_re = '(%s)' % insensitize(name)
# Make a group for the wildcard, so it will be captured by the regex.
version_group = '(%s)' % v.wildcard()
return version_group.join(parts)
# Split the string apart by things that match the name so that if the
# name contains numbers or things that look like versions, we don't
# catch them with the version wildcard.
name_parts = re.split(name_re, path)
# Even elements in the array did *not* match the name
for i in xrange(0, len(name_parts), 2):
# Split each part by things that look like versions.
vparts = re.split(v.wildcard(), name_parts[i])
# Replace each version with a generic capture group to find versions.
# And escape everything else so it's not interpreted as a regex
vgroup = '(%s)' % v.wildcard()
name_parts[i] = vgroup.join(re.escape(vp) for vp in vparts)
# Put it all back together with original name matches intact.
return ''.join(name_parts)