zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Overhaul Spack's URL parsing (#2972)

Browse Source 
* Remove fake URLs from Spack

* Ignore long lines for URLs that start with ftp:

* Preliminary changes to version regexes

* New redesign of version regexes

* Allow letters in version-only

* Fix detection of versions that end in Final

* Rearrange a few regexes and add examples

* Add tests for common download repositories

* Add test cases for common tarball naming schemes

* Finalize version regexes

* spack url test -> spack url summary

* Clean up comments

* Rearrange suffix checks

* Use query strings for name detection

* Remove no longer necessary url_for_version functions

* Strip off extraneous information after package name

* Add one more test

* Dot in square brackets does not need to be escaped

* Move renaming outside of parse_name_offset

* Fix versions for a couple more packages

* Fix flake8 and doc tests

* Correctly parse Python, Lua, and Bio++ package names

* Use effective URLs for mfem

* Add checksummed version to mitos

* Remove url_for_version from STAR-CCM+ package

* Revert changes to version numbers with underscores and dashes

* Fix name detection for tbb

* Correctly parse Ruby gems

* Reverted mfem back to shortened URLs.

* Updated instructions for better security

* Remove preferred=True from newest version

* Add tests for new `spack url list` flags

* Add tests for strip_name_suffixes

* Add unit tests for version separators

* Fix bugs related to parseable name but in parseable version

* Remove dead code, update docstring

* Ignore 'binary' at end of version string

* Remove platform from version

* Flip libedit version numbers

* Re-support weird NCO alpha/beta versions

* Rebase and remove one new fake URL

* Add / to beginning of regex to avoid picking up similarly named packages

* Ignore weird tar versions

* Fix bug in url parse --spider when no versions found

* Less strict version matching for spack versions

* Don't rename Python packages

* Be a little more selective, version must begin with a digit

* Re-add fake URLs

* Fix up several other packages

* Ignore more file endings

* Add parsing support for Miniconda

* Update tab completion

* XFAILS are now PASSES for 2 web tests
This commit is contained in:
Adam J. Stewart
2017-04-03 17:34:16 -05:00
committed by Todd Gamblin
parent 7e9777f294
commit 50df071ad9
80 changed files with 1357 additions and 839 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
lib/spack/docs/developer_guide.rst
View File

@@ -447,16 +447,16 @@ the string that it detected to be the name and version. The
``--incorrect-name`` and ``--incorrect-version`` flags can be used to
print URLs that were not being parsed correctly.
""""""""""""""""""
``spack url test``
""""""""""""""""""
"""""""""""""""""""""
``spack url summary``
"""""""""""""""""""""
This command attempts to parse every URL for every package in Spack
and prints a summary of how many of them are being correctly parsed.
It also prints a histogram showing which regular expressions are being
matched and how frequently:
.. command-output:: spack url test
.. command-output:: spack url summary
This command is essential for anyone adding or changing the regular
expressions that parse names and versions. By running this command

25
lib/spack/spack/cmd/create.py
View File

@@ -31,13 +31,13 @@
import spack
import spack.cmd
import spack.cmd.checksum
import spack.url
import spack.util.web
from llnl.util.filesystem import mkdirp
from spack.repository import Repo
from spack.spec import Spec
from spack.util.executable import which
from spack.util.naming import *
from spack.url import *
description = "create a new package file"
@@ -382,6 +382,10 @@ class BuildSystemGuesser:
can take a peek at the fetched tarball and discern the build system it uses
"""
def __init__(self):
"""Sets the default build system."""
self.build_system = 'generic'
def __call__(self, stage, url):
"""Try to guess the type of build system used by a project based on
the contents of its archive or the URL it was downloaded from."""
@@ -427,14 +431,11 @@ def __call__(self, stage, url):
# Determine the build system based on the files contained
# in the archive.
build_system = 'generic'
for pattern, bs in clues:
if any(re.search(pattern, l) for l in lines):
build_system = bs
self.build_system = bs
break
self.build_system = build_system
def get_name(args):
"""Get the name of the package based on the supplied arguments.
@@ -458,9 +459,9 @@ def get_name(args):
elif args.url:
# Try to guess the package name based on the URL
try:
name = spack.url.parse_name(args.url)
name = parse_name(args.url)
tty.msg("This looks like a URL for {0}".format(name))
except spack.url.UndetectableNameError:
except UndetectableNameError:
tty.die("Couldn't guess a name for this package.",
" Please report this bug. In the meantime, try running:",
" `spack create --name <name> <url>`")
@@ -515,11 +516,16 @@ def get_versions(args, name):
if args.url:
# Find available versions
url_dict = spack.util.web.find_versions_of_archive(args.url)
try:
url_dict = spack.util.web.find_versions_of_archive(args.url)
except UndetectableVersionError:
# Use fake versions
tty.warn("Couldn't detect version in: {0}".format(args.url))
return versions, guesser
if not url_dict:
# If no versions were found, revert to what the user provided
version = spack.url.parse_version(args.url)
version = parse_version(args.url)
url_dict = {version: args.url}
versions = spack.cmd.checksum.get_checksums(
@@ -611,6 +617,7 @@ def create(parser, args):
url = get_url(args)
versions, guesser = get_versions(args, name)
build_system = get_build_system(args, guesser)
name = simplify_name(name)
# Create the package template object
PackageClass = templates[build_system]

2
lib/spack/spack/cmd/flake8.py
View File

@@ -70,7 +70,7 @@
# exemptions applied to all files.
r'.py$': {
# Exempt lines with URLs from overlong line errors.
501: [r'(https?|file)\:']
501: [r'(https?|ftp|file)\:']
},
}

92
lib/spack/spack/cmd/url.py
View File

@@ -31,6 +31,7 @@
from llnl.util import tty
from spack.url import *
from spack.util.web import find_versions_of_archive
from spack.util.naming import simplify_name
description = "debugging tool for url parsing"
@@ -65,20 +66,27 @@ def setup_parser(subparser):
excl_args.add_argument(
'-n', '--incorrect-name', action='store_true',
help='only list urls for which the name was incorrectly parsed')
excl_args.add_argument(
'-N', '--correct-name', action='store_true',
help='only list urls for which the name was correctly parsed')
excl_args.add_argument(
'-v', '--incorrect-version', action='store_true',
help='only list urls for which the version was incorrectly parsed')
excl_args.add_argument(
'-V', '--correct-version', action='store_true',
help='only list urls for which the version was correctly parsed')
# Test
# Summary
sp.add_parser(
'test', help='print a summary of how well we are parsing package urls')
'summary',
help='print a summary of how well we are parsing package urls')
def url(parser, args):
action = {
'parse': url_parse,
'list': url_list,
'test': url_test
'parse': url_parse,
'list': url_list,
'summary': url_summary
}
action[args.subcommand](args)
@@ -116,6 +124,10 @@ def url_parse(args):
tty.msg('Spidering for versions:')
versions = find_versions_of_archive(url)
if not versions:
print(' Found no versions for {0}'.format(name))
return
max_len = max(len(str(v)) for v in versions)
for v in sorted(versions):
@@ -145,7 +157,7 @@ def url_list(args):
return len(urls)
def url_test(args):
def url_summary(args):
# Collect statistics on how many URLs were correctly parsed
total_urls = 0
correct_names = 0
@@ -205,19 +217,19 @@ def url_test(args):
correct_versions, total_urls, correct_versions / total_urls))
print()
tty.msg('Statistics on name regular expresions:')
tty.msg('Statistics on name regular expressions:')
print()
print(' Index Count Regular Expresion')
print(' Index Count Regular Expression')
for ni in name_regex_dict:
print(' {0:>3}: {1:>6} r{2!r}'.format(
ni, name_count_dict[ni], name_regex_dict[ni]))
print()
tty.msg('Statistics on version regular expresions:')
tty.msg('Statistics on version regular expressions:')
print()
print(' Index Count Regular Expresion')
print(' Index Count Regular Expression')
for vi in version_regex_dict:
print(' {0:>3}: {1:>6} r{2!r}'.format(
vi, version_count_dict[vi], version_regex_dict[vi]))
@@ -257,22 +269,38 @@ def url_list_parsing(args, urls, url, pkg):
:rtype: set
"""
if url:
if args.incorrect_name:
# Only add URLs whose name was incorrectly parsed
if args.correct_name or args.incorrect_name:
# Attempt to parse the name
try:
name = parse_name(url)
if not name_parsed_correctly(pkg, name):
if (args.correct_name and
name_parsed_correctly(pkg, name)):
# Add correctly parsed URLs
urls.add(url)
elif (args.incorrect_name and
not name_parsed_correctly(pkg, name)):
# Add incorrectly parsed URLs
urls.add(url)
except UndetectableNameError:
urls.add(url)
elif args.incorrect_version:
# Only add URLs whose version was incorrectly parsed
if args.incorrect_name:
# Add incorrectly parsed URLs
urls.add(url)
elif args.correct_version or args.incorrect_version:
# Attempt to parse the version
try:
version = parse_version(url)
if not version_parsed_correctly(pkg, version):
if (args.correct_version and
version_parsed_correctly(pkg, version)):
# Add correctly parsed URLs
urls.add(url)
elif (args.incorrect_version and
not version_parsed_correctly(pkg, version)):
# Add incorrectly parsed URLs
urls.add(url)
except UndetectableVersionError:
urls.add(url)
if args.incorrect_version:
# Add incorrectly parsed URLs
urls.add(url)
else:
urls.add(url)
@@ -289,6 +317,8 @@ def name_parsed_correctly(pkg, name):
"""
pkg_name = pkg.name
name = simplify_name(name)
# After determining a name, `spack create` determines a build system.
# Some build systems prepend a special string to the front of the name.
# Since this can't be guessed from the URL, it would be unfair to say
@@ -311,9 +341,33 @@ def version_parsed_correctly(pkg, version):
:returns: True if the name was correctly parsed, else False
:rtype: bool
"""
version = remove_separators(version)
# If the version parsed from the URL is listed in a version()
# directive, we assume it was correctly parsed
for pkg_version in pkg.versions:
if str(pkg_version) == str(version):
pkg_version = remove_separators(pkg_version)
if pkg_version == version:
return True
return False
def remove_separators(version):
"""Removes separator characters ('.', '_', and '-') from a version.
A version like 1.2.3 may be displayed as 1_2_3 in the URL.
Make sure 1.2.3, 1-2-3, 1_2_3, and 123 are considered equal.
Unfortunately, this also means that 1.23 and 12.3 are equal.
:param version: A version
:type version: str or Version
:returns: The version with all separator characters removed
:rtype: str
"""
version = str(version)
version = version.replace('.', '')
version = version.replace('_', '')
version = version.replace('-', '')
return version

2
lib/spack/spack/cmd/versions.py
View File

@@ -53,6 +53,6 @@ def versions(parser, args):
tty.debug("Check the list_url and list_depth attribute on the "
"package to help Spack find versions.")
else:
print(" Found no unckecksummed versions for %s" % pkg.name)
print(" Found no unchecksummed versions for %s" % pkg.name)
else:
colify(sorted(remote_versions, reverse=True), indent=2)

21
lib/spack/spack/test/cmd/url.py
View File

@@ -48,11 +48,12 @@ def test_name_parsed_correctly():
assert name_parsed_correctly(MyPackage('r-devtools', []), 'devtools')
assert name_parsed_correctly(MyPackage('py-numpy', []), 'numpy')
assert name_parsed_correctly(MyPackage('octave-splines', []), 'splines')
assert name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa
assert name_parsed_correctly(MyPackage('th-data', []), 'TH.data')
# Expected False
assert not name_parsed_correctly(MyPackage('', []), 'hdf5')
assert not name_parsed_correctly(MyPackage('hdf5', []), '')
assert not name_parsed_correctly(MyPackage('imagemagick', []), 'ImageMagick') # noqa
assert not name_parsed_correctly(MyPackage('yaml-cpp', []), 'yamlcpp')
assert not name_parsed_correctly(MyPackage('yamlcpp', []), 'yaml-cpp')
assert not name_parsed_correctly(MyPackage('r-py-parser', []), 'parser')
@@ -64,6 +65,8 @@ def test_version_parsed_correctly():
assert version_parsed_correctly(MyPackage('', ['1.2.3']), '1.2.3')
assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4a')
assert version_parsed_correctly(MyPackage('', ['5.4a', '5.4b']), '5.4b')
assert version_parsed_correctly(MyPackage('', ['1.63.0']), '1_63_0')
assert version_parsed_correctly(MyPackage('', ['0.94h']), '094h')
# Expected False
assert not version_parsed_correctly(MyPackage('', []), '1.2.3')
@@ -95,7 +98,7 @@ def test_url_list(parser):
colored_urls = url_list(args)
assert colored_urls == total_urls
# The following two options should print fewer URLs than the default.
# The following options should print fewer URLs than the default.
# If they print the same number of URLs, something is horribly broken.
# If they say we missed 0 URLs, something is probably broken too.
args = parser.parse_args(['list', '--incorrect-name'])
@@ -106,11 +109,19 @@ def test_url_list(parser):
incorrect_version_urls = url_list(args)
assert 0 < incorrect_version_urls < total_urls
args = parser.parse_args(['list', '--correct-name'])
correct_name_urls = url_list(args)
assert 0 < correct_name_urls < total_urls
def test_url_test(parser):
args = parser.parse_args(['test'])
args = parser.parse_args(['list', '--correct-version'])
correct_version_urls = url_list(args)
assert 0 < correct_version_urls < total_urls
def test_url_summary(parser):
args = parser.parse_args(['summary'])
(total_urls, correct_names, correct_versions,
name_count_dict, version_count_dict) = url_test(args)
name_count_dict, version_count_dict) = url_summary(args)
assert 0 < correct_names <= sum(name_count_dict.values()) <= total_urls # noqa
assert 0 < correct_versions <= sum(version_count_dict.values()) <= total_urls # noqa

101
lib/spack/spack/test/url_extrapolate.py
View File

@@ -1,101 +0,0 @@
##############################################################################
# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/llnl/spack
# Please also see the LICENSE file for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Tests ability of spack to extrapolate URL versions from
existing versions.
"""
import unittest
import spack.url as url
class UrlExtrapolateTest(unittest.TestCase):
def check_url(self, base, version, new_url):
self.assertEqual(url.substitute_version(base, version), new_url)
def test_libelf_version(self):
base = "http://www.mr511.de/software/libelf-0.8.13.tar.gz"
self.check_url(base, '0.8.13', base)
self.check_url(
base, '0.8.12', "http://www.mr511.de/software/libelf-0.8.12.tar.gz")
self.check_url(
base, '0.3.1', "http://www.mr511.de/software/libelf-0.3.1.tar.gz")
self.check_url(
base, '1.3.1b', "http://www.mr511.de/software/libelf-1.3.1b.tar.gz")
def test_libdwarf_version(self):
base = "http://www.prevanders.net/libdwarf-20130729.tar.gz"
self.check_url(base, '20130729', base)
self.check_url(
base, '8.12', "http://www.prevanders.net/libdwarf-8.12.tar.gz")
def test_dyninst_version(self):
# Dyninst has a version twice in the URL.
base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1.2/DyninstAPI-8.1.2.tgz"
self.check_url(base, '8.1.2', base)
self.check_url(base, '8.2',
"http://www.dyninst.org/sites/default/files/downloads/dyninst/8.2/DyninstAPI-8.2.tgz")
self.check_url(base, '8.3.1',
"http://www.dyninst.org/sites/default/files/downloads/dyninst/8.3.1/DyninstAPI-8.3.1.tgz")
def test_partial_version_prefix(self):
# Test now with a partial prefix earlier in the URL -- this is
# hard to figure out so Spack only substitutes the last
# instance of the version.
base = "http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.2.tgz"
self.check_url(base, '8.1.2', base)
self.check_url(base, '8.1.4',
"http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.1.4.tgz")
self.check_url(base, '8.2',
"http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.2.tgz")
self.check_url(base, '8.3.1',
"http://www.dyninst.org/sites/default/files/downloads/dyninst/8.1/DyninstAPI-8.3.1.tgz")
def test_scalasca_partial_version(self):
# Note that this probably doesn't actually work, but sites are
# inconsistent about their directory structure, so it's not
# clear what is right. This test is for consistency and to
# document behavior. If you figure out a good way to handle
# this case, fix the tests too.
self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1',
'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz')
self.check_url('http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz', '8.3.1',
'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-8.3.1.tar.gz')
def test_mpileaks_version(self):
self.check_url('https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3',
'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz')
def test_gcc(self):
self.check_url('http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.9.2.tar.bz2', '4.7',
'http://open-source-box.org/gcc/gcc-4.7/gcc-4.7.tar.bz2')
self.check_url('http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2', '4.4.7',
'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2')
def test_github_raw(self):
self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
self.check_url('https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true')

866
lib/spack/spack/test/url_parse.py
View File

@@ -22,92 +22,633 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""\
This file has a bunch of versions tests taken from the excellent version
detection in Homebrew.
"""
"""Tests Spack's ability to parse the name and version of a package
based on its URL."""
import os
import unittest
import spack.url as url
from spack.url import *
class UrlParseTest(unittest.TestCase):
class UrlStripVersionSuffixesTest(unittest.TestCase):
"""Tests for spack.url.strip_version_suffixes"""
def check(self, before, after):
stripped = strip_version_suffixes(before)
self.assertEqual(stripped, after)
def test_no_suffix(self):
self.check('rgb-1.0.6',
'rgb-1.0.6')
def test_misleading_prefix(self):
self.check('jpegsrc.v9b',
'jpegsrc.v9b')
self.check('turbolinux702',
'turbolinux702')
self.check('converge_install_2.3.16',
'converge_install_2.3.16')
# Download type
def test_src(self):
self.check('apache-ant-1.9.7-src',
'apache-ant-1.9.7')
self.check('go1.7.4.src',
'go1.7.4')
def test_source(self):
self.check('bowtie2-2.2.5-source',
'bowtie2-2.2.5')
self.check('grib_api-1.17.0-Source',
'grib_api-1.17.0')
def test_full(self):
self.check('julia-0.4.3-full',
'julia-0.4.3')
def test_bin(self):
self.check('apache-maven-3.3.9-bin',
'apache-maven-3.3.9')
def test_binary(self):
self.check('Jmol-14.8.0-binary',
'Jmol-14.8.0')
def test_gem(self):
self.check('rubysl-date-2.0.9.gem',
'rubysl-date-2.0.9')
def test_tar(self):
self.check('gromacs-4.6.1-tar',
'gromacs-4.6.1')
def test_sh(self):
self.check('Miniconda2-4.3.11-Linux-x86_64.sh',
'Miniconda2-4.3.11')
# Download version
def test_stable(self):
self.check('libevent-2.0.21-stable',
'libevent-2.0.21')
def test_final(self):
self.check('2.6.7-final',
'2.6.7')
def test_rel(self):
self.check('v1.9.5.1rel',
'v1.9.5.1')
def test_orig(self):
self.check('dash_0.5.5.1.orig',
'dash_0.5.5.1')
def test_plus(self):
self.check('ncbi-blast-2.6.0+-src',
'ncbi-blast-2.6.0')
# License
def test_gpl(self):
self.check('cppad-20170114.gpl',
'cppad-20170114')
# OS
def test_linux(self):
self.check('astyle_2.04_linux',
'astyle_2.04')
def test_unix(self):
self.check('install-tl-unx',
'install-tl')
def test_macos(self):
self.check('astyle_1.23_macosx',
'astyle_1.23')
self.check('haxe-2.08-osx',
'haxe-2.08')
# PyPI
def test_wheel(self):
self.check('entrypoints-0.2.2-py2.py3-none-any.whl',
'entrypoints-0.2.2')
self.check('numpy-1.12.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl', # noqa
'numpy-1.12.0')
def test_exe(self):
self.check('PyYAML-3.12.win-amd64-py3.5.exe',
'PyYAML-3.12')
# Combinations of multiple patterns
def test_complex_all(self):
self.check('p7zip_9.04_src_all',
'p7zip_9.04')
def test_complex_run(self):
self.check('cuda_8.0.44_linux.run',
'cuda_8.0.44')
def test_complex_file(self):
self.check('ack-2.14-single-file',
'ack-2.14')
def test_complex_jar(self):
self.check('antlr-3.4-complete.jar',
'antlr-3.4')
def test_complex_oss(self):
self.check('tbb44_20160128oss_src_0',
'tbb44_20160128')
def test_complex_darwin(self):
self.check('ghc-7.0.4-x86_64-apple-darwin',
'ghc-7.0.4')
self.check('ghc-7.0.4-i386-apple-darwin',
'ghc-7.0.4')
def test_complex_arch(self):
self.check('VizGlow_v2.2alpha17-R21November2016-Linux-x86_64-Install',
'VizGlow_v2.2alpha17-R21November2016')
self.check('jdk-8u92-linux-x64',
'jdk-8u92')
self.check('cuda_6.5.14_linux_64.run',
'cuda_6.5.14')
def test_complex_with(self):
self.check('mafft-7.221-with-extensions-src',
'mafft-7.221')
self.check('spark-2.0.0-bin-without-hadoop',
'spark-2.0.0')
def test_complex_public(self):
self.check('dakota-6.3-public.src',
'dakota-6.3')
def test_complex_universal(self):
self.check('synergy-1.3.6p2-MacOSX-Universal',
'synergy-1.3.6p2')
class UrlStripNameSuffixesTest(unittest.TestCase):
"""Tests for spack.url.strip_name_suffixes"""
def check(self, before, version, after):
stripped = strip_name_suffixes(before, version)
self.assertEqual(stripped, after)
def test_no_suffix(self):
self.check('rgb-1.0.6', '1.0.6',
'rgb')
self.check('nauty26r7', '26r7',
'nauty')
# Download type
def test_install(self):
self.check('converge_install_2.3.16', '2.3.16',
'converge')
def test_src(self):
self.check('jpegsrc.v9b', '9b',
'jpeg')
def test_std(self):
self.check('ghostscript-fonts-std-8.11', '8.11',
'ghostscript-fonts')
# Download version
def test_snapshot(self):
self.check('gts-snapshot-121130', '121130',
'gts')
def test_distrib(self):
self.check('zoltan_distrib_v3.83', '3.83',
'zoltan')
# VCS
def test_bazaar(self):
self.check('libvterm-0+bzr681', '681',
'libvterm')
# License
def test_gpl(self):
self.check('PyQt-x11-gpl-4.11.3', '4.11.3',
'PyQt-x11')
class UrlParseOffsetTest(unittest.TestCase):
def check(self, name, noffset, ver, voffset, path):
# Make sure parse_name_offset and parse_name_version are working
v, vstart, vlen, vi, vre = parse_version_offset(path)
n, nstart, nlen, ni, nre = parse_name_offset(path, v)
self.assertEqual(n, name)
self.assertEqual(v, ver)
self.assertEqual(nstart, noffset)
self.assertEqual(vstart, voffset)
def test_name_in_path(self):
self.check(
'antlr', 25, '2.7.7', 40,
'https://github.com/antlr/antlr/tarball/v2.7.7')
def test_name_in_stem(self):
self.check(
'gmp', 32, '6.0.0a', 36,
'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2')
def test_name_in_suffix(self):
# Don't think I've ever seen one of these before
# We don't look for it, so it would probably fail anyway
pass
def test_version_in_path(self):
self.check(
'nextflow', 31, '0.20.1', 59,
'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow')
def test_version_in_stem(self):
self.check(
'zlib', 24, '1.2.10', 29,
'http://zlib.net/fossils/zlib-1.2.10.tar.gz')
self.check(
'slepc', 51, '3.6.2', 57,
'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
'cloog', 61, '0.18.1', 67,
'http://www.bastoul.net/cloog/pages/download/count.php3?url=./cloog-0.18.1.tar.gz')
self.check(
'libxc', 58, '2.2.2', 64,
'http://www.tddft.org/programs/octopus/down.php?file=libxc/libxc-2.2.2.tar.gz')
def test_version_in_suffix(self):
self.check(
'swiftsim', 36, '0.3.0', 76,
'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
self.check(
'sionlib', 30, '1.7.1', 59,
'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
def test_regex_in_name(self):
self.check(
'voro++', 40, '0.4.6', 47,
'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz')
class UrlParseNameAndVersionTest(unittest.TestCase):
def assert_not_detected(self, string):
self.assertRaises(
url.UndetectableVersionError, url.parse_name_and_version, string)
UndetectableVersionError, parse_name_and_version, string)
def check(self, name, v, string, **kwargs):
# Make sure correct name and version are extracted.
parsed_name, parsed_v = url.parse_name_and_version(string)
parsed_name, parsed_v = parse_name_and_version(string)
self.assertEqual(parsed_name, name)
self.assertEqual(parsed_v, url.Version(v))
# Some URLs (like boost) are special and need to override the
# built-in functionality.
if kwargs.get('no_check_url', False):
return
self.assertEqual(parsed_v, Version(v))
# Make sure Spack formulates the right URL when we try to
# build one with a specific version.
self.assertEqual(string, url.substitute_version(string, v))
self.assertEqual(string, substitute_version(string, v))
def test_wwwoffle_version(self):
self.check(
'wwwoffle', '2.9h',
'http://www.gedanken.demon.co.uk/download-wwwoffle/wwwoffle-2.9h.tgz')
# Common Repositories
def test_version_sourceforge_download(self):
def test_github_downloads(self):
# name/archive/ver.ver
self.check(
'foo-bar', '1.21',
'http://sourceforge.net/foo_bar-1.21.tar.gz/download')
'nco', '4.6.2',
'https://github.com/nco/nco/archive/4.6.2.tar.gz')
# name/archive/vver.ver
self.check(
'foo-bar', '1.21',
'http://sf.net/foo_bar-1.21.tar.gz/download')
'vim', '8.0.0134',
'https://github.com/vim/vim/archive/v8.0.0134.tar.gz')
# name/archive/name-ver.ver
self.check(
'oce', '0.18',
'https://github.com/tpaviot/oce/archive/OCE-0.18.tar.gz')
# name/releases/download/vver/name-ver.ver
self.check(
'libmesh', '1.0.0',
'https://github.com/libMesh/libmesh/releases/download/v1.0.0/libmesh-1.0.0.tar.bz2')
# name/tarball/vver.ver
self.check(
'git', '2.7.1',
'https://github.com/git/git/tarball/v2.7.1')
# name/zipball/vver.ver
self.check(
'git', '2.7.1',
'https://github.com/git/git/zipball/v2.7.1')
def test_gitlab_downloads(self):
# name/repository/archive.ext?ref=vver.ver
self.check(
'swiftsim', '0.3.0',
'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
# name/repository/archive.ext?ref=name-ver.ver
self.check(
'icet', '1.2.3',
'https://gitlab.kitware.com/icet/icet/repository/archive.tar.gz?ref=IceT-1.2.3')
def test_bitbucket_downloads(self):
# name/get/ver.ver
self.check(
'eigen', '3.2.7',
'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2')
# name/get/vver.ver
self.check(
'hoomd-blue', '1.3.3',
'https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2')
# name/downloads/name-ver.ver
self.check(
'dolfin', '2016.1.0',
'https://bitbucket.org/fenics-project/dolfin/downloads/dolfin-2016.1.0.tar.gz')
def test_sourceforge_downloads(self):
# name-ver.ver
self.check(
'libpng', '1.6.27',
'http://download.sourceforge.net/libpng/libpng-1.6.27.tar.gz')
self.check(
'lcms2', '2.6',
'http://downloads.sourceforge.net/project/lcms/lcms/2.6/lcms2-2.6.tar.gz')
self.check(
'modules', '3.2.10',
'http://prdownloads.sourceforge.net/modules/modules-3.2.10.tar.gz')
# name-ver.ver.ext/download
self.check(
'glew', '2.0.0',
'https://sourceforge.net/projects/glew/files/glew/2.0.0/glew-2.0.0.tgz/download')
def test_cran_downloads(self):
# name.name_ver.ver-ver.ver
self.check(
'TH.data', '1.0-8',
'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz')
self.check(
'knitr', '1.14',
'https://cran.rstudio.com/src/contrib/knitr_1.14.tar.gz')
self.check(
'devtools', '1.12.0',
'https://cloud.r-project.org/src/contrib/devtools_1.12.0.tar.gz')
def test_pypi_downloads(self):
# name.name_name-ver.ver
self.check(
'3to2', '1.1.1',
'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip')
self.check(
'mpmath', '0.19',
'https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz')
self.check(
'pandas', '0.16.0',
'https://pypi.python.org/packages/source/p/pandas/pandas-0.16.0.tar.gz#md5=bfe311f05dc0c351f8955fbd1e296e73')
self.check(
'sphinx_rtd_theme', '0.1.10a0',
'https://pypi.python.org/packages/da/6b/1b75f13d8aa3333f19c6cdf1f0bc9f52ea739cae464fbee050307c121857/sphinx_rtd_theme-0.1.10a0.tar.gz')
self.check(
'backports.ssl_match_hostname', '3.5.0.1',
'https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz')
def test_bazaar_downloads(self):
self.check(
'libvterm', '681',
'http://www.leonerd.org.uk/code/libvterm/libvterm-0+bzr681.tar.gz')
# Common Tarball Formats
def test_version_only(self):
# ver.ver
self.check(
'eigen', '3.2.7',
'https://bitbucket.org/eigen/eigen/get/3.2.7.tar.bz2')
# ver.ver-ver
self.check(
'ImageMagick', '7.0.2-7',
'https://github.com/ImageMagick/ImageMagick/archive/7.0.2-7.tar.gz')
# vver.ver
self.check(
'CGNS', '3.3.0',
'https://github.com/CGNS/CGNS/archive/v3.3.0.tar.gz')
# vver_ver
self.check(
'luafilesystem', '1_6_3',
'https://github.com/keplerproject/luafilesystem/archive/v1_6_3.tar.gz')
def test_no_separators(self):
# namever
self.check(
'turbolinux', '702',
'file://{0}/turbolinux702.tar.gz'.format(os.getcwd()))
self.check(
'nauty', '26r7',
'http://pallini.di.uniroma1.it/nauty26r7.tar.gz')
def test_dashes_only(self):
# name-name-ver-ver
self.check(
'Trilinos', '12-10-1',
'https://github.com/trilinos/Trilinos/archive/trilinos-release-12-10-1.tar.gz')
self.check(
'panda', '2016-03-07',
'http://comopt.ifi.uni-heidelberg.de/software/PANDA/downloads/panda-2016-03-07.tar')
self.check(
'gts', '121130',
'http://gts.sourceforge.net/tarballs/gts-snapshot-121130.tar.gz')
self.check(
'cdd', '061a',
'http://www.cs.mcgill.ca/~fukuda/download/cdd/cdd-061a.tar.gz')
def test_underscores_only(self):
# name_name_ver_ver
self.check(
'tinyxml', '2_6_2',
'https://sourceforge.net/projects/tinyxml/files/tinyxml/2.6.2/tinyxml_2_6_2.tar.gz')
self.check(
'boost', '1_55_0',
'http://downloads.sourceforge.net/project/boost/boost/1.55.0/boost_1_55_0.tar.bz2')
self.check(
'yorick', '2_2_04',
'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz')
# name_namever_ver
self.check(
'tbb', '44_20160413',
'https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb44_20160413oss_src.tgz')
def test_dots_only(self):
# name.name.ver.ver
self.check(
'prank', '150803',
'http://wasabiapp.org/download/prank/prank.source.150803.tgz')
self.check(
'jpeg', '9b',
'http://www.ijg.org/files/jpegsrc.v9b.tar.gz')
self.check(
'openjpeg', '2.1',
'https://github.com/uclouvain/openjpeg/archive/version.2.1.tar.gz')
# name.namever.ver
self.check(
'atlas', '3.11.34',
'http://sourceforge.net/projects/math-atlas/files/Developer%20%28unstable%29/3.11.34/atlas3.11.34.tar.bz2')
self.check(
'visit', '2.10.1',
'http://portal.nersc.gov/project/visit/releases/2.10.1/visit2.10.1.tar.gz')
self.check(
'geant', '4.10.01.p03',
'http://geant4.cern.ch/support/source/geant4.10.01.p03.tar.gz')
self.check(
'tcl', '8.6.5',
'http://prdownloads.sourceforge.net/tcl/tcl8.6.5-src.tar.gz')
def test_dash_dot(self):
# name-name-ver.ver
# digit in name
self.check(
'm4', '1.4.17',
'https://ftp.gnu.org/gnu/m4/m4-1.4.17.tar.gz')
# letter in version
self.check(
'gmp', '6.0.0a',
'https://gmplib.org/download/gmp/gmp-6.0.0a.tar.bz2')
# version starts with 'v'
self.check(
'LaunchMON', '1.0.2',
'https://github.com/LLNL/LaunchMON/releases/download/v1.0.2/launchmon-v1.0.2.tar.gz')
# name-ver-ver.ver
self.check(
'libedit', '20150325-3.1',
'http://thrysoee.dk/editline/libedit-20150325-3.1.tar.gz')
def test_dash_underscore(self):
# name-name-ver_ver
self.check(
'icu4c', '57_1',
'http://download.icu-project.org/files/icu4c/57.1/icu4c-57_1-src.tgz')
def test_underscore_dot(self):
# name_name_ver.ver
self.check(
'superlu_dist', '4.1',
'http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz')
self.check(
'pexsi', '0.9.0',
'https://math.berkeley.edu/~linlin/pexsi/download/pexsi_v0.9.0.tar.gz')
# name_name.ver.ver
self.check(
'fer', '696',
'ftp://ftp.pmel.noaa.gov/ferret/pub/source/fer_source.v696.tar.gz')
def test_dash_dot_dash_dot(self):
# name-name-ver.ver-ver.ver
self.check(
'sowing', '1.1.23-p1',
'http://ftp.mcs.anl.gov/pub/petsc/externalpackages/sowing-1.1.23-p1.tar.gz')
self.check(
'bib2xhtml', '3.0-15-gf506',
'http://www.spinellis.gr/sw/textproc/bib2xhtml/bib2xhtml-v3.0-15-gf506.tar.gz')
# namever.ver-ver.ver
self.check(
'go', '1.4-bootstrap-20161024',
'https://storage.googleapis.com/golang/go1.4-bootstrap-20161024.tar.gz')
def test_underscore_dash_dot(self):
# name_name-ver.ver
self.check(
'the_silver_searcher', '0.32.0',
'http://geoff.greer.fm/ag/releases/the_silver_searcher-0.32.0.tar.gz')
self.check(
'sphinx_rtd_theme', '0.1.10a0',
'https://pypi.python.org/packages/source/s/sphinx_rtd_theme/sphinx_rtd_theme-0.1.10a0.tar.gz')
def test_dot_underscore_dot_dash_dot(self):
# name.name_ver.ver-ver.ver
self.check(
'TH.data', '1.0-8',
'https://cran.r-project.org/src/contrib/TH.data_1.0-8.tar.gz')
self.check(
'XML', '3.98-1.4',
'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz')
def test_dash_dot_underscore_dot(self):
# name-name-ver.ver_ver.ver
self.check(
'pypar', '2.1.5_108',
'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/pypar/pypar-2.1.5_108.tgz')
# name-namever.ver_ver.ver
self.check(
'STAR-CCM+', '11.06.010_02',
'file://{0}/STAR-CCM+11.06.010_02_linux-x86_64.tar.gz'.format(os.getcwd()))
# Weird URLS
def test_version_in_path(self):
# github.com/repo/name/releases/download/name-vver/name
self.check(
'nextflow', '0.20.1',
'https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow')
def test_suffix_queries(self):
self.check(
'swiftsim', '0.3.0',
'http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0')
self.check(
'sionlib', '1.7.1',
'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
def test_stem_queries(self):
self.check(
'slepc', '3.6.2',
'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
'otf', '1.12.5salmon',
'http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz')
def test_single_character_name(self):
self.check(
'R', '3.3.2',
'https://cloud.r-project.org/src/base/R-3/R-3.3.2.tar.gz')
def test_single_digit_version(self):
pass
def test_name_starts_with_digit(self):
self.check(
'3to2', '1.1.1',
'https://pypi.python.org/packages/source/3/3to2/3to2-1.1.1.zip')
def plus_in_name(self):
self.check(
'gtk+', '2.24.31',
'http://ftp.gnome.org/pub/gnome/sources/gtk+/2.24/gtk+-2.24.31.tar.xz')
self.check(
'voro++', '0.4.6',
'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz')
def test_no_version(self):
self.assert_not_detected('http://example.com/blah.tar')
self.assert_not_detected('foo')
self.assert_not_detected('http://www.netlib.org/blas/blast-forum/cblas.tgz')
self.assert_not_detected('http://www.netlib.org/voronoi/triangle.zip')
def test_version_all_dots(self):
def test_download_php(self):
# Name comes before download.php
self.check(
'foo-bar-la', '1.14', 'http://example.com/foo.bar.la.1.14.zip')
def test_version_underscore_separator(self):
'sionlib', '1.7.1',
'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
# Ignore download.php
self.check(
'grc', '1.1',
'http://example.com/grc_1.1.tar.gz')
def test_boost_version_style(self):
'slepc', '3.6.2',
'http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz')
self.check(
'boost', '1.39.0',
'http://example.com/boost_1_39_0.tar.bz2',
no_check_url=True)
def test_erlang_version_style(self):
self.check(
'otp', 'R13B',
'http://erlang.org/download/otp_src_R13B.tar.gz')
def test_another_erlang_version_style(self):
self.check(
'otp', 'R15B01',
'https://github.com/erlang/otp/tarball/OTP_R15B01')
def test_yet_another_erlang_version_style(self):
self.check(
'otp', 'R15B03-1',
'https://github.com/erlang/otp/tarball/OTP_R15B03-1')
def test_p7zip_version_style(self):
self.check(
'p7zip', '9.04',
'http://kent.dl.sourceforge.net/sourceforge/p7zip/p7zip_9.04_src_all.tar.bz2')
def test_new_github_style(self):
self.check(
'libnet', '1.1.4',
'https://github.com/sam-github/libnet/tarball/libnet-1.1.4')
'ScientificPython', '2.8.1',
'https://sourcesup.renater.fr/frs/download.php/file/4411/ScientificPython-2.8.1.tar.gz')
def test_gloox_beta_style(self):
self.check(
@@ -119,85 +660,11 @@ def test_sphinx_beta_style(self):
'sphinx', '1.10-beta',
'http://sphinxsearch.com/downloads/sphinx-1.10-beta.tar.gz')
def test_astyle_verson_style(self):
self.check(
'astyle', '1.23',
'http://kent.dl.sourceforge.net/sourceforge/astyle/astyle_1.23_macosx.tar.gz')
def test_version_dos2unix(self):
self.check(
'dos2unix', '3.1',
'http://www.sfr-fresh.com/linux/misc/dos2unix-3.1.tar.gz')
def test_version_internal_dash(self):
self.check(
'foo-arse', '1.1-2',
'http://example.com/foo-arse-1.1-2.tar.gz')
def test_version_single_digit(self):
self.check(
'foo-bar', '45',
'http://example.com/foo_bar.45.tar.gz')
def test_noseparator_single_digit(self):
self.check(
'foo-bar', '45',
'http://example.com/foo_bar45.tar.gz')
def test_version_developer_that_hates_us_format(self):
self.check(
'foo-bar-la', '1.2.3',
'http://example.com/foo-bar-la.1.2.3.tar.gz')
def test_version_regular(self):
self.check(
'foo-bar', '1.21',
'http://example.com/foo_bar-1.21.tar.gz')
def test_version_gitlab(self):
self.check(
'vtk', '7.0.0',
'https://gitlab.kitware.com/vtk/vtk/repository/'
'archive.tar.bz2?ref=v7.0.0')
self.check(
'icet', '1.2.3',
'https://gitlab.kitware.com/icet/icet/repository/'
'archive.tar.gz?ref=IceT-1.2.3')
self.check(
'foo', '42.1337',
'http://example.com/org/foo/repository/'
'archive.zip?ref=42.1337bar')
def test_version_github(self):
self.check(
'yajl', '1.0.5',
'http://github.com/lloyd/yajl/tarball/1.0.5')
def test_version_github_with_high_patch_number(self):
self.check(
'yajl', '1.2.34',
'http://github.com/lloyd/yajl/tarball/v1.2.34')
def test_yet_another_version(self):
self.check(
'mad', '0.15.1b',
'http://example.com/mad-0.15.1b.tar.gz')
def test_lame_version_style(self):
self.check(
'lame', '398-2',
'http://kent.dl.sourceforge.net/sourceforge/lame/lame-398-2.tar.gz')
def test_ruby_version_style(self):
self.check(
'ruby', '1.9.1-p243',
'ftp://ftp.ruby-lang.org/pub/ruby/1.9/ruby-1.9.1-p243.tar.gz')
def test_omega_version_style(self):
self.check(
'omega', '0.80.2',
'http://www.alcyone.com/binaries/omega/omega-0.80.2-src.tar.gz')
def test_rc_style(self):
self.check(
'libvorbis', '1.2.2rc1',
@@ -208,60 +675,14 @@ def test_dash_rc_style(self):
'js', '1.8.0-rc1',
'http://ftp.mozilla.org/pub/mozilla.org/js/js-1.8.0-rc1.tar.gz')
def test_angband_version_style(self):
self.check(
'angband', '3.0.9b',
'http://rephial.org/downloads/3.0/angband-3.0.9b-src.tar.gz')
def test_stable_suffix(self):
self.check(
'libevent', '1.4.14b',
'http://www.monkey.org/~provos/libevent-1.4.14b-stable.tar.gz')
def test_debian_style_1(self):
self.check(
'sl', '3.03',
'http://ftp.de.debian.org/debian/pool/main/s/sl/sl_3.03.orig.tar.gz')
def test_debian_style_2(self):
self.check(
'mmv', '1.01b',
'http://ftp.de.debian.org/debian/pool/main/m/mmv/mmv_1.01b.orig.tar.gz')
def test_imagemagick_style(self):
self.check(
'imagemagick', '6.7.5-7',
'http://downloads.sf.net/project/machomebrew/mirror/ImageMagick-6.7.5-7.tar.bz2')
def test_dash_version_dash_style(self):
self.check(
'antlr', '3.4',
'http://www.antlr.org/download/antlr-3.4-complete.jar')
def test_apache_version_style(self):
self.check(
'apache-cassandra', '1.2.0-rc2',
'http://www.apache.org/dyn/closer.cgi?path=/cassandra/1.2.0/apache-cassandra-1.2.0-rc2-bin.tar.gz')
def test_jpeg_style(self):
self.check(
'jpegsrc', '8d',
'http://www.ijg.org/files/jpegsrc.v8d.tar.gz')
def test_pypy_version(self):
self.check(
'pypy', '1.4.1',
'http://pypy.org/download/pypy-1.4.1-osx.tar.bz2')
def test_openssl_version(self):
self.check(
'openssl', '0.9.8s',
'http://www.openssl.org/source/openssl-0.9.8s.tar.gz')
def test_xaw3d_version(self):
self.check(
'xaw3d', '1.5E',
'Xaw3d', '1.5E',
'ftp://ftp.visi.com/users/hawkeyd/X/Xaw3d-1.5E.tar.gz')
def test_fann_version(self):
@@ -269,16 +690,6 @@ def test_fann_version(self):
'fann', '2.1.0beta',
'http://downloads.sourceforge.net/project/fann/fann/2.1.0beta/fann-2.1.0beta.zip')
def test_iges_version(self):
self.check(
'grads', '2.0.1',
'ftp://iges.org/grads/2.0/grads-2.0.1-bin-darwin9.8-intel.tar.gz')
def test_haxe_version(self):
self.check(
'haxe', '2.08',
'http://haxe.org/file/haxe-2.08-osx.tar.gz')
def test_imap_version(self):
self.check(
'imap', '2007f',
@@ -289,26 +700,6 @@ def test_suite3270_version(self):
'suite3270', '3.3.12ga7',
'http://sourceforge.net/projects/x3270/files/x3270/3.3.12ga7/suite3270-3.3.12ga7-src.tgz')
def test_synergy_version(self):
self.check(
'synergy', '1.3.6p2',
'http://synergy.googlecode.com/files/synergy-1.3.6p2-MacOSX-Universal.zip')
def test_mvapich2_19_version(self):
self.check(
'mvapich2', '1.9',
'http://mvapich.cse.ohio-state.edu/download/mvapich2/mv2/mvapich2-1.9.tgz')
def test_mvapich2_20_version(self):
self.check(
'mvapich2', '2.0',
'http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-2.0.tar.gz')
def test_hdf5_version(self):
self.check(
'hdf5', '1.8.13',
'http://www.hdfgroup.org/ftp/HDF5/current/src/hdf5-1.8.13.tar.bz2')
def test_scalasca_version(self):
self.check(
'cube', '4.2.3',
@@ -317,55 +708,20 @@ def test_scalasca_version(self):
'cube', '4.3-TP1',
'http://apps.fz-juelich.de/scalasca/releases/cube/4.3/dist/cube-4.3-TP1.tar.gz')
def test_mpileaks_version(self):
self.check(
'mpileaks', '1.0',
'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz')
self.check(
'mpileaks', '1.0',
'https://github.com/hpc/mpileaks/releases/download/1.0/mpileaks-1.0.tar.gz')
def test_gcc_version(self):
self.check(
'gcc', '4.4.7',
'http://open-source-box.org/gcc/gcc-4.4.7/gcc-4.4.7.tar.bz2')
def test_gcc_version_precedence(self):
# prefer the version in the tarball, not in the url prefix.
self.check(
'gcc', '4.4.7',
'http://open-source-box.org/gcc/gcc-4.9.2/gcc-4.4.7.tar.bz2')
def test_github_raw_url(self):
self.check(
'powerparser', '2.0.7',
'CLAMR', '2.0.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
def test_r_xml_version(self):
self.check(
'xml', '3.98-1.4',
'https://cran.r-project.org/src/contrib/XML_3.98-1.4.tar.gz')
def test_nco_version(self):
self.check(
'nco', '4.6.2-beta03',
'https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz')
self.check(
'nco', '4.6.3-alpha04',
'https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz')
def test_yorick_version(self):
self.check(
'yorick', '2_2_04',
'https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz')
def test_luaposix_version(self):
self.check(
'luaposix', '33.4.0',
'https://github.com/luaposix/luaposix/archive/release-v33.4.0.tar.gz')
def test_sionlib_version(self):
def test_nco_version(self):
self.check(
'sionlib', '1.7.1',
'http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1')
'nco', '4.6.2-beta03',
'https://github.com/nco/nco/archive/4.6.2-beta03.tar.gz')
self.check(
'nco', '4.6.3-alpha04',
'https://github.com/nco/nco/archive/4.6.3-alpha04.tar.gz')

84
lib/spack/spack/test/url_substitution.py
View File

@@ -22,44 +22,64 @@
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""\
This test does sanity checks on substituting new versions into URLs
"""
"""Tests Spack's ability to substitute a different version into a URL."""
import os
import unittest
import spack.url as url
from spack.url import substitute_version
base = "https://comp.llnl.gov/linear_solvers/download/hypre-2.9.0b.tar.gz"
stem = "https://comp.llnl.gov/linear_solvers/download/hypre-"
class UrlSubstitutionTest(unittest.TestCase):
def check(self, base, version, new_url):
self.assertEqual(substitute_version(base, version), new_url)
class PackageSanityTest(unittest.TestCase):
def test_same_version(self):
# Ensures that substituting the same version results in the same URL
self.check(
'http://www.mr511.de/software/libelf-0.8.13.tar.gz', '0.8.13',
'http://www.mr511.de/software/libelf-0.8.13.tar.gz')
def test_hypre_url_substitution(self):
self.assertEqual(url.substitute_version(base, '2.9.0b'), base)
self.assertEqual(
url.substitute_version(base, '2.8.0b'), stem + "2.8.0b.tar.gz")
self.assertEqual(
url.substitute_version(base, '2.7.0b'), stem + "2.7.0b.tar.gz")
self.assertEqual(
url.substitute_version(base, '2.6.0b'), stem + "2.6.0b.tar.gz")
self.assertEqual(
url.substitute_version(base, '1.14.0b'), stem + "1.14.0b.tar.gz")
self.assertEqual(
url.substitute_version(base, '1.13.0b'), stem + "1.13.0b.tar.gz")
self.assertEqual(
url.substitute_version(base, '2.0.0'), stem + "2.0.0.tar.gz")
self.assertEqual(
url.substitute_version(base, '1.6.0'), stem + "1.6.0.tar.gz")
def test_different_version(self):
# Test a completely different version syntax
self.check(
'http://www.prevanders.net/libdwarf-20130729.tar.gz', '8.12',
'http://www.prevanders.net/libdwarf-8.12.tar.gz')
def test_otf2_url_substitution(self):
base = "http://www.vi-hps.org/upload/packages/otf2/otf2-1.4.tar.gz"
def test_double_version(self):
# Test a URL where the version appears twice
# It should get substituted both times
self.check(
'https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz', '2.1.3',
'https://github.com/hpc/mpileaks/releases/download/v2.1.3/mpileaks-2.1.3.tar.gz')
self.assertEqual(url.substitute_version(base, '1.4'), base)
def test_partial_version_prefix(self):
# Test now with a partial prefix earlier in the URL
# This is hard to figure out so Spack only substitutes
# the last instance of the version
self.check(
'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2.0',
'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.0.tar.bz2')
self.check(
'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.1.0.tar.bz2', '2.2',
'https://www.open-mpi.org/software/ompi/v2.1/downloads/openmpi-2.2.tar.bz2')
self.assertEqual(
url.substitute_version(base, '1.3.1'),
"http://www.vi-hps.org/upload/packages/otf2/otf2-1.3.1.tar.gz")
self.assertEqual(
url.substitute_version(base, '1.2.1'),
"http://www.vi-hps.org/upload/packages/otf2/otf2-1.2.1.tar.gz")
def test_no_separator(self):
# No separator between the name and version of the package
self.check(
'file://{0}/turbolinux702.tar.gz'.format(os.getcwd()), '703',
'file://{0}/turbolinux703.tar.gz'.format(os.getcwd()))
def test_github_raw(self):
self.check(
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '2.0.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true')
self.check(
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v2.0.7.tgz?raw=true', '4.7',
'https://github.com/losalamos/CLAMR/blob/packages/PowerParser_v4.7.tgz?raw=true')
def test_regex(self):
# Package name contains regex characters
self.check(
'http://math.lbl.gov/voro++/download/dir/voro++-0.4.6.tar.gz', '1.2.3',
'http://math.lbl.gov/voro++/download/dir/voro++-1.2.3.tar.gz')

3
lib/spack/spack/test/web.py
View File

@@ -23,7 +23,6 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
"""Tests for web.py."""
import pytest
import os
import spack
@@ -141,7 +140,6 @@ def test_find_versions_of_archive_2():
assert ver('2.0.0') in versions
@pytest.mark.xfail
def test_find_exotic_versions_of_archive_2():
versions = find_versions_of_archive(root_tarball, root, list_depth=2)
# up for grabs to make this better.
@@ -157,7 +155,6 @@ def test_find_versions_of_archive_3():
assert ver('4.5') in versions
@pytest.mark.xfail
def test_find_exotic_versions_of_archive_3():
versions = find_versions_of_archive(root_tarball, root, list_depth=3)
assert ver('2.0.0b2') in versions

505
lib/spack/spack/url.py
View File

@@ -71,7 +71,7 @@ def find_list_url(url):
url_types = [
# e.g. https://github.com/llnl/callpath/archive/v1.0.1.tar.gz
(r'^(https://github.com/[^/]+/[^/]+)/archive/',
(r'(.*github\.com/[^/]+/[^/]+)/archive/',
lambda m: m.group(1) + '/releases')]
for pattern, fun in url_types:
@@ -101,6 +101,177 @@ def strip_query_and_fragment(path):
return (path, '') # Ignore URL parse errors here
def strip_version_suffixes(path):
"""Some tarballs contain extraneous information after the version:
* ``bowtie2-2.2.5-source``
* ``libevent-2.0.21-stable``
* ``cuda_8.0.44_linux.run``
These strings are not part of the version number and should be ignored.
This function strips those suffixes off and returns the remaining string.
The goal is that the version is always the last thing in ``path``:
* ``bowtie2-2.2.5``
* ``libevent-2.0.21``
* ``cuda_8.0.44``
:param str path: The filename or URL for the package
:return: The ``path`` with any extraneous suffixes removed
:rtype: str
"""
# NOTE: This could be done with complicated regexes in parse_version_offset
# NOTE: The problem is that we would have to add these regexes to the end
# NOTE: of every single version regex. Easier to just strip them off
# NOTE: permanently
suffix_regexes = [
# Download type
'[Ii]nstall',
'all',
'src(_0)?',
'[Ss]ources?',
'file',
'full',
'single',
'public',
'with[a-zA-Z_-]+',
'bin',
'binary',
'run',
'[Uu]niversal',
'jar',
'complete',
'oss',
'gem',
'tar',
'sh',
# Download version
'stable',
'[Ff]inal',
'rel',
'orig',
'dist',
'\+',
# License
'gpl',
# Arch
# Needs to come before and after OS, appears in both orders
'ia32',
'intel',
'amd64',
'x64',
'x86_64',
'x86',
'i[36]86',
'ppc64(le)?',
'armv?(7l|6l|64)',
# OS
'[Ll]inux(_64)?',
'[Uu]ni?x',
'[Ss]un[Oo][Ss]',
'[Mm]ac[Oo][Ss][Xx]?',
'[Oo][Ss][Xx]',
'[Dd]arwin(64)?',
'[Aa]pple',
'[Ww]indows',
'[Ww]in(64|32)?',
'[Cc]ygwin(64|32)?',
'[Mm]ingw',
# Arch
# Needs to come before and after OS, appears in both orders
'ia32',
'intel',
'amd64',
'x64',
'x86_64',
'x86',
'i[36]86',
'ppc64(le)?',
'armv?(7l|6l|64)?',
# PyPI
'[._-]py[23].*\.whl',
'[._-]cp[23].*\.whl',
'[._-]win.*\.exe',
]
for regex in suffix_regexes:
# Remove the suffix from the end of the path
# This may be done multiple times
path = re.sub(r'[._-]?' + regex + '$', '', path)
return path
def strip_name_suffixes(path, version):
"""Most tarballs contain a package name followed by a version number.
However, some also contain extraneous information in-between the name
and version:
* ``rgb-1.0.6``
* ``converge_install_2.3.16``
* ``jpegsrc.v9b``
These strings are not part of the package name and should be ignored.
This function strips the version number and any extraneous suffixes
off and returns the remaining string. The goal is that the name is
always the last thing in ``path``:
* ``rgb``
* ``converge``
* ``jpeg``
:param str path: The filename or URL for the package
:param str version: The version detected for this URL
:return: The ``path`` with any extraneous suffixes removed
:rtype: str
"""
# NOTE: This could be done with complicated regexes in parse_name_offset
# NOTE: The problem is that we would have to add these regexes to every
# NOTE: single name regex. Easier to just strip them off permanently
suffix_regexes = [
# Strip off the version and anything after it
# name-ver
# name_ver
# name.ver
r'[._-]v?' + str(version) + '.*',
# namever
str(version) + '.*',
# Download type
'install',
'src',
'(open)?[Ss]ources?',
'[._-]std',
# Download version
'snapshot',
'distrib',
# VCS
'0\+bzr',
# License
'gpl',
]
for regex in suffix_regexes:
# Remove the suffix from the end of the path
# This may be done multiple times
path = re.sub('[._-]?' + regex + '$', '', path)
return path
def split_url_extension(path):
"""Some URLs have a query string, e.g.:
@@ -125,7 +296,7 @@ def split_url_extension(path):
prefix, ext, suffix = path, '', ''
# Strip off sourceforge download suffix.
match = re.search(r'((?:sourceforge.net|sf.net)/.*)(/download)$', path)
match = re.search(r'((?:sourceforge\.net|sf\.net)/.*)(/download)$', path)
if match:
prefix, suffix = match.groups()
@@ -189,8 +360,20 @@ def parse_version_offset(path):
path, ext, suffix = split_url_extension(path)
# stem: Everything from path after the final '/'
stem = os.path.basename(path)
offset = len(path) - len(stem)
original_stem = os.path.basename(path)
# Try to strip off anything after the version number
stem = strip_version_suffixes(original_stem)
# Assumptions:
#
# 1. version always comes after the name
# 2. separators include '-', '_', and '.'
# 3. names can contain A-Z, a-z, 0-9, '+', separators
# 4. versions can contain A-Z, a-z, 0-9, separators
# 5. versions always start with a digit
# 6. versions are often prefixed by a 'v' character
# 7. separators are most reliable to determine name/version boundaries
# List of the following format:
#
@@ -202,87 +385,118 @@ def parse_version_offset(path):
# The first regex that matches string will be used to determine
# the version of the package. Thefore, hyperspecific regexes should
# come first while generic, catch-all regexes should come last.
# With that said, regular expressions are slow, so if possible, put
# ones that only catch one or two URLs at the bottom.
version_regexes = [
# GitHub tarballs, e.g. v1.2.3
(r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+)$', path),
# 1st Pass: Simplest case
# Assume name contains no digits and version contains no letters
# e.g. libpng-1.6.27
(r'^[a-zA-Z+._-]+[._-]v?(\d[\d._-]*)$', stem),
# e.g. https://github.com/sam-github/libnet/tarball/libnet-1.1.4
(r'github.com/.+/(?:zip|tar)ball/.*-((\d+\.)+\d+)$', path),
# 2nd Pass: Version only
# Assume version contains no letters
# e.g. https://github.com/isaacs/npm/tarball/v0.2.5-1
(r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+-(\d+))$', path),
# ver
# e.g. 3.2.7, 7.0.2-7, v3.3.0, v1_6_3
(r'^v?(\d[\d._-]*)$', stem),
# e.g. https://github.com/petdance/ack/tarball/1.93_02
(r'github.com/.+/(?:zip|tar)ball/v?((\d+\.)+\d+_(\d+))$', path),
# 3rd Pass: No separator characters are used
# Assume name contains no digits
# Yorick is very special.
# e.g. https://github.com/dhmunro/yorick/archive/y_2_2_04.tar.gz
(r'github.com/[^/]+/yorick/archive/y_(\d+(?:_\d+)*)$', path),
# namever
# e.g. turbolinux702, nauty26r7
(r'^[a-zA-Z+]*(\d[\da-zA-Z]*)$', stem),
# e.g. https://github.com/hpc/lwgrp/archive/v1.0.1.tar.gz
(r'github.com/[^/]+/[^/]+/archive/(?:release-)?v?(\w+(?:[.-]\w+)*)$', path), # noqa
# 4th Pass: A single separator character is used
# Assume name contains no digits
# e.g. https://github.com/erlang/otp/tarball/OTP_R15B01 (erlang style)
(r'[-_](R\d+[AB]\d*(-\d+)?)', path),
# name-name-ver-ver
# e.g. panda-2016-03-07, gts-snapshot-121130, cdd-061a
(r'^[a-zA-Z+-]*(\d[\da-zA-Z-]*)$', stem),
# e.g., https://github.com/hpc/libcircle/releases/download/0.2.1-rc.1/libcircle-0.2.1-rc.1.tar.gz
# e.g.,
# https://github.com/hpc/mpileaks/releases/download/v1.0/mpileaks-1.0.tar.gz
(r'github.com/[^/]+/[^/]+/releases/download/v?([^/]+)/.*$', path),
# name_name_ver_ver
# e.g. tinyxml_2_6_2, boost_1_55_0, tbb2017_20161128, v1_6_3
(r'^[a-zA-Z+_]*(\d[\da-zA-Z_]*)$', stem),
# GitLab syntax:
# {baseUrl}{/organization}{/projectName}/repository/archive.{fileEnding}?ref={gitTag}
# as with github releases, we hope a version can be found in the
# git tag
# Search dotted versions:
# e.g., https://gitlab.kitware.com/vtk/vtk/repository/archive.tar.bz2?ref=v7.0.0
# e.g., https://example.com/org/repo/repository/archive.tar.bz2?ref=SomePrefix-2.1.1
# e.g., http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
(r'\?ref=(?:.*-|v)*((\d+\.)+\d+).*$', suffix),
(r'\?version=((\d+\.)+\d+)', suffix),
# name.name.ver.ver
# e.g. prank.source.150803, jpegsrc.v9b, atlas3.11.34, geant4.10.01.p03
(r'^[a-zA-Z+.]*(\d[\da-zA-Z.]*)$', stem),
# e.g. boost_1_39_0
(r'((\d+_)+\d+)$', stem),
# 5th Pass: Two separator characters are used
# Name may contain digits, version may contain letters
# e.g. foobar-4.5.1-1
# e.g. ruby-1.9.1-p243
(r'-((\d+\.)*\d\.\d+-(p|rc|RC)?\d+)(?:[-._](?:bin|dist|stable|src|sources))?$', stem), # noqa
# name-name-ver.ver
# e.g. m4-1.4.17, gmp-6.0.0a, launchmon-v1.0.2
(r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem),
# e.g. lame-398-1
(r'-((\d)+-\d)', stem),
# name-name-ver_ver
# e.g. icu4c-57_1
(r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z_]*)$', stem),
# e.g. foobar_1.2-3 or 3.98-1.4
(r'_((\d+\.)+\d+(-(\d+(\.\d+)?))?[a-z]?)', stem),
# name_name_ver.ver
# e.g. superlu_dist_4.1, pexsi_v0.9.0
(r'^[a-zA-Z\d+_]+_v?(\d[\da-zA-Z.]*)$', stem),
# e.g. foobar-4.5.1
(r'-((\d+\.)*\d+)$', stem),
# name_name.ver.ver
# e.g. fer_source.v696
(r'^[a-zA-Z\d+_]+\.v?(\d[\da-zA-Z.]*)$', stem),
# e.g. foobar-4.5.1b, foobar4.5RC, foobar.v4.5.1b
(r'[-._]?v?((\d+\.)*\d+[-._]?([a-z]|rc|RC|tp|TP?)\d*)$', stem),
# name-name-ver.ver-ver.ver
# e.g. sowing-1.1.23-p1, bib2xhtml-v3.0-15-gf506, 4.6.3-alpha04
(r'^(?:[a-zA-Z\d+-]+-)?v?(\d[\da-zA-Z.-]*)$', stem),
# e.g. foobar-4.5.0-beta1, or foobar-4.50-beta
(r'-((\d+\.)*\d+-beta(\d+)?)$', stem),
# namever.ver-ver.ver
# e.g. go1.4-bootstrap-20161024
(r'^[a-zA-Z+]+v?(\d[\da-zA-Z.-]*)$', stem),
# e.g. foobar4.5.1
(r'((\d+\.)*\d+)$', stem),
# 6th Pass: All three separator characters are used
# Name may contain digits, version may contain letters
# e.g. foobar-4.5.0-bin
(r'-((\d+\.)+\d+[a-z]?)[-._](bin|dist|stable|src|sources?)$', stem),
# name_name-ver.ver
# e.g. the_silver_searcher-0.32.0, sphinx_rtd_theme-0.1.10a0
(r'^[a-zA-Z\d+_]+-v?(\d[\da-zA-Z.]*)$', stem),
# e.g. dash_0.5.5.1.orig.tar.gz (Debian style)
(r'_((\d+\.)+\d+[a-z]?)[.]orig$', stem),
# name.name_ver.ver-ver.ver
# e.g. TH.data_1.0-8, XML_3.98-1.4
(r'^[a-zA-Z\d+.]+_v?(\d[\da-zA-Z.-]*)$', stem),
# e.g. http://www.openssl.org/source/openssl-0.9.8s.tar.gz
(r'-v?([^-]+(-alpha|-beta)?)', stem),
# name-name-ver.ver_ver.ver
# e.g. pypar-2.1.5_108
(r'^[a-zA-Z\d+-]+-v?(\d[\da-zA-Z._]*)$', stem),
# e.g. astyle_1.23_macosx.tar.gz
(r'_([^_]+(_alpha|_beta)?)', stem),
# name.name_name-ver.ver
# e.g. tap.py-1.6, backports.ssl_match_hostname-3.5.0.1
(r'^[a-zA-Z\d+._]+-v?(\d[\da-zA-Z.]*)$', stem),
# e.g. http://mirrors.jenkins-ci.org/war/1.486/jenkins.war
(r'\/(\d\.\d+)\/', path),
# name-namever.ver_ver.ver
# e.g. STAR-CCM+11.06.010_02
(r'^[a-zA-Z+-]+(\d[\da-zA-Z._]*)$', stem),
# e.g. http://www.ijg.org/files/jpegsrc.v8d.tar.gz
(r'\.v(\d+[a-z]?)', stem)
# 7th Pass: Specific VCS
# bazaar
# e.g. libvterm-0+bzr681
(r'bzr(\d[\da-zA-Z._-]*)$', stem),
# 8th Pass: Version in path
# github.com/repo/name/releases/download/vver/name
# e.g. https://github.com/nextflow-io/nextflow/releases/download/v0.20.1/nextflow
(r'github\.com/[^/]+/[^/]+/releases/download/[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)/', path), # noqa
# 9th Pass: Query strings
# e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
(r'\?ref=[a-zA-Z+._-]*v?(\d[\da-zA-Z._-]*)$', suffix),
# e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
(r'\?version=v?(\d[\da-zA-Z._-]*)$', suffix),
# e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz
(r'\?filename=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem),
# e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz
(r'\?package=[a-zA-Z\d+-]+&get=[a-zA-Z\d+-]+-v?(\d[\da-zA-Z.]*)$', stem), # noqa
]
for i, version_regex in enumerate(version_regexes):
@@ -292,9 +506,15 @@ def parse_version_offset(path):
version = match.group(1)
start = match.start(1)
# if we matched from the basename, then add offset in.
# If we matched from the stem or suffix, we need to add offset
offset = 0
if match_string is stem:
start += offset
offset = len(path) - len(original_stem)
elif match_string is suffix:
offset = len(path)
if ext:
offset += len(ext) + 1 # .tar.gz is converted to tar.gz
start += offset
return version, start, len(version), i, regex
@@ -342,7 +562,7 @@ def parse_name_offset(path, v=None):
except UndetectableVersionError:
# Not all URLs contain a version. We still want to be able
# to determine a name if possible.
v = ''
v = 'unknown'
# path: The prefix of the URL, everything before the ext and suffix
# ext: The file extension
@@ -350,8 +570,10 @@ def parse_name_offset(path, v=None):
path, ext, suffix = split_url_extension(path)
# stem: Everything from path after the final '/'
stem = os.path.basename(path)
offset = len(path) - len(stem)
original_stem = os.path.basename(path)
# Try to strip off anything after the package name
stem = strip_name_suffixes(original_stem, v)
# List of the following format:
#
@@ -363,26 +585,45 @@ def parse_name_offset(path, v=None):
# The first regex that matches string will be used to determine
# the name of the package. Thefore, hyperspecific regexes should
# come first while generic, catch-all regexes should come last.
# With that said, regular expressions are slow, so if possible, put
# ones that only catch one or two URLs at the bottom.
name_regexes = [
(r'/sourceforge/([^/]+)/', path),
(r'github.com/[^/]+/[^/]+/releases/download/%s/(.*)-%s$' %
(v, v), path),
(r'/([^/]+)/(tarball|zipball)/', path),
(r'/([^/]+)[_.-](bin|dist|stable|src|sources)[_.-]%s' % v, path),
(r'github.com/[^/]+/([^/]+)/archive', path),
(r'github.com/[^/]+/([^/]+)/releases', path),
(r'[^/]+/([^/]+)/repository/archive', path), # gitlab
(r'([^/]+)/download.php', path),
# 1st Pass: Common repositories
(r'([^/]+)[_.-]v?%s' % v, stem), # prefer the stem
(r'([^/]+)%s' % v, stem),
# GitHub: github.com/repo/name/
# e.g. https://github.com/nco/nco/archive/4.6.2.tar.gz
(r'github\.com/[^/]+/([^/]+)', path),
# accept the path if name is not in stem.
(r'/([^/]+)[_.-]v?%s' % v, path),
(r'/([^/]+)%s' % v, path),
# GitLab: gitlab.*/repo/name/
# e.g. http://gitlab.cosma.dur.ac.uk/swift/swiftsim/repository/archive.tar.gz?ref=v0.3.0
(r'gitlab[^/]+/[^/]+/([^/]+)', path),
(r'^([^/]+)[_.-]v?%s' % v, path),
(r'^([^/]+)%s' % v, path)
# Bitbucket: bitbucket.org/repo/name/
# e.g. https://bitbucket.org/glotzer/hoomd-blue/get/v1.3.3.tar.bz2
(r'bitbucket\.org/[^/]+/([^/]+)', path),
# PyPI: pypi.(python.org|io)/packages/source/first-letter/name/
# e.g. https://pypi.python.org/packages/source/m/mpmath/mpmath-all-0.19.tar.gz
# e.g. https://pypi.io/packages/source/b/backports.ssl_match_hostname/backports.ssl_match_hostname-3.5.0.1.tar.gz
(r'pypi\.(?:python\.org|io)/packages/source/[A-Za-z\d]/([^/]+)', path),
# 2nd Pass: Query strings
# ?filename=name-ver.ver
# e.g. http://slepc.upv.es/download/download.php?filename=slepc-3.6.2.tar.gz
(r'\?filename=([A-Za-z\d+-]+)$', stem),
# ?package=name
# e.g. http://wwwpub.zih.tu-dresden.de/%7Emlieber/dcount/dcount.php?package=otf&get=OTF-1.12.5salmon.tar.gz
(r'\?package=([A-Za-z\d+-]+)', stem),
# download.php
# e.g. http://apps.fz-juelich.de/jsc/sionlib/download.php?version=1.7.1
(r'([^/]+)/download.php$', path),
# 3rd Pass: Name followed by version in archive
(r'^([A-Za-z\d+\._-]+)$', stem),
]
for i, name_regex in enumerate(name_regexes):
@@ -392,13 +633,15 @@ def parse_name_offset(path, v=None):
name = match.group(1)
start = match.start(1)
# if we matched from the basename, then add offset in.
# If we matched from the stem or suffix, we need to add offset
offset = 0
if match_string is stem:
start += offset
# package names should be lowercase and separated by dashes.
name = name.lower()
name = re.sub('[_.]', '-', name)
offset = len(path) - len(original_stem)
elif match_string is suffix:
offset = len(path)
if ext:
offset += len(ext) + 1 # .tar.gz is converted to tar.gz
start += offset
return name, start, len(name), i, regex
@@ -431,6 +674,9 @@ def parse_name_and_version(path):
The version of the package
:rtype: tuple
:raises UndetectableVersionError: If the URL does not match any regexes
:raises UndetectableNameError: If the URL does not match any regexes
"""
ver = parse_version(path)
name = parse_name(path, ver)
@@ -457,6 +703,22 @@ def cumsum(elts, init=0, fn=lambda x: x):
return sums
def find_all(substring, string):
"""Returns a list containing the indices of
every occurrence of substring in string."""
occurrences = []
index = 0
while index < len(string):
index = string.find(substring, index)
if index == -1:
break
occurrences.append(index)
index += len(substring)
return occurrences
def substitution_offsets(path):
"""This returns offsets for substituting versions and names in the
provided path. It is a helper for :func:`substitute_version`.
@@ -468,65 +730,34 @@ def substitution_offsets(path):
except UndetectableNameError:
return (None, -1, -1, (), ver, vs, vl, (vs,))
except UndetectableVersionError:
return (None, -1, -1, (), None, -1, -1, ())
try:
name, ns, nl, ni, nregex = parse_name_offset(path)
return (name, ns, nl, (ns,), None, -1, -1, ())
except UndetectableNameError:
return (None, -1, -1, (), None, -1, -1, ())
# protect extensions like bz2 from getting inadvertently
# considered versions.
path = comp.strip_extension(path)
# Find the index of every occurrence of name and ver in path
name_offsets = find_all(name, path)
ver_offsets = find_all(ver, path)
# Construct a case-insensitive regular expression for the package name.
name_re = '(%s)' % insensitize(name)
# Split the string apart by things that match the name so that if the
# name contains numbers or things that look like versions, we don't
# accidentally substitute them with a version.
name_parts = re.split(name_re, path)
offsets = cumsum(name_parts, 0, len)
name_offsets = offsets[1::2]
ver_offsets = []
for i in range(0, len(name_parts), 2):
vparts = re.split(ver, name_parts[i])
voffsets = cumsum(vparts, offsets[i], len)
ver_offsets.extend(voffsets[1::2])
return (name, ns, nl, tuple(name_offsets),
ver, vs, vl, tuple(ver_offsets))
return (name, ns, nl, name_offsets,
ver, vs, vl, ver_offsets)
def wildcard_version(path):
"""Find the version in the supplied path, and return a regular expression
that will match this path with any version in its place.
"""
# Get name and version, so we can treat them specially
name, v = parse_name_and_version(path)
# Get version so we can replace it with a wildcard
version = parse_version(path)
path, ext, suffix = split_url_extension(path)
# Split path by versions
vparts = path.split(str(version))
# Construct a case-insensitive regular expression for the package name.
name_re = '(%s)' % insensitize(name)
# Replace each version with a generic capture group to find versions
# and escape everything else so it's not interpreted as a regex
result = '(\d.*)'.join(re.escape(vp) for vp in vparts)
# Split the string apart by things that match the name so that if the
# name contains numbers or things that look like versions, we don't
# catch them with the version wildcard.
name_parts = re.split(name_re, path)
# Even elements in the array did *not* match the name
for i in range(0, len(name_parts), 2):
# Split each part by things that look like versions.
vparts = re.split(v.wildcard(), name_parts[i])
# Replace each version with a generic capture group to find versions.
# And escape everything else so it's not interpreted as a regex
vgroup = '(%s)' % v.wildcard()
name_parts[i] = vgroup.join(re.escape(vp) for vp in vparts)
# Put it all back together with original name matches intact.
result = ''.join(name_parts)
if ext:
result += '.' + ext
result += suffix
return result

45
lib/spack/spack/util/naming.py
View File

@@ -39,6 +39,7 @@
'validate_fully_qualified_module_name',
'validate_module_name',
'possible_spack_module_names',
'simplify_name',
'NamespaceTrie']
# Valid module names can contain '-' but can't start with it.
@@ -108,6 +109,50 @@ def possible_spack_module_names(python_mod_name):
return results
def simplify_name(name):
"""Simplifies a name which may include uppercase letters, periods,
underscores, and pluses. In general, we want our package names to
only contain lowercase letters, digits, and dashes.
:param str name: The original name of the package
:return: The new name of the package
:rtype: str
"""
# Convert CamelCase to Dashed-Names
# e.g. ImageMagick -> Image-Magick
# e.g. SuiteSparse -> Suite-Sparse
# name = re.sub('([a-z])([A-Z])', r'\1-\2', name)
# Rename Intel downloads
# e.g. l_daal, l_ipp, l_mkl -> daal, ipp, mkl
if name.startswith('l_'):
name = name[2:]
# Convert UPPERCASE to lowercase
# e.g. SAMRAI -> samrai
name = name.lower()
# Replace '_' and '.' with '-'
# e.g. backports.ssl_match_hostname -> backports-ssl-match-hostname
name = name.replace('_', '-')
name = name.replace('.', '-')
# Replace "++" with "pp" and "+" with "-plus"
# e.g. gtk+ -> gtk-plus
# e.g. voro++ -> voropp
name = name.replace('++', 'pp')
name = name.replace('+', '-plus')
# Simplify Lua package names
# We don't want "lua" to occur multiple times in the name
name = re.sub('^(lua)([^-])', r'\1-\2', name)
# Simplify Bio++ package names
name = re.sub('^(bpp)([^-])', r'\1-\2', name)
return name
def valid_module_name(mod_name):
"""Return whether mod_name is valid for use in Spack."""
return bool(re.match(_valid_module_re, mod_name))

12
lib/spack/spack/util/web.py
View File

@@ -268,6 +268,14 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0):
# part, not the full path.
url_regex = os.path.basename(url_regex)
# We need to add a / to the beginning of the regex to prevent
# Spack from picking up similarly named packages like:
# https://cran.r-project.org/src/contrib/pls_2.6-0.tar.gz
# https://cran.r-project.org/src/contrib/enpls_5.7.tar.gz
# https://cran.r-project.org/src/contrib/autopls_1.3.tar.gz
# https://cran.r-project.org/src/contrib/matrixpls_1.0.4.tar.gz
url_regex = '/' + url_regex
# We need to add a $ anchor to the end of the regex to prevent
# Spack from picking up signature files like:
# .asc
@@ -275,7 +283,9 @@ def find_versions_of_archive(archive_urls, list_url=None, list_depth=0):
# .sha256
# .sig
# However, SourceForge downloads still need to end in '/download'.
regexes.append(url_regex + '(\/download)?$')
url_regex += '(\/download)?$'
regexes.append(url_regex)
# Build a dict version -> URL from any links that match the wildcards.
versions = {}

29
lib/spack/spack/version.py
View File

@@ -194,35 +194,6 @@ def satisfies(self, other):
nother = len(other.version)
return nother <= nself and self.version[:nother] == other.version
def wildcard(self):
"""Create a regex that will match variants of this version string."""
def a_or_n(seg):
if type(seg) == int:
return r'[0-9]+'
else:
return r'[a-zA-Z]+'
version = self.version
# Use a wildcard for separators, in case a version is written
# two different ways (e.g., boost writes 1_55_0 and 1.55.0)
sep_re = '[_.-]'
separators = ('',) + (sep_re,) * len(self.separators)
version += (version[-1],) * 2
separators += (sep_re,) * 2
segments = [a_or_n(seg) for seg in version]
wc = segments[0]
for i in range(1, len(separators)):
wc += '(?:' + separators[i] + segments[i]
# Add possible alpha or beta indicator at the end of each segemnt
# We treat these specially b/c they're so common.
wc += '(?:[a-z]|alpha|beta)?)?' * (len(segments) - 1)
return wc
def __iter__(self):
return iter(self.version)