Rework Spack's Mercurial support (#3834)

* Add tests to mercurial package

* Add support for --insecure with mercurial fetching

* Install man pages and tab-completion scripts

* Add tests and latest version for all deps

* Flake8 fix

* Use certifi module to find CA certificate

* Flake8 fix

* Unset PYTHONPATH when running hg

* svn_fetch should use to svn-test, not hg-test

* Drop Python 3 support in Mercurial

Python 3 support is a work in progress and isn't currently
recommended:

https://www.mercurial-scm.org/wiki/SupportedPythonVersions

* Test both secure and insecure hg fetching

* Test both secure and insecure git and svn fetching
This commit is contained in:
Adam J. Stewart 2017-04-19 20:59:04 -05:00 committed by Todd Gamblin
parent 53763f7698
commit f4858cb7a7
15 changed files with 176 additions and 52 deletions

View File

@ -808,8 +808,17 @@ def __init__(self, **kwargs):
@property @property
def hg(self): def hg(self):
""":returns: The hg executable
:rtype: Executable
"""
if not self._hg: if not self._hg:
self._hg = which('hg', required=True) self._hg = which('hg', required=True)
# When building PythonPackages, Spack automatically sets
# PYTHONPATH. This can interfere with hg, which is a Python
# script. Unset PYTHONPATH while running hg.
self._hg.add_default_env('PYTHONPATH', '')
return self._hg return self._hg
@property @property
@ -829,9 +838,15 @@ def fetch(self):
args.append('at revision %s' % self.revision) args.append('at revision %s' % self.revision)
tty.msg("Trying to clone Mercurial repository:", self.url, *args) tty.msg("Trying to clone Mercurial repository:", self.url, *args)
args = ['clone', self.url] args = ['clone']
if spack.insecure:
args.append('--insecure')
args.append(self.url)
if self.revision: if self.revision:
args += ['-r', self.revision] args.extend(['-r', self.revision])
self.hg(*args) self.hg(*args)

View File

@ -37,8 +37,15 @@ def type_of_test(request):
return request.param return request.param
@pytest.fixture(params=[True, False])
def secure(request):
"""Attempt both secure and insecure fetching"""
return request.param
def test_fetch( def test_fetch(
type_of_test, type_of_test,
secure,
mock_git_repository, mock_git_repository,
config, config,
refresh_builtin_mock refresh_builtin_mock
@ -62,7 +69,12 @@ def test_fetch(
pkg.versions[ver('git')] = t.args pkg.versions[ver('git')] = t.args
# Enter the stage directory and check some properties # Enter the stage directory and check some properties
with pkg.stage: with pkg.stage:
try:
spack.insecure = secure
pkg.do_stage() pkg.do_stage()
finally:
spack.insecure = False
assert h('HEAD') == h(t.revision) assert h('HEAD') == h(t.revision)
file_path = join_path(pkg.stage.source_path, t.file) file_path = join_path(pkg.stage.source_path, t.file)

View File

@ -37,8 +37,15 @@ def type_of_test(request):
return request.param return request.param
@pytest.fixture(params=[True, False])
def secure(request):
"""Attempt both secure and insecure fetching"""
return request.param
def test_fetch( def test_fetch(
type_of_test, type_of_test,
secure,
mock_hg_repository, mock_hg_repository,
config, config,
refresh_builtin_mock refresh_builtin_mock
@ -62,7 +69,12 @@ def test_fetch(
pkg.versions[ver('hg')] = t.args pkg.versions[ver('hg')] = t.args
# Enter the stage directory and check some properties # Enter the stage directory and check some properties
with pkg.stage: with pkg.stage:
try:
spack.insecure = secure
pkg.do_stage() pkg.do_stage()
finally:
spack.insecure = False
assert h() == t.revision assert h() == t.revision
file_path = join_path(pkg.stage.source_path, t.file) file_path = join_path(pkg.stage.source_path, t.file)

View File

@ -33,12 +33,19 @@
@pytest.fixture(params=['default', 'rev0']) @pytest.fixture(params=['default', 'rev0'])
def type_of_test(request): def type_of_test(request):
"""Returns one of the test type available for the mock_hg_repository""" """Returns one of the test type available for the mock_svn_repository"""
return request.param
@pytest.fixture(params=[True, False])
def secure(request):
"""Attempt both secure and insecure fetching"""
return request.param return request.param
def test_fetch( def test_fetch(
type_of_test, type_of_test,
secure,
mock_svn_repository, mock_svn_repository,
config, config,
refresh_builtin_mock refresh_builtin_mock
@ -56,13 +63,18 @@ def test_fetch(
t = mock_svn_repository.checks[type_of_test] t = mock_svn_repository.checks[type_of_test]
h = mock_svn_repository.hash h = mock_svn_repository.hash
# Construct the package under test # Construct the package under test
spec = Spec('hg-test') spec = Spec('svn-test')
spec.concretize() spec.concretize()
pkg = spack.repo.get(spec, new=True) pkg = spack.repo.get(spec, new=True)
pkg.versions[ver('hg')] = t.args pkg.versions[ver('svn')] = t.args
# Enter the stage directory and check some properties # Enter the stage directory and check some properties
with pkg.stage: with pkg.stage:
try:
spack.insecure = secure
pkg.do_stage() pkg.do_stage()
finally:
spack.insecure = False
assert h() == t.revision assert h() == t.revision
file_path = join_path(pkg.stage.source_path, t.file) file_path = join_path(pkg.stage.source_path, t.file)

View File

@ -23,16 +23,23 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
from spack import * from spack import *
import llnl.util.tty as tty from llnl.util import tty
import os
class Mercurial(Package): class Mercurial(PythonPackage):
"""Mercurial is a free, distributed source control management tool.""" """Mercurial is a free, distributed source control management tool."""
homepage = "https://www.mercurial-scm.org" homepage = "https://www.mercurial-scm.org"
url = "https://www.mercurial-scm.org/release/mercurial-3.9.tar.gz" url = "https://www.mercurial-scm.org/release/mercurial-4.1.2.tar.gz"
import_modules = [
'hgext', 'hgext3rd', 'mercurial', 'hgext.convert', 'hgext.fsmonitor',
'hgext.highlight', 'hgext.largefiles', 'hgext.zeroconf',
'hgext.fsmonitor.pywatchman', 'mercurial.hgweb',
'mercurial.httpclient', 'mercurial.pure'
]
version('4.1.2', '934c99808bdc8385e074b902d59b0d93')
version('3.9.1', '3759dd10edb8c1a6dfb8ff0ce82658ce') version('3.9.1', '3759dd10edb8c1a6dfb8ff0ce82658ce')
version('3.9', 'e2b355da744e94747daae3a5339d28a0') version('3.9', 'e2b355da744e94747daae3a5339d28a0')
version('3.8.4', 'cec2c3db688cb87142809089c6ae13e9') version('3.8.4', 'cec2c3db688cb87142809089c6ae13e9')
@ -40,30 +47,62 @@ class Mercurial(Package):
version('3.8.2', 'c38daa0cbe264fc621dc3bb05933b0b3') version('3.8.2', 'c38daa0cbe264fc621dc3bb05933b0b3')
version('3.8.1', '172a8c588adca12308c2aca16608d7f4') version('3.8.1', '172a8c588adca12308c2aca16608d7f4')
extends('python')
depends_on('python@2.6:2.8') depends_on('python@2.6:2.8')
depends_on('py-docutils', type='build') depends_on('py-docutils', type='build')
depends_on('py-pygments', type=('build', 'run'))
depends_on('py-certifi', type=('build', 'run'))
def install(self, spec, prefix): @run_after('install')
make('install', 'PREFIX={0}'.format(prefix)) def post_install(self):
prefix = self.prefix
# Configuration of HTTPS certificate authorities # Install man pages
# https://www.mercurial-scm.org/wiki/CACertificates mkdirp(prefix.man1)
hgrc_filename = join_path(prefix.etc, 'mercurial', 'hgrc') mkdirp(prefix.man5)
mkdirp(os.path.dirname(hgrc_filename)) mkdirp(prefix.man8)
with working_dir('doc'):
install('hg.1', prefix.man1)
install('hgignore.5', prefix.man5)
install('hgrc.5', prefix.man5)
install('hg-ssh.8', prefix.man8)
with open(hgrc_filename, 'w') as hgrc: # Install completion scripts
if os.path.exists('/etc/ssl/certs/ca-certificates.crt'): contrib = join_path(prefix, 'contrib')
# Debian/Ubuntu/Gentoo/Arch Linux mkdir(contrib)
hgrc.write('[web]\ncacerts = /etc/ssl/certs/ca-certificates.crt') # noqa with working_dir('contrib'):
elif os.path.exists('/etc/pki/tls/certs/ca-bundle.crt'): install('bash_completion', join_path(contrib, 'bash_completion'))
# Fedora/RHEL/CentOS install('zsh_completion', join_path(contrib, 'zsh_completion'))
hgrc.write('[web]\ncacerts = /etc/pki/tls/certs/ca-bundle.crt')
elif os.path.exists('/etc/ssl/ca-bundle.pem'): @run_after('install')
# openSUSE/SLE def configure_certificates(self):
hgrc.write('[web]\ncacerts = /etc/ssl/ca-bundle.pem') """Configuration of HTTPS certificate authorities
else: https://www.mercurial-scm.org/wiki/CACertificates"""
etc_dir = join_path(self.prefix.etc, 'mercurial')
mkdirp(etc_dir)
hgrc_filename = join_path(etc_dir, 'hgrc')
# Use certifi to find the location of the CA certificate
certificate = python('-c', 'import certifi; print certifi.where()',
output=str)
if not certificate:
tty.warn('CA certificate not found. You may not be able to ' tty.warn('CA certificate not found. You may not be able to '
'connect to an HTTPS server. If your CA certificate ' 'connect to an HTTPS server. If your CA certificate '
'is in a non-standard location, you should add it to ' 'is in a non-standard location, you should add it to '
'{0}'.format(hgrc_filename)) '{0}.'.format(hgrc_filename))
# Write the global mercurial configuration file
with open(hgrc_filename, 'w') as hgrc:
hgrc.write('[web]\ncacerts = {0}'.format(certificate))
@run_after('install')
@on_package_attributes(run_tests=True)
def check_install(self):
"""Sanity-check setup."""
hg = Executable(join_path(self.prefix.bin, 'hg'))
hg('debuginstall')
hg('version')

View File

@ -30,8 +30,11 @@ class PyAppdirs(PythonPackage):
dirs, e.g. a "user data dir".""" dirs, e.g. a "user data dir"."""
homepage = "https://github.com/ActiveState/appdirs" homepage = "https://github.com/ActiveState/appdirs"
url = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.0.tar.gz" url = "https://pypi.io/packages/source/a/appdirs/appdirs-1.4.3.tar.gz"
import_modules = ['appdirs']
version('1.4.3', '44c679904082a2133f5566c8a0d3ab42')
version('1.4.0', '1d17b4c9694ab84794e228f28dc3275b') version('1.4.0', '1d17b4c9694ab84794e228f28dc3275b')
patch('setuptools-import.patch', when='@:1.4.0') patch('setuptools-import.patch', when='@:1.4.0')

View File

@ -29,9 +29,12 @@ class PyCertifi(PythonPackage):
"""Certifi: A carefully curated collection of Root Certificates for validating """Certifi: A carefully curated collection of Root Certificates for validating
the trustworthiness of SSL certificates while verifying the identity of TLS the trustworthiness of SSL certificates while verifying the identity of TLS
hosts.""" hosts."""
homepage = "https://github.com/certifi/python-certifi" homepage = "http://certifi.io/"
url = "https://github.com/certifi/python-certifi/archive/2016.02.28.tar.gz" url = "https://pypi.io/packages/source/c/certifi/certifi-2017.1.23.tar.gz"
import_modules = ['certifi']
version('2017.1.23', 'b72521a8badff5e89a8eabea586d79ab')
version('2016.02.28', '5ccfc23bd5e931863f0b01ef3e9d2dbd3bef0e1b') version('2016.02.28', '5ccfc23bd5e931863f0b01ef3e9d2dbd3bef0e1b')
depends_on('py-setuptools', type='build') depends_on('py-setuptools', type='build')

View File

@ -23,14 +23,17 @@
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
############################################################################## ##############################################################################
from spack import * from spack import *
import sys
class PyCffi(PythonPackage): class PyCffi(PythonPackage):
"""Foreign Function Interface for Python calling C code""" """Foreign Function Interface for Python calling C code"""
homepage = "http://cffi.readthedocs.org/en/latest/" homepage = "http://cffi.readthedocs.org/en/latest/"
# base https://pypi.python.org/pypi/cffi url = "https://pypi.io/packages/source/c/cffi/cffi-1.10.0.tar.gz"
url = "https://pypi.python.org/packages/source/c/cffi/cffi-1.1.2.tar.gz"
import_modules = ['cffi']
version('1.10.0', '2b5fa41182ed0edaf929a789e602a070')
version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea') version('1.1.2', 'ca6e6c45b45caa87aee9adc7c796eaea')
depends_on('py-setuptools', type='build') depends_on('py-setuptools', type='build')
@ -44,4 +47,5 @@ def setup_environment(self, spack_env, run_env):
# other compilation. We are setting the 'LDSHARED" to the # other compilation. We are setting the 'LDSHARED" to the
# spack compiler wrapper plus a few extra flags necessary for # spack compiler wrapper plus a few extra flags necessary for
# building the shared library. # building the shared library.
if not sys.platform == 'darwin':
spack_env.set('LDSHARED', "{0} -shared -pthread".format(spack_cc)) spack_env.set('LDSHARED', "{0} -shared -pthread".format(spack_cc))

View File

@ -33,8 +33,18 @@ class PyDocutils(PythonPackage):
markup language.""" markup language."""
homepage = "http://docutils.sourceforge.net/" homepage = "http://docutils.sourceforge.net/"
url = "https://pypi.python.org/packages/source/d/docutils/docutils-0.12.tar.gz" url = "https://pypi.io/packages/source/d/docutils/docutils-0.13.1.tar.gz"
version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53', import_modules = [
url="https://pypi.python.org/packages/05/25/7b5484aca5d46915493f1fd4ecb63c38c333bd32aa9ad6e19da8d08895ae/docutils-0.13.1.tar.gz") 'docutils', 'docutils.languages', 'docutils.parsers',
'docutils.readers', 'docutils.transforms', 'docutils.utils',
'docutils.writers', 'docutils.parsers.rst',
'docutils.parsers.rst.directives', 'docutils.parsers.rst.languages',
'docutils.utils.math', 'docutils.writers.html4css1',
'docutils.writers.html5_polyglot', 'docutils.writers.latex2e',
'docutils.writers.odf_odt', 'docutils.writers.pep_html',
'docutils.writers.s5_html', 'docutils.writers.xetex'
]
version('0.13.1', 'ea4a893c633c788be9b8078b6b305d53')
version('0.12', '4622263b62c5c771c03502afa3157768') version('0.12', '4622263b62c5c771c03502afa3157768')

View File

@ -31,6 +31,8 @@ class PyPackaging(PythonPackage):
homepage = "https://github.com/pypa/packaging" homepage = "https://github.com/pypa/packaging"
url = "https://pypi.io/packages/source/p/packaging/packaging-16.8.tar.gz" url = "https://pypi.io/packages/source/p/packaging/packaging-16.8.tar.gz"
import_modules = ['packaging']
version('16.8', '53895cdca04ecff80b54128e475b5d3b') version('16.8', '53895cdca04ecff80b54128e475b5d3b')
# Not needed for the installation, but used at runtime # Not needed for the installation, but used at runtime

View File

@ -28,8 +28,11 @@
class PyPycparser(PythonPackage): class PyPycparser(PythonPackage):
"""A complete parser of the C language, written in pure python.""" """A complete parser of the C language, written in pure python."""
homepage = "https://github.com/eliben/pycparser" homepage = "https://github.com/eliben/pycparser"
url = "https://pypi.python.org/packages/source/p/pycparser/pycparser-2.13.tar.gz" url = "https://pypi.io/packages/source/p/pycparser/pycparser-2.17.tar.gz"
import_modules = ['pycparser', 'pycparser.ply']
version('2.17', 'ca98dcb50bc1276f230118f6af5a40c7')
version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f') version('2.13', 'e4fe1a2d341b22e25da0d22f034ef32f')
depends_on('py-setuptools', type='build') depends_on('py-setuptools', type='build')

View File

@ -29,8 +29,14 @@ class PyPygments(PythonPackage):
"""Pygments is a syntax highlighting package written in Python.""" """Pygments is a syntax highlighting package written in Python."""
homepage = "https://pypi.python.org/pypi/pygments" homepage = "https://pypi.python.org/pypi/pygments"
url = "https://pypi.python.org/packages/source/P/Pygments/Pygments-2.0.1.tar.gz" url = "https://pypi.io/packages/source/P/Pygments/Pygments-2.2.0.tar.gz"
import_modules = [
'pygments', 'pygments.filters', 'pygments.formatters',
'pygments.lexers', 'pygments.styles'
]
version('2.2.0', '13037baca42f16917cbd5ad2fab50844')
version('2.1.3', 'ed3fba2467c8afcda4d317e4ef2c6150') version('2.1.3', 'ed3fba2467c8afcda4d317e4ef2c6150')
version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c') version('2.0.1', 'e0daf4c14a4fe5b630da765904de4d6c')
version('2.0.2', '238587a1370d62405edabd0794b3ec4a') version('2.0.2', '238587a1370d62405edabd0794b3ec4a')

View File

@ -28,8 +28,11 @@
class PyPyparsing(PythonPackage): class PyPyparsing(PythonPackage):
"""A Python Parsing Module.""" """A Python Parsing Module."""
homepage = "https://pypi.python.org/pypi/pyparsing" homepage = "https://pypi.python.org/pypi/pyparsing"
url = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.0.3.tar.gz" url = "https://pypi.io/packages/source/p/pyparsing/pyparsing-2.2.0.tar.gz"
import_modules = ['pyparsing']
version('2.2.0', '0214e42d63af850256962b6744c948d9')
version('2.1.10', '065908b92904e0d3634eb156f44cc80e') version('2.1.10', '065908b92904e0d3634eb156f44cc80e')
version('2.0.3', '0fe479be09fc2cf005f753d3acc35939') version('2.0.3', '0fe479be09fc2cf005f753d3acc35939')

View File

@ -32,12 +32,10 @@ class PySetuptools(PythonPackage):
homepage = "https://pypi.python.org/pypi/setuptools" homepage = "https://pypi.python.org/pypi/setuptools"
url = "https://pypi.io/packages/source/s/setuptools/setuptools-25.2.0.tar.gz" url = "https://pypi.io/packages/source/s/setuptools/setuptools-25.2.0.tar.gz"
import_modules = [ import_modules = ['pkg_resources', 'setuptools', 'setuptools.command']
'pkg_resources', 'setuptools', 'pkg_resources.extern',
'pkg_resources._vendor', 'pkg_resources._vendor.packaging',
'setuptools.extern', 'setuptools.command'
]
version('34.4.1', '5f9b07aeaafd29eac2548fc0b89a4934',
url="https://pypi.io/packages/source/s/setuptools/setuptools-34.4.1.zip")
version('34.2.0', '41b630da4ea6cfa5894d9eb3142922be', version('34.2.0', '41b630da4ea6cfa5894d9eb3142922be',
url="https://pypi.io/packages/source/s/setuptools/setuptools-34.2.0.zip") url="https://pypi.io/packages/source/s/setuptools/setuptools-34.2.0.zip")
version('25.2.0', 'a0dbb65889c46214c691f6c516cf959c') version('25.2.0', 'a0dbb65889c46214c691f6c516cf959c')

View File

@ -29,7 +29,9 @@ class PySix(PythonPackage):
"""Python 2 and 3 compatibility utilities.""" """Python 2 and 3 compatibility utilities."""
homepage = "https://pypi.python.org/pypi/six" homepage = "https://pypi.python.org/pypi/six"
url = "https://pypi.io/packages/source/s/six/six-1.9.0.tar.gz" url = "https://pypi.io/packages/source/s/six/six-1.10.0.tar.gz"
import_modules = ['six']
version('1.10.0', '34eed507548117b2ab523ab14b2f8b55') version('1.10.0', '34eed507548117b2ab523ab14b2f8b55')
version('1.9.0', '476881ef4012262dfc8adc645ee786c4') version('1.9.0', '476881ef4012262dfc8adc645ee786c4')