zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tests and bugfixes

Browse Source
This commit is contained in:
Gregory Becker
2020-03-26 16:01:03 -07:00
committed by Tamara Dahlgren
parent 3ce2efe32a
commit 0dc212e67d
16 changed files with 404 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
lib/spack/spack/build_environment.py
View File

@@ -55,6 +55,7 @@
import spack.schema.environment
import spack.store
import spack.architecture as arch
import spack.util.path
from spack.util.string import plural
from spack.util.environment import (
env_flag, filter_system_paths, get_path, is_system_path,
@@ -820,7 +821,7 @@ def modifications_from_dependencies(spec, context):
return env
def fork(pkg, function, dirty, fake, context='build'):
def fork(pkg, function, dirty, fake, context='build', **kwargs):
"""Fork a child process to do part of a spack build.
Args:
@@ -860,9 +861,6 @@ def child_process(child_pipe, input_stream):
if input_stream is not None:
sys.stdin = input_stream
# Record starting directory
start_dir = os.getcwd()
try:
if not fake:
setup_package(pkg, dirty=dirty, context=context)
@@ -891,7 +889,11 @@ def child_process(child_pipe, input_stream):
test_log = None
if context == 'test':
test_log = os.path.join(start_dir, pkg.test_log_name)
test_log = os.path.join(
spack.util.path.canonicalize_path(
spack.config.get('config:test_stage')),
kwargs.get('test_name'),
pkg.test_log_name)
# make a pickleable exception to send to parent.
msg = "%s: %s" % (exc_type.__name__, str(exc))
@@ -983,12 +985,22 @@ def make_stack(tb, stack=None):
if isinstance(obj, spack.package.PackageBase):
break
# Determine whether we are in a package file
# Package files are named `package.py` and are not in the lib/spack/spack
# Package files have a line added at import time, so we adjust the lineno
# when we are getting context from a package file instead of a base class
filename = inspect.getfile(frame.f_code)
packagebase_filename = inspect.getfile(spack.package.PackageBase)
in_package = (filename != packagebase_filename and
os.path.basename(filename) == 'package.py')
adjust = 0 if in_package else 1
# We found obj, the Package implementation we care about.
# Point out the location in the install method where we failed.
lines = [
'{0}:{1:d}, in {2}:'.format(
inspect.getfile(frame.f_code),
frame.f_lineno - 1, # subtract 1 because f_lineno is 0-indexed
filename,
tb.tb_lineno - adjust, # adjust for import mangling
frame.f_code.co_name
)
]
@@ -997,8 +1009,8 @@ def make_stack(tb, stack=None):
sourcelines, start = inspect.getsourcelines(frame)
# Calculate lineno of the error relative to the start of the function.
# Subtract 1 because f_lineno is 0-indexed.
fun_lineno = frame.f_lineno - start - 1
# Adjust for import mangling of package files.
fun_lineno = tb.tb_lineno - start - adjust
start_ctx = max(0, fun_lineno - context)
sourcelines = sourcelines[start_ctx:fun_lineno + context + 1]
@@ -1098,6 +1110,7 @@ def write_log_summary(log_type, log):
if self.build_log and os.path.exists(self.build_log):
write_log_summary('build', self.build_log)
if self.test_log and os.path.exists(self.test_log):
write_log_summary('test', self.test_log)

15
lib/spack/spack/cmd/test.py
View File

@@ -23,8 +23,8 @@
def setup_parser(subparser):
subparser.add_argument('--keep-tmpdir', action='store_true',
help='Keep testing directory for debuggin')
subparser.add_argument('--keep-stage', action='store_true',
help='Keep testing directory for debugging')
subparser.add_argument(
'--log-format',
default=None,
@@ -107,13 +107,20 @@ def test(parser, args):
reporter.filename = log_file
reporter.specs = specs_to_test
with reporter('test'):
# test_stage_dir
test_name = now.strftime('%Y-%m-%d_%H:%M:%S')
stage = os.path.join(
spack.util.path.canonicalize_path(
spack.config.get('config:test_stage', os.getcwd())),
test_name)
with reporter('test', stage):
if args.smoke_test:
for spec in specs_to_test:
try:
spec.package.do_test(
time=now,
remove_directory=not args.keep_tmpdir,
remove_directory=not args.keep_stage,
dirty=args.dirty)
except BaseException as e:
pass # Test is logged, go on to other tests

20
lib/spack/spack/error.py
View File

@@ -16,6 +16,26 @@
debug = False
def re_raise(e, exc_info=None):
"""Re-raise an exception with it's original context."""
exc_type, context, tb = sys.exc_info()
# construct arguments to re-raise error from type
args = []
if hasattr(e, 'message'):
args.append(e.message)
if hasattr(e, 'long_message'):
args.append(e.long_message)
if sys.version_info[0] < 3:
# ugly hack: exec to avoid the fact this is a syntax
# error in python 3
exec("raise exc_type(*args), None, tb",
globals(), locals())
else:
raise exc_type(*args).with_traceback(tb)
class SpackError(Exception):
"""This is the superclass for all Spack errors.
Subclasses can be found in the modules they have to do with.

54
lib/spack/spack/package.py
View File

@@ -1612,16 +1612,17 @@ def do_test(self, time, remove_directory=False, dirty=False):
self.spec.compiler)
return
test_name = time.strftime('%Y-%m-%d_%H:%M:%S')
test_stage = Prefix(os.path.join(
sup.canonicalize_path(
spack.config.get('config:test_stage', os.getcwd())),
time.strftime('%Y-%m-%d_%H:%M:%S')))
test_name))
if not os.path.exists(test_stage):
mkdirp(test_stage)
test_log_file = os.path.join(test_stage, self.test_log_name)
def test_process():
with log_output(test_log_file) as logger:
with tty.log.log_output(test_log_file) as logger:
with logger.force_echo():
tty.msg('Testing package %s' %
self.spec.format('{name}-{hash:7}'))
@@ -1647,40 +1648,28 @@ def test_process():
except Exception as e:
# Catch the error and print a summary to the log file
# so that cdash and junit reporters know about it
exc_type, context, traceback = sys.exc_info()
exc_info = sys.exc_info()
print('Error: %s' % e)
# construct arguments to re-raise error from type
args = []
if hasattr(e, 'message'):
args.append(e.message)
if hasattr(e, 'long_message'):
args.append(e.long_message)
if sys.version_info[0] < 3:
# ugly hack: exec to avoid the fact this is a syntax
# error in python 3
exec("raise exc_type(*args), None, traceback",
globals(), locals())
else:
raise exc_type(*args).with_traceback(traceback)
finally:
# reset debug level
tty.set_debug(old_debug)
# cleanup test directory
import traceback
traceback.print_tb(exc_info[2])
spack.error.re_raise(e)
else:
# cleanup test directory on success
if remove_directory:
shutil.rmtree(testdir)
if not os.listdir(test_stage):
shutil.rmtree(test_stage)
finally:
# reset debug level
tty.set_debug(old_debug)
spack.build_environment.fork(
self, test_process, dirty=dirty, fake=False, context='test')
self, test_process, dirty=dirty, fake=False, context='test', test_name=test_name)
def test(self):
pass
def run_test(self, exe, options, expected, status):
def run_test(self, exe, options=[], expected=[], status=None):
"""Run the test and confirm obtain the expected results
Args:
@@ -1698,15 +1687,18 @@ def run_test(self, exe, options, expected, status):
try:
output = runner(*options, output=str.split, error=str.split)
assert not status, 'Expected execution to fail'
for check in expected:
cmd = ' '.join([exe] + options)
msg = "Expected '%s' in output of `%s`" % (check, cmd)
# msg += '\n%s' % output
assert check in output, msg
except ProcessError as err:
output = str(err)
status_msg = 'exited with status {0}'.format(status)
expected_msg = 'Expected \'{0}\' in \'{1}\''.format(
status_msg, err.message)
assert status_msg in output, expected_msg
for check in expected:
assert check in output
if status_msg not in output:
spack.error.re_raise(err)
def unit_test_check(self):
"""Hook for unit tests to assert things about package internals.

4
lib/spack/spack/report.py
View File

@@ -272,9 +272,9 @@ def __init__(self, cls, function, format_name, args):
.format(self.format_name))
self.report_writer = report_writers[self.format_name](args)
def __call__(self, type):
def __call__(self, type, dir=os.getcwd()):
self.type = type
self.dir = os.getcwd()
self.dir = dir
return self
def concretization_report(self, msg):

5
lib/spack/spack/reporters/cdash.py
View File

@@ -72,8 +72,10 @@ def __init__(self, args):
tty.verbose("Using CDash auth token from environment")
self.authtoken = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
if args.spec:
if getattr(args, 'spec', ''):
packages = args.spec
elif getattr(args, 'specs', ''):
packages = args.specs
else:
packages = []
for file in args.specfiles:
@@ -262,6 +264,7 @@ def test_report_for_package(self, directory_name, package, duration):
self.current_package_name = package['name']
self.buildname = "{0} - {1}".format(
self.base_buildname, package['name'])
report_data = self.initialize_report(directory_name)
for phase in ('test', 'update'):

234
lib/spack/spack/test/cmd/test.py
View File

@@ -3,93 +3,191 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import argparse
import os
import filecmp
import re
from six.moves import builtins
import time
import pytest
import llnl.util.filesystem as fs
import spack.config
import spack.hash_types as ht
import spack.package
import spack.cmd.install
from spack.error import SpackError
from spack.spec import Spec
from spack.main import SpackCommand
import spack.environment as ev
spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/test.py'
from six.moves.urllib.error import HTTPError, URLError
install = SpackCommand('install')
spack_test = SpackCommand('test')
def test_list():
output = spack_test('--list')
assert "test.py" in output
assert "spec_semantics.py" in output
assert "test_list" not in output
@pytest.fixture()
def mock_test_stage(mutable_config, tmpdir):
# NOTE: This fixture MUST be applied after any fixture that uses
# the config fixture under the hood
# No need to unset because we use mutable_config
tmp_stage = str(tmpdir.join('test_stage'))
mutable_config.set('config:test_stage', tmp_stage)
yield tmp_stage
def test_list_with_pytest_arg():
output = spack_test('--list', cmd_test_py)
assert output.strip() == cmd_test_py
def test_test_package_not_installed(
tmpdir, mock_packages, mock_archive, mock_fetch, config,
install_mockery):
output = spack_test('libdwarf')
assert "No installed packages match spec libdwarf" in output
def test_list_with_keywords():
output = spack_test('--list', '-k', 'cmd/test.py')
assert output.strip() == cmd_test_py
@pytest.mark.parametrize('arguments,expected', [
([], spack.config.get('config:dirty')), # default from config file
(['--clean'], False),
(['--dirty'], True),
])
def test_test_dirty_flag(arguments, expected):
parser = argparse.ArgumentParser()
spack.cmd.test.setup_parser(parser)
args = parser.parse_args(arguments)
assert args.dirty == expected
def test_list_long(capsys):
with capsys.disabled():
output = spack_test('--list-long')
assert "test.py::\n" in output
assert "test_list" in output
assert "test_list_with_pytest_arg" in output
assert "test_list_with_keywords" in output
assert "test_list_long" in output
assert "test_list_long_with_pytest_arg" in output
assert "test_list_names" in output
assert "test_list_names_with_pytest_arg" in output
def test_test_output(install_mockery, mock_archive, mock_fetch, mock_test_stage):
"""Ensure output printed from pkgs is captured by output redirection."""
install('printing-package')
spack_test('printing-package')
assert "spec_dag.py::\n" in output
assert 'test_installed_deps' in output
assert 'test_test_deptype' in output
contents = os.listdir(mock_test_stage)
assert len(contents) == 1
testdir = os.path.join(mock_test_stage, contents[0])
contents = os.listdir(testdir)
assert len(contents) == 1
outfile = os.path.join(testdir, contents[0])
with open(outfile, 'r') as f:
output = f.read()
assert "BEFORE TEST" in output
assert "true: expect to succeed" in output
assert "AFTER TEST" in output
assert "rror" not in output # no error
def test_list_long_with_pytest_arg(capsys):
with capsys.disabled():
output = spack_test('--list-long', cmd_test_py)
assert "test.py::\n" in output
assert "test_list" in output
assert "test_list_with_pytest_arg" in output
assert "test_list_with_keywords" in output
assert "test_list_long" in output
assert "test_list_long_with_pytest_arg" in output
assert "test_list_names" in output
assert "test_list_names_with_pytest_arg" in output
def test_test_output_on_error(mock_packages, mock_archive, mock_fetch,
install_mockery, capfd, mock_test_stage):
install('test-error')
# capfd interferes with Spack's capturing
with capfd.disabled():
out = spack_test('test-error', fail_on_error=False)
assert "spec_dag.py::\n" not in output
assert 'test_installed_deps' not in output
assert 'test_test_deptype' not in output
assert "ProcessError: Command exited with status 1" in out
def test_list_names():
output = spack_test('--list-names')
assert "test.py::test_list\n" in output
assert "test.py::test_list_with_pytest_arg\n" in output
assert "test.py::test_list_with_keywords\n" in output
assert "test.py::test_list_long\n" in output
assert "test.py::test_list_long_with_pytest_arg\n" in output
assert "test.py::test_list_names\n" in output
assert "test.py::test_list_names_with_pytest_arg\n" in output
def test_test_output_on_failure(mock_packages, mock_archive, mock_fetch,
install_mockery, capfd, mock_test_stage):
install('test-fail')
with capfd.disabled():
out = spack_test('test-fail', fail_on_error=False)
assert "spec_dag.py::test_installed_deps\n" in output
assert 'spec_dag.py::test_test_deptype\n' in output
assert "Expected 'not in the output' in output of `true`" in out
def test_list_names_with_pytest_arg():
output = spack_test('--list-names', cmd_test_py)
assert "test.py::test_list\n" in output
assert "test.py::test_list_with_pytest_arg\n" in output
assert "test.py::test_list_with_keywords\n" in output
assert "test.py::test_list_long\n" in output
assert "test.py::test_list_long_with_pytest_arg\n" in output
assert "test.py::test_list_names\n" in output
assert "test.py::test_list_names_with_pytest_arg\n" in output
def test_show_log_on_error(mock_packages, mock_archive, mock_fetch,
install_mockery, capfd, mock_test_stage):
"""Make sure spack prints location of test log on failure."""
install('test-error')
with capfd.disabled():
out = spack_test('test-error', fail_on_error=False)
assert "spec_dag.py::test_installed_deps\n" not in output
assert 'spec_dag.py::test_test_deptype\n' not in output
assert 'See test log' in out
assert mock_test_stage in out
def test_pytest_help():
output = spack_test('--pytest-help')
assert "-k EXPRESSION" in output
assert "pytest-warnings:" in output
assert "--collect-only" in output
@pytest.mark.usefixtures(
'mock_packages', 'mock_archive', 'mock_fetch', 'install_mockery', 'mock_test_stage'
)
@pytest.mark.parametrize('pkg_name,msgs', [
('test-error', ['Error: Command exited', 'ProcessError']),
('test-fail', ['Error: Expected', 'AssertionError'])
])
def test_junit_output_with_failures(tmpdir, pkg_name, msgs):
install(pkg_name)
with tmpdir.as_cwd():
spack_test(
'--log-format=junit', '--log-file=test.xml',
pkg_name)
files = tmpdir.listdir()
filename = tmpdir.join('test.xml')
assert filename in files
content = filename.open().read()
# Count failures and errors correctly
assert 'tests="1"' in content
assert 'failures="1"' in content
assert 'errors="0"' in content
# We want to have both stdout and stderr
assert '<system-out>' in content
for msg in msgs:
assert msg in content
def test_cdash_output_test_error(
tmpdir, mock_fetch, install_mockery, mock_packages, mock_archive,
mock_test_stage, capfd):
install('test-error')
with tmpdir.as_cwd():
spack_test(
'--log-format=cdash',
'--log-file=cdash_reports',
'test-error')
report_dir = tmpdir.join('cdash_reports')
print(tmpdir.listdir())
assert report_dir in tmpdir.listdir()
report_file = report_dir.join('test-error_Test.xml')
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert 'Error: Command exited with status 1' in content
def test_cdash_upload_clean_test(
tmpdir, mock_fetch, install_mockery, mock_packages, mock_archive,
mock_test_stage):
install('printing-package')
with tmpdir.as_cwd():
spack_test(
'--log-file=cdash_reports',
'--log-format=cdash',
'printing-package')
report_dir = tmpdir.join('cdash_reports')
assert report_dir in tmpdir.listdir()
report_file = report_dir.join('printing-package_Test.xml')
assert report_file in report_dir.listdir()
content = report_file.open().read()
assert '</Test>' in content
assert '<Text>' not in content
def test_test_help_does_not_show_cdash_options(capsys):
"""Make sure `spack test --help` does not describe CDash arguments"""
with pytest.raises(SystemExit):
spack_test('--help')
captured = capsys.readouterr()
assert 'CDash URL' not in captured.out
def test_test_help_cdash():
"""Make sure `spack test --help-cdash` describes CDash arguments"""
out = spack_test('--help-cdash')
assert 'CDash URL' in out

96
lib/spack/spack/test/cmd/unit_test.py Normal file
View File

@@ -0,0 +1,96 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.main import SpackCommand
spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/unit_test.py'
def test_list():
output = spack_test('--list')
assert "unit_test.py" in output
assert "spec_semantics.py" in output
assert "test_list" not in output
def test_list_with_pytest_arg():
output = spack_test('--list', cmd_test_py)
assert output.strip() == cmd_test_py
def test_list_with_keywords():
output = spack_test('--list', '-k', 'cmd/unit_test.py')
assert output.strip() == cmd_test_py
def test_list_long(capsys):
with capsys.disabled():
output = spack_test('--list-long')
assert "unit_test.py::\n" in output
assert "test_list" in output
assert "test_list_with_pytest_arg" in output
assert "test_list_with_keywords" in output
assert "test_list_long" in output
assert "test_list_long_with_pytest_arg" in output
assert "test_list_names" in output
assert "test_list_names_with_pytest_arg" in output
assert "spec_dag.py::\n" in output
assert 'test_installed_deps' in output
assert 'test_test_deptype' in output
def test_list_long_with_pytest_arg(capsys):
with capsys.disabled():
output = spack_test('--list-long', cmd_test_py)
print(output)
assert "unit_test.py::\n" in output
assert "test_list" in output
assert "test_list_with_pytest_arg" in output
assert "test_list_with_keywords" in output
assert "test_list_long" in output
assert "test_list_long_with_pytest_arg" in output
assert "test_list_names" in output
assert "test_list_names_with_pytest_arg" in output
assert "spec_dag.py::\n" not in output
assert 'test_installed_deps' not in output
assert 'test_test_deptype' not in output
def test_list_names():
output = spack_test('--list-names')
assert "unit_test.py::test_list\n" in output
assert "unit_test.py::test_list_with_pytest_arg\n" in output
assert "unit_test.py::test_list_with_keywords\n" in output
assert "unit_test.py::test_list_long\n" in output
assert "unit_test.py::test_list_long_with_pytest_arg\n" in output
assert "unit_test.py::test_list_names\n" in output
assert "unit_test.py::test_list_names_with_pytest_arg\n" in output
assert "spec_dag.py::test_installed_deps\n" in output
assert 'spec_dag.py::test_test_deptype\n' in output
def test_list_names_with_pytest_arg():
output = spack_test('--list-names', cmd_test_py)
assert "unit_test.py::test_list\n" in output
assert "unit_test.py::test_list_with_pytest_arg\n" in output
assert "unit_test.py::test_list_with_keywords\n" in output
assert "unit_test.py::test_list_long\n" in output
assert "unit_test.py::test_list_long_with_pytest_arg\n" in output
assert "unit_test.py::test_list_names\n" in output
assert "unit_test.py::test_list_names_with_pytest_arg\n" in output
assert "spec_dag.py::test_installed_deps\n" not in output
assert 'spec_dag.py::test_test_deptype\n' not in output
def test_pytest_help():
output = spack_test('--pytest-help')
assert "-k EXPRESSION" in output
assert "pytest-warnings:" in output
assert "--collect-only" in output

8
lib/spack/spack/test/conftest.py
View File

@@ -468,6 +468,14 @@ def mutable_config(tmpdir_factory, configuration_dir):
*[spack.config.ConfigScope(name, str(mutable_dir.join(name)))
for name in ['site', 'system', 'user']])
# Set variables needed for tests to run
# values should not be tied to any specific
# TODO: Fix this
cfg.set('config:build_jobs',
spack.config.get('config:build_jobs'))
cfg.set('config:template_dirs',
spack.config.get('config:template_dirs'))
with use_configuration(cfg):
yield cfg

2
lib/spack/spack/test/mirror.py
View File

@@ -16,7 +16,7 @@
from llnl.util.filesystem import resolve_link_target_relative_to_the_link
pytestmark = pytest.mark.usefixtures('config', 'mutable_mock_repo')
pytestmark = pytest.mark.usefixtures('mutable_config', 'mutable_mock_repo')
# paths in repos that shouldn't be in the mirror tarballs.
exclude = ['.hg', '.git', '.svn']

2
share/spack/spack-completion.bash
View File

@@ -1467,7 +1467,7 @@ _spack_stage() {
_spack_test() {
if $list_options
then
SPACK_COMPREPLY="-h --help --keep-tmpdir --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --smoke --capability --clean --dirty"
SPACK_COMPREPLY="-h --help --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --smoke --capability --clean --dirty"
else
_all_packages
fi

5
var/spack/repos/builtin.mock/packages/printing-package/package.py
View File

@@ -24,3 +24,8 @@ def install(self, spec, prefix):
make('install')
print("AFTER INSTALL")
def test(self):
print("BEFORE TEST")
self.run_test('true') # run /bin/true
print("AFTER TEST")

21
var/spack/repos/builtin.mock/packages/test-error/package.py Normal file
View File

@@ -0,0 +1,21 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class TestError(Package):
"""This package has a test method that fails in a subprocess."""
homepage = "http://www.example.com/test-failure"
url = "http://www.test-failure.test/test-failure-1.0.tar.gz"
version('1.0', 'foobarbaz')
def install(self, spec, prefix):
mkdirp(prefix.bin)
def test(self):
self.run_test('false')

21
var/spack/repos/builtin.mock/packages/test-fail/package.py Normal file
View File

@@ -0,0 +1,21 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class TestFail(Package):
"""This package has a test method that fails in a subprocess."""
homepage = "http://www.example.com/test-failure"
url = "http://www.test-failure.test/test-failure-1.0.tar.gz"
version('1.0', 'foobarbaz')
def install(self, spec, prefix):
mkdirp(prefix.bin)
def test(self):
self.run_test('true', expected=['not in the output'])

2
var/spack/repos/builtin/packages/python/package.py
View File

@@ -1111,6 +1111,8 @@ def remove_files_from_view(self, view, merge_map):
os.remove(dst)
def test(self):
self.run_test('true', expected=['not in output'])
# contains python executable
python = which('python')
assert os.path.dirname(python.path) == os.path.dirname(self.command.path)

1
var/spack/repos/builtin/packages/zlib/package.py
View File

@@ -44,6 +44,7 @@ def setup_build_environment(self, env):
env.append_flags('CFLAGS', '-O2')
def install(self, spec, prefix):
raise Exception
config_args = []
if '~shared' in spec:
config_args.append('--static')