zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

WIP infrastructure for Spack test command to test existing installations

Browse Source
This commit is contained in:
Gregory Becker
2019-12-18 11:27:32 -08:00
committed by Tamara Dahlgren
parent e294a1e0a6
commit 4151224ef2
7 changed files with 269 additions and 176 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
lib/spack/spack/build_environment.py
View File

@@ -59,7 +59,7 @@
from spack.util.environment import (
env_flag, filter_system_paths, get_path, is_system_path,
EnvironmentModifications, validate, preserve_environment)
from spack.util.environment import system_dirs
from spack.util.environment import system_dirs, inspect_path
from spack.error import NoLibrariesError, NoHeadersError
from spack.util.executable import Executable
from spack.util.module_cmd import load_module, path_from_modules, module
@@ -711,28 +711,40 @@ def load_external_modules(pkg):
load_module(external_module)
def setup_package(pkg, dirty):
def setup_package(pkg, dirty, context='build'):
"""Execute all environment setup routines."""
build_env = EnvironmentModifications()
env = EnvironmentModifications()
# clean environment
if not dirty:
clean_environment()
set_compiler_environment_variables(pkg, build_env)
set_build_environment_variables(pkg, build_env, dirty)
pkg.architecture.platform.setup_platform_environment(pkg, build_env)
# setup compilers and build tools for build contexts
if context == 'build':
set_compiler_environment_variables(pkg, env)
set_build_environment_variables(pkg, env, dirty)
build_env.extend(
modifications_from_dependencies(pkg.spec, context='build')
# architecture specific setup
pkg.architecture.platform.setup_platform_environment(pkg, env)
# recursive post-order dependency information
env.extend(
modifications_from_dependencies(pkg.spec, context=context)
)
if (not dirty) and (not build_env.is_unset('CPATH')):
if context == 'build' and (not dirty) and (not env.is_unset('CPATH')):
tty.debug("A dependency has updated CPATH, this may lead pkg-config"
" to assume that the package is part of the system"
" includes and omit it when invoked with '--cflags'.")
# setup package itself
set_module_variables_for_package(pkg)
pkg.setup_build_environment(build_env)
if context == 'build':
pkg.setup_build_environment(env)
elif context == 'test':
import spack.user_environment as uenv # avoid circular import
env.extend(inspect_path(pkg.spec.prefix,
uenv.prefix_inspections(pkg.spec.platform)))
# Loading modules, in particular if they are meant to be used outside
# of Spack, can change environment variables that are relevant to the
@@ -742,15 +754,16 @@ def setup_package(pkg, dirty):
# unnecessary. Modules affecting these variables will be overwritten anyway
with preserve_environment('CC', 'CXX', 'FC', 'F77'):
# All module loads that otherwise would belong in previous
# functions have to occur after the build_env object has its
# functions have to occur after the env object has its
# modifications applied. Otherwise the environment modifications
# could undo module changes, such as unsetting LD_LIBRARY_PATH
# after a module changes it.
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
if context == 'build':
for mod in pkg.compiler.modules:
# Fixes issue https://github.com/spack/spack/issues/3153
if os.environ.get("CRAY_CPU_TARGET") == "mic-knl":
load_module("cce")
load_module(mod)
# kludge to handle cray libsci being automatically loaded by PrgEnv
# modules on cray platform. Module unload does no damage when
@@ -768,8 +781,8 @@ def setup_package(pkg, dirty):
':'.join(implicit_rpaths))
# Make sure nothing's strange about the Spack environment.
validate(build_env, tty.warn)
build_env.apply_modifications()
validate(env, tty.warn)
env.apply_modifications()
def modifications_from_dependencies(spec, context):
@@ -789,7 +802,8 @@ def modifications_from_dependencies(spec, context):
deptype_and_method = {
'build': (('build', 'link', 'test'),
'setup_dependent_build_environment'),
'run': (('link', 'run'), 'setup_dependent_run_environment')
'run': (('link', 'run'), 'setup_dependent_run_environment'),
'test': (('link', 'run', 'test'), 'setup_dependent_run_environment')
}
deptype, method = deptype_and_method[context]
@@ -803,7 +817,7 @@ def modifications_from_dependencies(spec, context):
return env
def fork(pkg, function, dirty, fake):
def fork(pkg, function, dirty, fake, context='build'):
"""Fork a child process to do part of a spack build.
Args:
@@ -815,6 +829,8 @@ def fork(pkg, function, dirty, fake):
dirty (bool): If True, do NOT clean the environment before
building.
fake (bool): If True, skip package setup b/c it's not a real build
context (string): If 'build', setup build environment. If 'test', setup
test environment.
Usage::
@@ -843,7 +859,7 @@ def child_process(child_pipe, input_stream):
try:
if not fake:
setup_package(pkg, dirty=dirty)
setup_package(pkg, dirty=dirty, context=context)
return_value = function()
child_pipe.send(return_value)

226
lib/spack/spack/cmd/test.py
View File

@@ -4,166 +4,94 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from __future__ import print_function
from __future__ import division
import collections
import sys
import re
import os
import argparse
import pytest
from six import StringIO
import llnl.util.tty.color as color
from llnl.util.filesystem import working_dir
from llnl.util.tty.colify import colify
import llnl.util.tty as tty
import spack.paths
import spack.environment as ev
import spack.cmd
description = "run spack's unit tests (wrapper around pytest)"
section = "developer"
description = "run spack's tests for an install"
section = "administrator"
level = "long"
def setup_parser(subparser):
# subparser.add_argument(
# '--log-format',
# default=None,
# choices=spack.report.valid_formats,
# help="format to be used for log files"
# )
# subparser.add_argument(
# '--output-file',
# default=None,
# help="filename for the log file. if not passed a default will be used"
# )
# subparser.add_argument(
# '--cdash-upload-url',
# default=None,
# help="CDash URL where reports will be uploaded"
# )
# subparser.add_argument(
# '--cdash-build',
# default=None,
# help="""The name of the build that will be reported to CDash.
# Defaults to spec of the package to install."""
# )
# subparser.add_argument(
# '--cdash-site',
# default=None,
# help="""The site name that will be reported to CDash.
# Defaults to current system hostname."""
# )
# cdash_subgroup = subparser.add_mutually_exclusive_group()
# cdash_subgroup.add_argument(
# '--cdash-track',
# default='Experimental',
# help="""Results will be reported to this group on CDash.
# Defaults to Experimental."""
# )
# cdash_subgroup.add_argument(
# '--cdash-buildstamp',
# default=None,
# help="""Instead of letting the CDash reporter prepare the
# buildstamp which, when combined with build name, site and project,
# uniquely identifies the build, provide this argument to identify
# the build yourself. Format: %%Y%%m%%d-%%H%%M-[cdash-track]"""
# )
# arguments.add_common_arguments(subparser, ['yes_to_all'])
length_group = subparser.add_mutually_exclusive_group()
length_group.add_argument(
'--smoke', action='store_true', dest='smoke_test', default=True,
help='run smoke tests (default)')
length_group.add_argument(
'--capability', action='store_false', dest='smoke_test', default=True,
help='run full capability tests using pavilion')
subparser.add_argument(
'-H', '--pytest-help', action='store_true', default=False,
help="show full pytest help, with advanced options")
# extra spack arguments to list tests
list_group = subparser.add_argument_group("listing tests")
list_mutex = list_group.add_mutually_exclusive_group()
list_mutex.add_argument(
'-l', '--list', action='store_const', default=None,
dest='list', const='list', help="list test filenames")
list_mutex.add_argument(
'-L', '--list-long', action='store_const', default=None,
dest='list', const='long', help="list all test functions")
list_mutex.add_argument(
'-N', '--list-names', action='store_const', default=None,
dest='list', const='names', help="list full names of all tests")
# use tests for extension
subparser.add_argument(
'--extension', default=None,
help="run test for a given spack extension")
# spell out some common pytest arguments, so they'll show up in help
pytest_group = subparser.add_argument_group(
"common pytest arguments (spack test --pytest-help for more details)")
pytest_group.add_argument(
"-s", action='append_const', dest='parsed_args', const='-s',
help="print output while tests run (disable capture)")
pytest_group.add_argument(
"-k", action='store', metavar="EXPRESSION", dest='expression',
help="filter tests by keyword (can also use w/list options)")
pytest_group.add_argument(
"--showlocals", action='append_const', dest='parsed_args',
const='--showlocals', help="show local variable values in tracebacks")
# remainder is just passed to pytest
subparser.add_argument(
'pytest_args', nargs=argparse.REMAINDER, help="arguments for pytest")
'specs', nargs=argparse.REMAINDER,
help="list of specs to test")
def do_list(args, extra_args):
"""Print a lists of tests than what pytest offers."""
# Run test collection and get the tree out.
old_output = sys.stdout
try:
sys.stdout = output = StringIO()
pytest.main(['--collect-only'] + extra_args)
finally:
sys.stdout = old_output
def test(parser, args):
env = ev.get_env(args, 'test')
hashes = env.all_hashes() if env else None
lines = output.getvalue().split('\n')
tests = collections.defaultdict(lambda: set())
prefix = []
specs = spack.cmd.parse_specs(args.specs) if args.specs else [None]
specs_to_test = []
for spec in specs:
matching = spack.store.db.query_local(spec, hashes=hashes)
if spec and not matching:
tty.warn("No installed packages match spec %s" % spec)
specs_to_test.extend(matching)
# collect tests into sections
for line in lines:
match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line)
if not match:
continue
indent, nodetype, name = match.groups()
log_dir = os.getcwd()
# strip parametrized tests
if "[" in name:
name = name[:name.index("[")]
depth = len(indent) // 2
if nodetype.endswith("Function"):
key = tuple(prefix)
tests[key].add(name)
else:
prefix = prefix[:depth]
prefix.append(name)
def colorize(c, prefix):
if isinstance(prefix, tuple):
return "::".join(
color.colorize("@%s{%s}" % (c, p))
for p in prefix if p != "()"
)
return color.colorize("@%s{%s}" % (c, prefix))
if args.list == "list":
files = set(prefix[0] for prefix in tests)
color_files = [colorize("B", file) for file in sorted(files)]
colify(color_files)
elif args.list == "long":
for prefix, functions in sorted(tests.items()):
path = colorize("*B", prefix) + "::"
functions = [colorize("c", f) for f in sorted(functions)]
color.cprint(path)
colify(functions, indent=4)
print()
else: # args.list == "names"
all_functions = [
colorize("*B", prefix) + "::" + colorize("c", f)
for prefix, functions in sorted(tests.items())
for f in sorted(functions)
]
colify(all_functions)
def add_back_pytest_args(args, unknown_args):
"""Add parsed pytest args, unknown args, and remainder together.
We add some basic pytest arguments to the Spack parser to ensure that
they show up in the short help, so we have to reassemble things here.
"""
result = args.parsed_args or []
result += unknown_args or []
result += args.pytest_args or []
if args.expression:
result += ["-k", args.expression]
return result
def test(parser, args, unknown_args):
if args.pytest_help:
# make the pytest.main help output more accurate
sys.argv[0] = 'spack test'
return pytest.main(['-h'])
# add back any parsed pytest args we need to pass to pytest
pytest_args = add_back_pytest_args(args, unknown_args)
# The default is to test the core of Spack. If the option `--extension`
# has been used, then test that extension.
pytest_root = spack.paths.spack_root
if args.extension:
target = args.extension
extensions = spack.config.get('config:extensions')
pytest_root = spack.extensions.path_for_extension(target, *extensions)
# pytest.ini lives in the root of the spack repository.
with working_dir(pytest_root):
if args.list:
do_list(args, pytest_args)
return
return pytest.main(pytest_args)
if args.smoke_test:
for spec in specs_to_test:
log_file = os.path.join(log_dir, 'test-%s' % spec.dag_hash())
spec.package.do_test(log_file)
else:
raise NotImplementedError

105
lib/spack/spack/cmd/unit_test.py Normal file
View File

@@ -0,0 +1,105 @@
# Copyright 2013-2019 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from __future__ import print_function
import sys
import os
import re
import argparse
import pytest
from six import StringIO
from llnl.util.filesystem import working_dir
from llnl.util.tty.colify import colify
import spack.paths
description = "run spack's unit tests"
section = "developer"
level = "long"
def setup_parser(subparser):
subparser.add_argument(
'-H', '--pytest-help', action='store_true', default=False,
help="print full pytest help message, showing advanced options")
list_group = subparser.add_mutually_exclusive_group()
list_group.add_argument(
'-l', '--list', action='store_true', default=False,
help="list basic test names")
list_group.add_argument(
'-L', '--long-list', action='store_true', default=False,
help="list the entire hierarchy of tests")
subparser.add_argument(
'--extension', default=None,
help="run test for a given Spack extension"
)
subparser.add_argument(
'tests', nargs=argparse.REMAINDER,
help="list of tests to run (will be passed to pytest -k)")
def do_list(args, unknown_args):
"""Print a lists of tests than what pytest offers."""
# Run test collection and get the tree out.
old_output = sys.stdout
try:
sys.stdout = output = StringIO()
pytest.main(['--collect-only'])
finally:
sys.stdout = old_output
# put the output in a more readable tree format.
lines = output.getvalue().split('\n')
output_lines = []
for line in lines:
match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line)
if not match:
continue
indent, nodetype, name = match.groups()
# only print top-level for short list
if args.list:
if not indent:
output_lines.append(
os.path.basename(name).replace('.py', ''))
else:
print(indent + name)
if args.list:
colify(output_lines)
def unit_test(parser, args, unknown_args):
if args.pytest_help:
# make the pytest.main help output more accurate
sys.argv[0] = 'spack unit-test'
pytest.main(['-h'])
return
# The default is to test the core of Spack. If the option `--extension`
# has been used, then test that extension.
pytest_root = spack.paths.test_path
if args.extension:
target = args.extension
extensions = spack.config.get('config:extensions')
pytest_root = spack.extensions.path_for_extension(target, *extensions)
# pytest.ini lives in the root of the spack repository.
with working_dir(pytest_root):
# --list and --long-list print the test output better.
if args.list or args.long_list:
do_list(args, unknown_args)
return
# Allow keyword search without -k if no options are specified
if (args.tests and not unknown_args and
not any(arg.startswith('-') for arg in args.tests)):
return pytest.main(['-k'] + args.tests)
# Just run the pytest command
return pytest.main(unknown_args + args.tests)

22
lib/spack/spack/package.py
View File

@@ -1592,6 +1592,28 @@ def do_install(self, **kwargs):
do_install.__doc__ += install_args_docstring
def do_test(self, log_file, dirty=False):
def test_process():
with log_output(log_file) as logger:
with logger.force_echo():
tty.msg('Testing package %s' %
self.spec.format('{name}-{hash:7}'))
old_debug = tty.is_debug()
tty.set_debug(True)
self.test()
tty.set_debug(old_debug)
try:
spack.build_environment.fork(
self, test_process, dirty=dirty, fake=False, context='test')
except Exception as e:
tty.error('Tests failed. See test log for details\n'
' %s\n' % log_file)
def test(self):
pass
def unit_test_check(self):
"""Hook for unit tests to assert things about package internals.

2
lib/spack/spack/test/cmd/test.py
View File

@@ -5,7 +5,7 @@
from spack.main import SpackCommand
spack_test = SpackCommand('test')
spack_test = SpackCommand('unit-test')
cmd_test_py = 'lib/spack/spack/test/cmd/test.py'

17
lib/spack/spack/util/executable.py
View File

@@ -2,7 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import sys
import os
import re
import shlex
@@ -98,6 +98,9 @@ def __call__(self, *args, **kwargs):
If both ``output`` and ``error`` are set to ``str``, then one string
is returned containing output concatenated with error. Not valid
for ``input``
* ``str.split``, as in the ``split`` method of the Python string type.
Behaves the same as ``str``, except that value is also written to
``stdout`` or ``stderr``.
By default, the subprocess inherits the parent's file descriptors.
@@ -132,7 +135,7 @@ def __call__(self, *args, **kwargs):
def streamify(arg, mode):
if isinstance(arg, string_types):
return open(arg, mode), True
elif arg is str:
elif arg in (str, str.split):
return subprocess.PIPE, False
else:
return arg, False
@@ -168,12 +171,16 @@ def streamify(arg, mode):
out, err = proc.communicate()
result = None
if output is str or error is str:
if output in (str, str.split) or error in (str, str.split):
result = ''
if output is str:
if output in (str, str.split):
result += text_type(out.decode('utf-8'))
if error is str:
if output is str.split:
sys.stdout.write(out)
if error in (str, str.split):
result += text_type(err.decode('utf-8'))
if error is str.split:
sys.stderr.write(err)
rc = self.returncode = proc.returncode
if fail_on_error and rc != 0 and (rc not in ignore_errors):