zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

init: remove package_testing global

Browse Source 
- refactor the way test dependencies are passed to the concretizer
- remove global state
- update tests
This commit is contained in:
Todd Gamblin 2018-04-27 14:41:12 -07:00 committed by scheibelp
parent bc9f5f084f
commit 47dc96224d
9 changed files with 82 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/spack/spack/__init__.py
View File

@ -43,11 +43,6 @@
tty.die('while initializing Spack RepoPath:', e.message)
#: Needed for test dependencies
from spack.package_prefs import PackageTesting
package_testing = PackageTesting()
#-----------------------------------------------------------------------------
# When packages call 'from spack import *', we import a set of things that
# should be useful for builds.

5
lib/spack/spack/cmd/__init__.py
View File

@ -139,14 +139,15 @@ def parse_specs(args, **kwargs):
"""
concretize = kwargs.get('concretize', False)
normalize = kwargs.get('normalize', False)
tests = kwargs.get('tests', False)
try:
specs = spack.spec.parse(args)
for spec in specs:
if concretize:
spec.concretize() # implies normalize
spec.concretize(tests=tests) # implies normalize
elif normalize:
spec.normalize()
spec.normalize(tests=tests)
return specs

6
lib/spack/spack/cmd/bootstrap.py
View File

@ -50,11 +50,6 @@ def setup_parser(subparser):
cd_group = subparser.add_mutually_exclusive_group()
arguments.add_common_arguments(cd_group, ['clean', 'dirty'])
subparser.add_argument(
'--run-tests', action='store_true', dest='run_tests',
help="run package level tests during installation"
)
def bootstrap(parser, args, **kwargs):
kwargs.update({
@ -62,7 +57,6 @@ def bootstrap(parser, args, **kwargs):
'keep_stage': args.keep_stage,
'install_deps': 'dependencies',
'make_jobs': args.jobs,
'run_tests': args.run_tests,
'verbose': args.verbose,
'dirty': args.dirty
})

10
lib/spack/spack/cmd/install.py
View File

@ -202,14 +202,16 @@ def install(parser, args, **kwargs):
reporter.filename = args.log_file
specs = spack.cmd.parse_specs(args.package)
tests = False
if args.test == 'all' or args.run_tests:
spack.package_testing.test_all()
tests = True
elif args.test == 'root':
for spec in specs:
spack.package_testing.test(spec.name)
tests = [spec.name for spec in specs]
kwargs['tests'] = tests
try:
specs = spack.cmd.parse_specs(args.package, concretize=True)
specs = spack.cmd.parse_specs(
args.package, concretize=True, tests=tests)
except SpackError as e:
reporter.concretization_report(e.message)
raise

32
lib/spack/spack/package.py
View File

@ -1380,6 +1380,7 @@ def do_install(self,
make_jobs=None,
fake=False,
explicit=False,
tests=False,
dirty=None,
**kwargs):
"""Called by commands to install a package and its dependencies.
@ -1405,6 +1406,8 @@ def do_install(self,
fake (bool): Don't really build; install fake stub files instead.
explicit (bool): True if package was explicitly installed, False
if package was implicitly installed (as a dependency).
tests (bool or list or set): False to run no tests, True to test
all packages, or a list of package names to run tests for some
dirty (bool): Don't clean the build environment before installing.
force (bool): Install again, even if already installed.
"""
@ -1435,7 +1438,6 @@ def do_install(self,
# is installed
if keep_stage is False:
self.stage.destroy()
return self._update_explicit_entry_in_db(rec, explicit)
self._do_install_pop_kwargs(kwargs)
@ -1454,6 +1456,7 @@ def do_install(self,
skip_patch=skip_patch,
verbose=verbose,
make_jobs=make_jobs,
tests=tests,
dirty=dirty,
**kwargs)
@ -1470,8 +1473,9 @@ def do_install(self,
tty.msg('No binary for %s found: installing from source'
% self.name)
# Set run_tests flag before starting build.
self.run_tests = spack.package_testing.check(self.name)
# Set run_tests flag before starting build
self.run_tests = (tests is True or
tests and self.name in tests)
# Set parallelism before starting build.
self.make_jobs = make_jobs
@ -1555,6 +1559,11 @@ def build_process():
# preserve verbosity across runs
return echo
# hook that allow tests to inspect this Package before installation
# see unit_test_check() docs.
if not self.unit_test_check():
return
try:
# Create the install prefix and fork the build process.
if not os.path.exists(self.prefix):
@ -1594,6 +1603,23 @@ def build_process():
# check the filesystem for it.
self.stage.created = False
def unit_test_check(self):
"""Hook for unit tests to assert things about package internals.
Unit tests can override this function to perform checks after
``Package.install`` and all post-install hooks run, but before
the database is updated.
The overridden function may indicate that the install procedure
should terminate early (before updating the database) by
returning ``False`` (or any value such that ``bool(result)`` is
``False``).
Return:
(bool): ``True`` to continue, ``False`` to skip ``install()``
"""
return True
def check_for_unfinished_installation(
self, keep_prefix=False, restage=False):
"""Check for leftover files from partially-completed prior install to

19
lib/spack/spack/package_prefs.py
View File

@ -204,25 +204,6 @@ def preferred_variants(cls, pkg_name):
if name in pkg.variants)
class PackageTesting(object):
def __init__(self):
self.packages_to_test = set()
self._test_all = False
def test(self, package_name):
self.packages_to_test.add(package_name)
def test_all(self):
self._test_all = True
def clear(self):
self._test_all = False
self.packages_to_test.clear()
def check(self, package_name):
return self._test_all or (package_name in self.packages_to_test)
def spec_externals(spec):
"""Return a list of external specs (w/external directory path filled in),
one for each known external installation."""

34
lib/spack/spack/spec.py
View File

@ -1808,10 +1808,14 @@ def feq(cfield, sfield):
return changed
def concretize(self):
def concretize(self, tests=False):
"""A spec is concrete if it describes one build of a package uniquely.
This will ensure that this spec is concrete.
Args:
tests (list or bool): list of packages that will need test
dependencies, or True/False for test all/none
If this spec could describe more than one version, variant, or build
of a package, this will add constraints to make it concrete.
@ -1830,7 +1834,7 @@ def concretize(self):
force = False
while changed:
changes = (self.normalize(force),
changes = (self.normalize(force, tests),
self._expand_virtual_packages(),
self._concretize_helper())
changed = any(changes)
@ -2050,7 +2054,7 @@ def _find_provider(self, vdep, provider_index):
raise UnsatisfiableProviderSpecError(required[0], vdep)
def _merge_dependency(
self, dependency, visited, spec_deps, provider_index):
self, dependency, visited, spec_deps, provider_index, tests):
"""Merge dependency information from a Package into this Spec.
Args:
@ -2146,10 +2150,10 @@ def _merge_dependency(
self._add_dependency(spec_dependency, dependency.type)
changed |= spec_dependency._normalize_helper(
visited, spec_deps, provider_index)
visited, spec_deps, provider_index, tests)
return changed
def _normalize_helper(self, visited, spec_deps, provider_index):
def _normalize_helper(self, visited, spec_deps, provider_index, tests):
"""Recursive helper function for _normalize."""
if self.name in visited:
return False
@ -2170,16 +2174,23 @@ def _normalize_helper(self, visited, spec_deps, provider_index):
for dep_name in self.package_class.dependencies:
# Do we depend on dep_name? If so pkg_dep is not None.
dep = self._evaluate_dependency_conditions(dep_name)
# If dep is a needed dependency, merge it.
if dep and (spack.package_testing.check(self.name) or
set(dep.type) - set(['test'])):
changed |= self._merge_dependency(
dep, visited, spec_deps, provider_index)
if dep:
merge = (
# caller requested test dependencies
tests is True or (tests and self.name in tests) or
# this is not a test-only dependency
dep.type - set(['test']))
if merge:
changed |= self._merge_dependency(
dep, visited, spec_deps, provider_index, tests)
any_change |= changed
return any_change
def normalize(self, force=False):
def normalize(self, force=False, tests=False):
"""When specs are parsed, any dependencies specified are hanging off
the root, and ONLY the ones that were explicitly provided are there.
Normalization turns a partial flat spec into a DAG, where:
@ -2220,7 +2231,8 @@ def normalize(self, force=False):
# to package files & their 'when' specs
visited = set()
any_change = self._normalize_helper(visited, spec_deps, provider_index)
any_change = self._normalize_helper(
visited, spec_deps, provider_index, tests)
# If there are deps specified but not visited, they're not
# actually deps of this package. Raise an error.

42
lib/spack/spack/test/cmd/install.py
View File

@ -53,10 +53,8 @@ def parser():
@pytest.fixture()
def noop_install(monkeypatch):
def noop(*args, **kwargs):
return
pass
monkeypatch.setattr(spack.package.PackageBase, 'do_install', noop)
@ -77,31 +75,31 @@ def test_install_package_and_dependency(
assert 'errors="0"' in content
@pytest.mark.usefixtures('noop_install', 'builtin_mock', 'config')
def test_install_runtests():
assert not spack.package_testing._test_all
assert not spack.package_testing.packages_to_test
@pytest.mark.disable_clean_stage_check
def test_install_runtests_notests(monkeypatch, builtin_mock, install_mockery):
def check(pkg):
assert not pkg.run_tests
monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
install('-v', 'dttop')
@pytest.mark.disable_clean_stage_check
def test_install_runtests_root(monkeypatch, builtin_mock, install_mockery):
def check(pkg):
assert pkg.run_tests == (pkg.name == 'dttop')
monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
install('--test=root', 'dttop')
assert not spack.package_testing._test_all
assert spack.package_testing.packages_to_test == set(['dttop'])
spack.package_testing.clear()
@pytest.mark.disable_clean_stage_check
def test_install_runtests_all(monkeypatch, builtin_mock, install_mockery):
def check(pkg):
assert pkg.run_tests
monkeypatch.setattr(spack.package.PackageBase, 'unit_test_check', check)
install('--test=all', 'a')
assert spack.package_testing._test_all
assert not spack.package_testing.packages_to_test
spack.package_testing.clear()
install('--run-tests', 'a')
assert spack.package_testing._test_all
assert not spack.package_testing.packages_to_test
spack.package_testing.clear()
assert not spack.package_testing._test_all
assert not spack.package_testing.packages_to_test
def test_install_package_already_installed(

4
lib/spack/spack/test/spec_dag.py
View File

@ -98,16 +98,14 @@ def test_test_deptype():
mock_repo = MockPackageMultiRepo([w, x, y, z])
try:
spack.package_testing.test(w.name)
spack.repo = mock_repo
spec = Spec('w')
spec.concretize()
spec.concretize(tests=(w.name,))
assert ('x' in spec)
assert ('z' not in spec)
finally:
spack.repo = saved_repo
spack.package_testing.clear()
@pytest.mark.usefixtures('refresh_builtin_mock')