From 099fa1df34a157a323aa023b69c52399a441b81b Mon Sep 17 00:00:00 2001 From: Peter Scheibel Date: Thu, 12 Nov 2015 13:23:19 -0800 Subject: [PATCH] Use nose to run unit tests. 1. Adding a plugin to keep track of the total number of tests run as well as the number of tests with failures/errors. 2. Some nose plugins (including xunit which will be added in a future commit) assign stdout to a stream object that does not have a .fileno attribute. spack.util.executable.Executable now avoids passing stdout to subprocess (and always uses subprocess.PIPE) TODO: 1. Still need to figure out how to activate the plugin (as of now it is being ignored by nose). Newer versions of nose appear to make this simpler (e.g. the "addplugins" argument to nose.run) 2. Need to include new version of nose in order to use xunit --- lib/spack/spack/test/__init__.py | 27 +++++----- lib/spack/spack/test/tally_plugin.py | 73 ++++++++++++++++++++++++++++ lib/spack/spack/test/unit_install.py | 6 +-- lib/spack/spack/util/executable.py | 7 ++- 4 files changed, 93 insertions(+), 20 deletions(-) create mode 100644 lib/spack/spack/test/tally_plugin.py diff --git a/lib/spack/spack/test/__init__.py b/lib/spack/spack/test/__init__.py index 0f776bfea46..9d577d65397 100644 --- a/lib/spack/spack/test/__init__.py +++ b/lib/spack/spack/test/__init__.py @@ -24,7 +24,9 @@ ############################################################################## import sys import unittest +import nose +from spack.test.tally_plugin import Tally import llnl.util.tty as tty from llnl.util.tty.colify import colify @@ -81,28 +83,23 @@ def run(names, verbose=False): "Valid names are:") colify(sorted(test_names), indent=4) sys.exit(1) - - runner = unittest.TextTestRunner(verbosity=verbosity) - - testsRun = errors = failures = 0 + + tally = Tally() + tally.enabled = True for test in names: module = 'spack.test.' + test print module - suite = unittest.defaultTestLoader.loadTestsFromName(module) - + tty.msg("Running test: %s" % test) - result = runner.run(suite) - testsRun += result.testsRun - errors += len(result.errors) - failures += len(result.failures) + result = nose.run(argv=["", module], plugins=[tally]) - succeeded = not errors and not failures + succeeded = not tally.failCount and not tally.errorCount tty.msg("Tests Complete.", - "%5d tests run" % testsRun, - "%5d failures" % failures, - "%5d errors" % errors) + "%5d tests run" % tally.numberOfTests, + "%5d failures" % tally.failCount, + "%5d errors" % tally.errorCount) - if not errors and not failures: + if succeeded: tty.info("OK", format='g') else: tty.info("FAIL", format='r') diff --git a/lib/spack/spack/test/tally_plugin.py b/lib/spack/spack/test/tally_plugin.py new file mode 100644 index 00000000000..d8638bf7e3e --- /dev/null +++ b/lib/spack/spack/test/tally_plugin.py @@ -0,0 +1,73 @@ +############################################################################## +# Copyright (c) 2013, Lawrence Livermore National Security, LLC. +# Produced at the Lawrence Livermore National Laboratory. +# +# This file is part of Spack. +# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved. +# LLNL-CODE-647188 +# +# For details, see https://scalability-llnl.github.io/spack +# Please also see the LICENSE file for our notice and the LGPL. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License (as published by +# the Free Software Foundation) version 2.1 dated February 1999. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and +# conditions of the GNU General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public License +# along with this program; if not, write to the Free Software Foundation, +# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +############################################################################## +from nose.plugins import Plugin + +import os + +class Tally(Plugin): + name = 'tally' + + def __init__(self): + super(Tally, self).__init__() + self.successes = set() + self.failures = set() + self.errors = set() + + @property + def successCount(self): + return len(self.successes) + + @property + def failCount(self): + return len(self.failures) + + @property + def errorCount(self): + return len(self.errors) + + @property + def numberOfTests(self): + return self.errorCount + self.failCount + self.successCount + + def options(self, parser, env=os.environ): + super(Tally, self).options(parser, env=env) + + def configure(self, options, conf): + super(Tally, self).configure(options, conf) + + def begin(self): + print ">>> TALLY PLUGIN BEGIN" + + def addSuccess(self, test): + self.successes.add(test) + + def addError(self, test, err): + self.errors.add(test) + + def addFailure(self, test, err): + test.failures.add(test) + + def finalize(self, result): + pass diff --git a/lib/spack/spack/test/unit_install.py b/lib/spack/spack/test/unit_install.py index c4b9092f051..0bcb3f37671 100644 --- a/lib/spack/spack/test/unit_install.py +++ b/lib/spack/spack/test/unit_install.py @@ -90,7 +90,7 @@ def test_installing_both(self): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX, specY], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED, @@ -101,7 +101,7 @@ def test_dependency_already_installed(self): pkgX.installed = True pkgY.installed = True - test_install.create_test_output(specX, [specX], mo, getLogFunc=test_fetch_log) + test_install.create_test_output(specX, [specX], mo, getLogFunc=mock_fetch_log) self.assertEqual(mo.results, {bIdX:test_install.TestResult.PASSED}) @@ -116,6 +116,6 @@ def __init__(self, init=None): def get(self, spec): return self.specToPkg[spec] -def test_fetch_log(path): +def mock_fetch_log(path): return [] diff --git a/lib/spack/spack/util/executable.py b/lib/spack/spack/util/executable.py index d1dfb62ffb6..6848a7e2a54 100644 --- a/lib/spack/spack/util/executable.py +++ b/lib/spack/spack/util/executable.py @@ -95,11 +95,14 @@ def streamify(arg, mode): proc = subprocess.Popen( cmd, stdin=input, - stderr=error, - stdout=subprocess.PIPE if return_output else output) + stderr=subprocess.PIPE, + stdout=subprocess.PIPE) out, err = proc.communicate() self.returncode = proc.returncode + output.write(out) + error.write(err) + rc = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:"