Better test output -- include totals.

lib/spack/spack/cmd/test.py

@@ -9,7 +9,7 @@
 
 def setup_parser(subparser):
     subparser.add_argument(
-        'names', nargs='*', help="Names of packages to install")
+        'names', nargs='*', help="Names of tests to run.")
     subparser.add_argument(
         '-l', '--list', action='store_true', dest='list', help="Show available tests")
     subparser.add_argument(
@@ -17,24 +17,10 @@ def setup_parser(subparser):
         help="verbose output")
 
 
-def find_test_modules():
-    """Include all the modules under test, unless they set skip_test=True"""
-    for name in list_modules(spack.test_path):
-        module = __import__('spack.test.' + name, fromlist='skip_test')
-        if not getattr(module, 'skip_test', False):
-            yield name
-
-
 def test(parser, args):
     if args.list:
         print "Available tests:"
-        colify(find_test_modules())
+        colify(spack.test.list_tests(), indent=2)
-
-    elif not args.names:
-        for name in find_test_modules():
-            print "Running Tests: %s" % name
-            spack.test.run(name, verbose=args.verbose)
 
     else:
-        for name in  args.names:
+        spack.test.run(args.names, args.verbose)
-            spack.test.run(name, verbose=args.verbose)

lib/spack/spack/test/__init__.py

@@ -1,13 +1,58 @@
 import sys
 import unittest
+
 import spack
+from spack.colify import colify
+import spack.tty as tty
 
-def run(test_name, verbose=False):
+test_names = ['versions',
-    __import__(__package__ + "." + test_name)
+              'url_parse',
+              'stage',
+              'spec_syntax',
+              'spec_dag',
+              'concretize']
 
-    # This just runs unittest.main on the module with the provided name
-    test_module = getattr(spack.test, test_name)
 
-    verbosity=1
+def list_tests():
-    if verbose: verbosity = 2
+    return test_names
-    unittest.main(module=test_module, argv=sys.argv[:1], verbosity=verbosity, exit=False)
+
+
+def run(names, verbose=False):
+    verbosity = 1 if not verbose else 2
+
+    if not names:
+        names = test_names
+    else:
+        for test in names:
+            if test not in test_names:
+                tty.error("%s is not a valid spack test name." % test,
+                          "Valid names are:")
+                colify(test_names, indent=4)
+                sys.exit(1)
+
+    runner = unittest.TextTestRunner(verbosity=verbosity)
+
+    testsRun = errors = failures = skipped = 0
+    for test in names:
+        module = 'spack.test.' + test
+        suite = unittest.defaultTestLoader.loadTestsFromName(module)
+
+        tty.msg("Running test: %s" % test)
+        result = runner.run(suite)
+        testsRun += result.testsRun
+        errors   += len(result.errors)
+        failures += len(result.failures)
+        skipped  += len(result.skipped)
+
+    succeeded = not errors and not failures
+    tty.msg("Tests Complete.",
+            "%5d tests run" % testsRun,
+            "%5d skipped" % skipped,
+            "%5d failures" % failures,
+            "%5d errors" % errors)
+
+    if not errors and not failures:
+        tty.info("OK", format='g')
+    else:
+        tty.info("FAIL", format='r')
+        sys.exit(1)