Testing: optionally run tests on externally installed packages (#28701)

Since Spack does not install external packages, this commit skips them by
default when running stand-alone tests. The assumption is that such packages
have likely undergone an acceptance test process. 

However, the tests can be run against installed externals using 
```
% spack test run --externals ...
```

lib/spack/spack/cmd/test.py

@@ -53,6 +53,10 @@ def setup_parser(subparser):
         '--fail-first', action='store_true',
         help="Stop after the first failed package."
     )
+    run_parser.add_argument(
+        '--externals', action='store_true',
+        help="Test packages that are externally installed."
+    )
     run_parser.add_argument(
         '--keep-stage',
         action='store_true',
@@ -203,7 +207,8 @@ def test_run(args):
     with reporter('test', test_suite.stage):
         test_suite(remove_directory=not args.keep_stage,
                    dirty=args.dirty,
-                   fail_first=args.fail_first)
+                   fail_first=args.fail_first,
+                   externals=args.externals)
 
 
 def test_list(args):

lib/spack/spack/install_test.py

@@ -128,6 +128,7 @@ def __call__(self, *args, **kwargs):
         remove_directory = kwargs.get('remove_directory', True)
         dirty = kwargs.get('dirty', False)
         fail_first = kwargs.get('fail_first', False)
+        externals = kwargs.get('externals', False)
 
         for spec in self.specs:
             try:
@@ -149,9 +150,7 @@ def __call__(self, *args, **kwargs):
                 fs.mkdirp(test_dir)
 
                 # run the package tests
-                spec.package.do_test(
-                    dirty=dirty
-                )
+                spec.package.do_test(dirty=dirty, externals=externals)
 
                 # Clean up on success
                 if remove_directory:
@@ -160,7 +159,18 @@ def __call__(self, *args, **kwargs):
                 # Log test status based on whether any non-pass-only test
                 # functions were called
                 tested = os.path.exists(self.tested_file_for_spec(spec))
-                status = 'PASSED' if tested else 'NO-TESTS'
+                if tested:
+                    status = 'PASSED'
+                else:
+                    self.ensure_stage()
+                    if spec.external and not externals:
+                        status = 'SKIPPED'
+                        msg = 'Skipped external package'
+                    else:
+                        status = 'NO-TESTS'
+                        msg = 'No tests to run'
+                    _add_msg_to_file(self.log_file_for_spec(spec), msg)
+
                 self.write_test_result(spec, status)
             except BaseException as exc:
                 self.fails += 1

lib/spack/spack/package.py

@@ -1796,7 +1796,7 @@ def cache_extra_test_sources(self, srcs):
                 fsys.mkdirp(os.path.dirname(dest_path))
                 fsys.copy(src_path, dest_path)
 
-    def do_test(self, dirty=False):
+    def do_test(self, dirty=False, externals=False):
         if self.test_requires_compiler:
             compilers = spack.compilers.compilers_for_spec(
                 self.spec.compiler, arch_spec=self.spec.architecture)
@@ -1813,6 +1813,12 @@ def do_test(self, dirty=False):
         self.tested_file = self.test_suite.tested_file_for_spec(self.spec)
         fsys.touch(self.test_log_file)  # Otherwise log_parse complains
 
+        if self.spec.external and not externals:
+            with open(self.test_log_file, 'w') as ofd:
+                ofd.write('Testing package {0}\n'
+                          .format(self.test_suite.test_pkg_id(self.spec)))
+            return
+
         kwargs = {'dirty': dirty, 'fake': False, 'context': 'test'}
         spack.build_environment.start_build_process(self, test_process, kwargs)
 

lib/spack/spack/report.py

@@ -171,8 +171,15 @@ def wrapper(instance, *args, **kwargs):
                 value = None
                 try:
                     value = do_fn(instance, *args, **kwargs)
-                    package['result'] = 'success'
-                    package['stdout'] = fetch_log(pkg, do_fn, self.dir)
+
+                    externals = kwargs.get('externals', False)
+                    skip_externals = pkg.spec.external and not externals
+                    if do_fn.__name__ == 'do_test' and skip_externals:
+                        package['result'] = 'skipped'
+                        package['stdout'] = 'Skipped external package'
+                    else:
+                        package['result'] = 'success'
+                        package['stdout'] = fetch_log(pkg, do_fn, self.dir)
                     package['installed_from_binary_cache'] = \
                         pkg.installed_from_binary_cache
                     if do_fn.__name__ == '_install_task' and installed_already:

lib/spack/spack/test/test_suite.py

@@ -87,6 +87,34 @@ def test_do_test(mock_packages, install_mockery, mock_test_stage):
     assert os.path.exists(data_filename)
 
 
+@pytest.mark.parametrize('arguments,status,msg', [
+    ({}, 'SKIPPED', 'Skipped'),
+    ({'externals': True}, 'NO-TESTS', 'No tests'),
+])
+def test_test_external(mock_packages, install_mockery, mock_test_stage,
+                       arguments, status, msg):
+    def ensure_results(filename, expected):
+        assert os.path.exists(filename)
+        with open(filename, 'r') as fd:
+            lines = fd.readlines()
+            have = False
+            for line in lines:
+                if expected in line:
+                    have = True
+                    break
+            assert have
+
+    name = 'trivial-smoke-test'
+    spec = spack.spec.Spec(name).concretized()
+    spec.external_path = '/path/to/external/{0}'.format(name)
+
+    test_suite = spack.install_test.TestSuite([spec])
+    test_suite(**arguments)
+
+    ensure_results(test_suite.results_file, status)
+    ensure_results(test_suite.log_file_for_spec(spec), msg)
+
+
 def test_test_stage_caches(mock_packages, install_mockery, mock_test_stage):
     def ensure_current_cache_fail(test_suite):
         with pytest.raises(spack.install_test.TestSuiteSpecError):

share/spack/spack-completion.bash

@@ -1706,7 +1706,7 @@ _spack_test() {
 _spack_test_run() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --alias --fail-fast --fail-first --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --clean --dirty"
+        SPACK_COMPREPLY="-h --help --alias --fail-fast --fail-first --externals --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --clean --dirty"
     else
         _installed_packages
     fi