Stand-alone tests: distinguish NO-TESTS from PASSED (#25880)

Co-authored-by: Seth R. Johnson <johnsonsr@ornl.gov>

lib/spack/spack/install_test.py

@@ -135,10 +135,15 @@ def __call__(self, *args, **kwargs):
                     dirty=dirty
                 )
 
-                # Clean up on success and log passed test
+                # Clean up on success
                 if remove_directory:
                     shutil.rmtree(test_dir)
-                self.write_test_result(spec, 'PASSED')
+
+                # Log test status based on whether any non-pass-only test
+                # functions were called
+                tested = os.path.exists(self.tested_file_for_spec(spec))
+                status = 'PASSED' if tested else 'NO-TESTS'
+                self.write_test_result(spec, status)
             except BaseException as exc:
                 self.fails += 1
                 if isinstance(exc, (SyntaxError, TestSuiteSpecError)):
@@ -194,6 +199,13 @@ def log_file_for_spec(self, spec):
     def test_dir_for_spec(self, spec):
         return self.stage.join(self.test_pkg_id(spec))
 
+    @classmethod
+    def tested_file_name(cls, spec):
+        return '%s-tested.txt' % cls.test_pkg_id(spec)
+
+    def tested_file_for_spec(self, spec):
+        return self.stage.join(self.tested_file_name(spec))
+
     @property
     def current_test_cache_dir(self):
         if not (self.current_test_spec and self.current_base_spec):

lib/spack/spack/package.py

@@ -1785,12 +1785,14 @@ def do_test(self, dirty=False):
         # Clear test failures
         self.test_failures = []
         self.test_log_file = self.test_suite.log_file_for_spec(self.spec)
+        self.tested_file = self.test_suite.tested_file_for_spec(self.spec)
         fsys.touch(self.test_log_file)  # Otherwise log_parse complains
 
         kwargs = {'dirty': dirty, 'fake': False, 'context': 'test'}
         spack.build_environment.start_build_process(self, test_process, kwargs)
 
     def test(self):
+        # Defer tests to virtual and concrete packages
         pass
 
     def run_test(self, exe, options=[], expected=[], status=0,
@@ -2605,6 +2607,7 @@ def test_process(pkg, kwargs):
         test_specs = [pkg.spec] + [spack.spec.Spec(v_name)
                                    for v_name in sorted(v_names)]
 
+        ran_actual_test_function = False
         try:
             with fsys.working_dir(
                     pkg.test_suite.test_dir_for_spec(pkg.spec)):
@@ -2639,7 +2642,16 @@ def test_process(pkg, kwargs):
                     if not isinstance(test_fn, types.FunctionType):
                         test_fn = test_fn.__func__
 
+                    # Skip any test methods consisting solely of 'pass'
+                    # since they do not contribute to package testing.
+                    source = (inspect.getsource(test_fn)).splitlines()[1:]
+                    lines = (ln.strip() for ln in source)
+                    statements = [ln for ln in lines if not ln.startswith('#')]
+                    if len(statements) > 0 and statements[0] == 'pass':
+                        continue
+
                     # Run the tests
+                    ran_actual_test_function = True
                     test_fn(pkg)
 
             # If fail-fast was on, we error out above
@@ -2651,6 +2663,11 @@ def test_process(pkg, kwargs):
             # reset debug level
             tty.set_debug(old_debug)
 
+            # flag the package as having been tested (i.e., ran one or more
+            # non-pass-only methods
+            if ran_actual_test_function:
+                fsys.touch(pkg.tested_file)
+
 
 inject_flags = PackageBase.inject_flags
 env_flags = PackageBase.env_flags