Stand-alone/Smoke tests: copy cached test sources to test stage (#23713)

lib/spack/docs/packaging_guide.rst

@@ -4293,19 +4293,37 @@ can be implemented as shown below.
 
 In this case, the method copies the associated files from the build
 stage **after** the software is installed to the package's metadata
-directory. The result is the following directory and files will be
-available for use in stand-alone tests:
+directory. The result is the directory and files will be cached in
+paths under ``self.install_test_root`` as follows:
 
-* ``join_path(self.install_test_root, 'tests')`` along with its files and subdirectories
+* ``join_path(self.install_test_root, 'tests')`` along with its files
+  and subdirectories
 * ``join_path(self.install_test_root, 'examples', 'foo.c')``
 * ``join_path(self.install_test_root, 'examples', 'bar.c')``
 
+These paths are **automatically copied** to the test stage directory
+where they are available to the package's ``test`` method through the
+``self.test_suite.current_test_cache_dir`` property. In our example,
+the method can access the directory and files using the following
+paths:
+
+* ``join_path(self.test_suite.current_test_cache_dir, 'tests')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+* ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'bar.c')``
+
+.. note::
+
+    Library developers will want to build the associated tests under
+    the ``self.test_suite.current_test_cache_dir`` and against their
+    **installed** libraries before running them.
+
 .. note::
 
     While source and input files are generally recommended, binaries
     **may** also be cached by the build process for install testing.
     Only you, as the package writer or maintainer, know whether these
-    would be appropriate stand-alone tests.
+    would be appropriate for ensuring the installed software continues
+    to work as the underlying system evolves.
 
 .. note::
 
@@ -4327,11 +4345,12 @@ Examples include:
 - expected test output
 
 These extra files should be added to the ``test`` subdirectory of the
-package in the Spack repository. Spack will automatically copy any files
-in that directory to the test staging directory during stand-alone testing.
+package in the Spack repository. 
 
-The ``test`` method can access those files from the
-``self.test_suite.current_test_data_dir`` directory.
+Spack will **automatically copy** the contents of that directory to the
+test staging directory for stand-alone testing. The ``test`` method can
+access those files using the ``self.test_suite.current_test_data_dir``
+property.
 
 .. _expected_test_output_from_file:
 
@@ -4530,13 +4549,17 @@ where each argument has the following meaning:
   
   The default of ``None`` corresponds to the current directory (``'.'``).
 
+"""""""""""""""""""""""""""""""""""""""""
+Accessing package- and test-related files
+"""""""""""""""""""""""""""""""""""""""""
+
 You may need to access files from one or more locations when writing
-the tests. This can happen if the software's repository does not
+stand-alone tests. This can happen if the software's repository does not
 include test source files or includes files but no way to build the
 executables using the installed headers and libraries. In these
 cases, you may need to reference the files relative to one or more
-root directory and associated package property. These are given in
-the table below.
+root directory. The properties containing package- and test-related
+directory paths are provided in the table below.
 
 .. list-table:: Directory-to-property mapping
    :header-rows: 1
@@ -4550,10 +4573,16 @@ the table below.
    * - Package Dependency's Files
      - ``self.spec['<dependency-package>'].prefix``
      - ``self.spec['trilinos'].prefix.include``
-   * - Copied Build-time Files
+   * - Test Suite Stage Files
+     - ``self.test_suite.stage``
+     - ``join_path(self.test_suite.stage, 'results.txt')``
+   * - Cached Build-time Files
      - ``self.install_test_root``
      - ``join_path(self.install_test_root, 'examples', 'foo.c')``
-   * - Custom Package Files
+   * - Staged Cached Build-time Files
+     - ``self.test_suite.current_test_cache_dir``
+     - ``join_path(self.test_suite.current_test_cache_dir, 'examples', 'foo.c')``
+   * - Staged Custom Package Files
      - ``self.test_suite.current_test_data_dir``
      - ``join_path(self.test_suite.current_test_data_dir, 'hello.f90')``
 

lib/spack/spack/install_test.py

@@ -187,6 +187,13 @@ def log_file_for_spec(self, spec):
     def test_dir_for_spec(self, spec):
         return self.stage.join(self.test_pkg_id(spec))
 
+    @property
+    def current_test_cache_dir(self):
+        assert self.current_test_spec and self.current_base_spec
+        test_spec = self.current_test_spec
+        base_spec = self.current_base_spec
+        return self.test_dir_for_spec(base_spec).cache.join(test_spec.name)
+
     @property
     def current_test_data_dir(self):
         assert self.current_test_spec and self.current_base_spec

lib/spack/spack/package.py

@@ -2603,6 +2603,14 @@ def test_process(pkg, kwargs):
                     except spack.repo.UnknownPackageError:
                         continue
 
+                    # copy installed test sources cache into test cache dir
+                    if spec.concrete:
+                        cache_source = spec_pkg.install_test_root
+                        cache_dir = pkg.test_suite.current_test_cache_dir
+                        if (os.path.isdir(cache_source) and
+                                not os.path.exists(cache_dir)):
+                            fsys.install_tree(cache_source, cache_dir)
+
                     # copy test data into test data dir
                     data_source = Prefix(spec_pkg.package_dir).test
                     data_dir = pkg.test_suite.current_test_data_dir

lib/spack/spack/test/cmd/mark.py

@@ -30,7 +30,7 @@ def test_mark_all_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
 
 @pytest.mark.db
@@ -47,7 +47,7 @@ def test_mark_one_explicit(mutable_database):
     uninstall('-y', '-a', 'mpileaks')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 2
+    assert len(all_specs) == 3
 
 
 @pytest.mark.db
@@ -55,7 +55,7 @@ def test_mark_one_implicit(mutable_database):
     mark('-i', 'externaltest')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 13
+    assert len(all_specs) == 14
 
 
 @pytest.mark.db
@@ -64,4 +64,4 @@ def test_mark_all_implicit_then_explicit(mutable_database):
     mark('-e', '-a')
     gc('-y')
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15

lib/spack/spack/test/cmd/uninstall.py

@@ -42,7 +42,7 @@ def test_recursive_uninstall(mutable_database):
     uninstall('-y', '-a', '--dependents', 'callpath')
 
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 8
+    assert len(all_specs) == 9
     # query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
     callpath_specs = [s for s in all_specs if s.satisfies('callpath')]
@@ -56,7 +56,7 @@ def test_recursive_uninstall(mutable_database):
 @pytest.mark.db
 @pytest.mark.regression('3690')
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 7), ('libelf', 5)
+    ('dyninst', 8), ('libelf', 6)
 ])
 def test_uninstall_spec_with_multiple_roots(
         constraint, expected_number_of_specs, mutable_database
@@ -69,7 +69,7 @@ def test_uninstall_spec_with_multiple_roots(
 
 @pytest.mark.db
 @pytest.mark.parametrize('constraint,expected_number_of_specs', [
-    ('dyninst', 13), ('libelf', 13)
+    ('dyninst', 14), ('libelf', 14)
 ])
 def test_force_uninstall_spec_with_ref_count_not_zero(
         constraint, expected_number_of_specs, mutable_database
@@ -141,7 +141,8 @@ def db_specs():
             [s for s in all_specs if s.satisfies('mpi')]
         )
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 13
+    total_specs = len(all_specs)
+    assert total_specs == 14
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 2
     assert len(mpi_specs) == 3
@@ -152,7 +153,7 @@ def db_specs():
     validate_callpath_spec(True)
 
     all_specs, mpileaks_specs, callpath_specs, mpi_specs = db_specs()
-    assert len(all_specs) == 14      # back to 14
+    assert len(all_specs) == total_specs + 1   # back to total_specs+1
     assert len(mpileaks_specs) == 3
     assert len(callpath_specs) == 3  # back to 3
     assert len(mpi_specs) == 3

lib/spack/spack/test/conftest.py

@@ -567,6 +567,7 @@ def _install(spec):
     _install('mpileaks ^mpich2')
     _install('mpileaks ^zmpi')
     _install('externaltest')
+    _install('trivial-smoke-test')
 
 
 @pytest.fixture(scope='session')

lib/spack/spack/test/database.py

@@ -424,7 +424,7 @@ def test_005_db_exists(database):
 def test_010_all_install_sanity(database):
     """Ensure that the install layout reflects what we think it does."""
     all_specs = spack.store.layout.all_specs()
-    assert len(all_specs) == 14
+    assert len(all_specs) == 15
 
     # Query specs with multiple configurations
     mpileaks_specs = [s for s in all_specs if s.satisfies('mpileaks')]
@@ -545,7 +545,8 @@ def test_041_ref_counts_deprecate(mutable_database):
 def test_050_basic_query(database):
     """Ensure querying database is consistent with what is installed."""
     # query everything
-    assert len(spack.store.db.query()) == 16
+    total_specs = len(spack.store.db.query())
+    assert total_specs == 17
 
     # query specs with multiple configurations
     mpileaks_specs = database.query('mpileaks')
@@ -571,10 +572,10 @@ def test_050_basic_query(database):
     assert len(database.query('mpileaks ^zmpi')) == 1
 
     # Query by date
-    assert len(database.query(start_date=datetime.datetime.min)) == 16
+    assert len(database.query(start_date=datetime.datetime.min)) == total_specs
     assert len(database.query(start_date=datetime.datetime.max)) == 0
     assert len(database.query(end_date=datetime.datetime.min)) == 0
-    assert len(database.query(end_date=datetime.datetime.max)) == 16
+    assert len(database.query(end_date=datetime.datetime.max)) == total_specs
 
 
 def test_060_remove_and_add_root_package(mutable_database):

lib/spack/spack/test/test_suite.py

@@ -3,6 +3,8 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 import os
+
+import llnl.util.filesystem as fs
 import spack.install_test
 import spack.spec
 
@@ -51,3 +53,32 @@ def test_write_test_result(mock_packages, mock_test_stage):
         msg = lines[0]
         assert result in msg
         assert spec.name in msg
+
+
+def test_do_test(mock_packages, mock_test_stage, install_mockery):
+    """Perform a stand-alone test with files to copy."""
+    spec = spack.spec.Spec('trivial-smoke-test').concretized()
+    test_name = 'test_do_test'
+    test_filename = 'test_file.in'
+
+    pkg = spec.package
+    pkg.create_extra_test_source()
+
+    test_suite = spack.install_test.TestSuite([spec], test_name)
+    test_suite.current_test_spec = spec
+    test_suite.current_base_spec = spec
+    test_suite.ensure_stage()
+
+    # Save off target paths for current spec since test suite processing
+    # assumes testing multiple specs.
+    cached_filename = fs.join_path(test_suite.current_test_cache_dir,
+                                   pkg.test_source_filename)
+    data_filename = fs.join_path(test_suite.current_test_data_dir,
+                                 test_filename)
+
+    # Run the test, making sure to retain the test stage directory
+    # so we can ensure the files were copied.
+    test_suite(remove_directory=False)
+
+    assert os.path.exists(cached_filename)
+    assert os.path.exists(data_filename)

var/spack/repos/builtin.mock/packages/trivial-smoke-test/package.py

@@ -0,0 +1,25 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class TrivialSmokeTest(Package):
+    """This package is a stub with trivial smoke test features."""
+    homepage = "http://www.example.com/trivial_test"
+    url      = "http://www.unit-test-should-replace-this-url/trivial_test-1.0.tar.gz"
+
+    version('1.0', 'foobarbaz')
+
+    test_source_filename = 'cached_file.in'
+
+    @run_before('install')
+    def create_extra_test_source(self):
+        mkdirp(self.install_test_root)
+        touch(join_path(self.install_test_root, self.test_source_filename))
+
+    @run_after('install')
+    def copy_test_sources(self):
+        self.cache_extra_test_sources([self.test_source_filename])

var/spack/repos/builtin/packages/hypre/package.py

@@ -220,26 +220,36 @@ def install(self, spec, prefix):
                         '-rhsone')
             make("install")
 
+    extra_install_tests = join_path('src', 'examples')
+
     @run_after('install')
     def cache_test_sources(self):
-        srcs = ['src/examples']
-        self.cache_extra_test_sources(srcs)
+        self.cache_extra_test_sources(self.extra_install_tests)
+
+    @property
+    def _cached_tests_work_dir(self):
+        """The working directory for cached test sources."""
+        return join_path(self.test_suite.current_test_cache_dir,
+                         self.extra_install_tests)
 
     def test(self):
         """Perform smoke test on installed HYPRE package."""
+        if '+mpi' not in self.spec:
+            print('Skipping: HYPRE must be installed with +mpi to run tests')
+            return
 
-        if '+mpi' in self.spec:
-            examples_dir = join_path(self.install_test_root, 'src/examples')
-            with working_dir(examples_dir, create=False):
-                make("HYPRE_DIR=" + self.prefix, "bigint")
+        # Build copied and cached test examples
+        self.run_test('make',
+                      ['HYPRE_DIR={0}'.format(self.prefix), 'bigint'],
+                      purpose='test: building selected examples',
+                      work_dir=self._cached_tests_work_dir)
 
-                reason = "test: ensuring HYPRE examples run"
-                self.run_test('./ex5big', [], [], installed=True,
-                              purpose=reason, skip_missing=True, work_dir='.')
-                self.run_test('./ex15big', [], [], installed=True,
-                              purpose=reason, skip_missing=True, work_dir='.')
-
-                make("distclean")
+        # Run the examples built above
+        for exe in ['./ex5big', './ex15big']:
+            self.run_test(exe, [], [], installed=False,
+                          purpose='test: ensuring {0} runs'.format(exe),
+                          skip_missing=True,
+                          work_dir=self._cached_tests_work_dir)
 
     @property
     def headers(self):

var/spack/repos/builtin/packages/openmpi/package.py

@@ -891,9 +891,10 @@ def _test_bin_ops(self):
             'shmemrun': ls,
         }
 
-        for exe in checks:
-            options, expected, status = checks[exe]
-            reason = 'test: checking {0} output'.format(exe)
+        for binary in checks:
+            options, expected, status = checks[binary]
+            exe = join_path(self.prefix.bin, binary)
+            reason = 'test: checking {0} output'.format(binary)
             self.run_test(exe, options, expected, status, installed=True,
                           purpose=reason, skip_missing=True)
 
@@ -942,36 +943,28 @@ def _test_check_versions(self):
             'shmemcxx': comp_vers,
         }
 
-        for exe in checks:
-            expected = checks[exe]
+        for binary in checks:
+            expected = checks[binary]
             purpose = 'test: ensuring version of {0} is {1}' \
-                .format(exe, expected)
+                .format(binary, expected)
+            exe = join_path(self.prefix.bin, binary)
             self.run_test(exe, '--version', expected, installed=True,
                           purpose=purpose, skip_missing=True)
 
-    def _test_build_examples(self):
-        # Build the examples copied during installation and return "status"
-        reason = 'test: ensuring ability to build the examples'
-        return self.run_test('make', ['all'], [],
-                             purpose=reason,
-                             work_dir=join_path(self.install_test_root,
-                                                self.extra_install_tests))
-
-    def _test_clean_examples(self):
-        # Clean up any example build files
-        reason = 'test: ensuring copied examples cleaned up'
-        return self.run_test('make', ['clean'], [],
-                             purpose=reason,
-                             work_dir=join_path(self.install_test_root,
-                                                self.extra_install_tests))
+    @property
+    def _cached_tests_work_dir(self):
+        """The working directory for cached test sources."""
+        return join_path(self.test_suite.current_test_cache_dir,
+                         self.extra_install_tests)
 
     def _test_examples(self):
-        # First ensure can build copied examples
-        if not self._test_build_examples():
-            self._test_clean_examples()
-            return
+        """Run test examples copied from source at build-time."""
+        # Build the copied, cached test examples
+        self.run_test('make', ['all'], [],
+                      purpose='test: building cached test examples',
+                      work_dir=self._cached_tests_work_dir)
 
-        # Now run examples with known, simple-to-verify results
+        # Run examples with known, simple-to-verify results
         have_spml = self.spec.satisfies('@2.0.0:2.1.6')
 
         hello_world = (['Hello, world', 'I am', '0 of', '1'], 0)
@@ -1015,14 +1008,11 @@ def _test_examples(self):
                 .format(exe, status)
             self.run_test(exe, [], expected, status, installed=False,
                           purpose=reason, skip_missing=True,
-                          work_dir=join_path(self.install_test_root,
-                                             self.extra_install_tests))
-
-        self._test_clean_examples()
+                          work_dir=self._cached_tests_work_dir)
 
     def test(self):
-        """Perform smoke tests on the installed package."""
-        # Simple version check tests on known packages
+        """Perform stand-alone/smoke tests on the installed package."""
+        # Simple version check tests on selected installed binaries
         self._test_check_versions()
 
         # Test the operation of selected executables