Packages can tune the list of files to be archived at the end of install (#7760)

Fixes #2781

This PR introduces a new attribute for packages called
`archive_files`, which designates files that should be saved from
a package build (e.g. the config.log generated during autotools
builds).

The attribute contains a list of glob expressions; Any file that
matches will be archived in the `<prefix>/.spack/archived-files`
directory. Errors that occur when archiving files are collected and
reported in a file named `<prefix>/.spack/archived-files/errors.txt`.

`AutotoolsPackage` and `CMakePackage` provide a sensible default
override for this attribute.

lib/spack/spack/build_systems/autotools.py

@@ -94,6 +94,11 @@ class AutotoolsPackage(PackageBase):
     #: Options to be passed to autoreconf when using the default implementation
     autoreconf_extra_args = []
 
+    @property
+    def archive_files(self):
+        """Files to archive for packages based on autotools"""
+        return [os.path.join(self.build_directory, 'config.log')]
+
     @run_after('autoreconf')
     def _do_patch_config_guess(self):
         """Some packages ship with an older config.guess and need to have

lib/spack/spack/build_systems/cmake.py

@@ -90,6 +90,11 @@ class CMakePackage(PackageBase):
 
     depends_on('cmake', type='build')
 
+    @property
+    def archive_files(self):
+        """Files to archive for packages based on CMake"""
+        return [os.path.join(self.build_directory, 'CMakeCache.txt')]
+
     @property
     def root_cmakelists_dir(self):
         """The relative path to the directory containing CMakeLists.txt

lib/spack/spack/package.py

@@ -37,6 +37,7 @@
 import contextlib
 import copy
 import functools
+import glob
 import hashlib
 import inspect
 import itertools
@@ -529,6 +530,12 @@ class SomePackage(Package):
     #: directories, sanity checks will fail.
     sanity_check_is_dir = []
 
+    #: List of glob expressions. Each expression must either be
+    #: absolute or relative to the package source path.
+    #: Matching artifacts found at the end of the build process will
+    #: be copied in the same directory tree as build.env and build.out.
+    archive_files = []
+
     #
     # Set default licensing information
     #
@@ -1647,8 +1654,53 @@ def log(self):
             # FIXME : this potentially catches too many things...
             pass
 
+        # Archive the whole stdout + stderr for the package
         install(self.log_path, log_install_path)
+        # Archive the environment used for the build
         install(self.env_path, env_install_path)
+        # Finally, archive files that are specific to each package
+        with working_dir(self.stage.source_path):
+            errors = StringIO()
+            target_dir = os.path.join(
+                spack.store.layout.metadata_path(self.spec), 'archived-files'
+            )
+            for glob_expr in self.archive_files:
+                # Check that we are trying to copy things that are
+                # in the source_path tree (not arbitrary files)
+                abs_expr = os.path.realpath(glob_expr)
+                if os.path.realpath(self.stage.source_path) not in abs_expr:
+                    errors.write(
+                        '[OUTSIDE SOURCE PATH]: {0}\n'.format(glob_expr)
+                    )
+                    continue
+                # Now that we are sure that the path is within the correct
+                # folder, make it relative and check for matches
+                if os.path.isabs(glob_expr):
+                    glob_expr = os.path.relpath(
+                        glob_expr, self.stage.source_path
+                    )
+                files = glob.glob(glob_expr)
+                for f in files:
+                    try:
+                        target = os.path.join(target_dir, f)
+                        # We must ensure that the directory exists before
+                        # copying a file in
+                        mkdirp(os.path.dirname(target))
+                        install(f, target)
+                    except Exception:
+                        # Here try to be conservative, and avoid discarding
+                        # the whole install procedure because of copying a
+                        # single file failed
+                        errors.write('[FAILED TO ARCHIVE]: {0}'.format(f))
+
+            if errors.getvalue():
+                error_file = os.path.join(target_dir, 'errors.txt')
+                mkdirp(target_dir)
+                with open(error_file, 'w') as err:
+                    err.write(errors.getvalue())
+                tty.warn('Errors occurred when archiving files.\n\t'
+                         'See: {0}'.format(error_file))
+
         dump_packages(self.spec, packages_dir)
 
     def sanity_check_prefix(self):

lib/spack/spack/test/cmd/install.py

@@ -376,3 +376,22 @@ def test_install_mix_cli_and_files(clispecs, filespecs, tmpdir):
 
     install(*args, fail_on_error=False)
     assert install.returncode == 0
+
+
+@pytest.mark.usefixtures(
+    'builtin_mock', 'mock_archive', 'mock_fetch', 'config', 'install_mockery'
+)
+def test_extra_files_are_archived():
+    s = Spec('archive-files')
+    s.concretize()
+
+    install('archive-files')
+
+    archive_dir = os.path.join(
+        spack.store.layout.metadata_path(s), 'archived-files'
+    )
+    config_log = os.path.join(archive_dir, 'config.log')
+    assert os.path.exists(config_log)
+
+    errors_txt = os.path.join(archive_dir, 'errors.txt')
+    assert os.path.exists(errors_txt)