Packages can tune the list of files to be archived at the end of install (#7760)

Fixes #2781

This PR introduces a new attribute for packages called
`archive_files`, which designates files that should be saved from
a package build (e.g. the config.log generated during autotools
builds).

The attribute contains a list of glob expressions; Any file that
matches will be archived in the `<prefix>/.spack/archived-files`
directory. Errors that occur when archiving files are collected and
reported in a file named `<prefix>/.spack/archived-files/errors.txt`.

`AutotoolsPackage` and `CMakePackage` provide a sensible default
override for this attribute.

lib/spack/spack/build_systems/autotools.py

@@ -94,6 +94,11 @@ class AutotoolsPackage(PackageBase):
     #: Options to be passed to autoreconf when using the default implementation
     autoreconf_extra_args = []
 
+    @property
+    def archive_files(self):
+        """Files to archive for packages based on autotools"""
+        return [os.path.join(self.build_directory, 'config.log')]
+
     @run_after('autoreconf')
     def _do_patch_config_guess(self):
         """Some packages ship with an older config.guess and need to have

lib/spack/spack/build_systems/cmake.py

@@ -90,6 +90,11 @@ class CMakePackage(PackageBase):
 
     depends_on('cmake', type='build')
 
+    @property
+    def archive_files(self):
+        """Files to archive for packages based on CMake"""
+        return [os.path.join(self.build_directory, 'CMakeCache.txt')]
+
     @property
     def root_cmakelists_dir(self):
         """The relative path to the directory containing CMakeLists.txt

lib/spack/spack/package.py

@@ -37,6 +37,7 @@
 import contextlib
 import copy
 import functools
+import glob
 import hashlib
 import inspect
 import itertools
@@ -529,6 +530,12 @@ class SomePackage(Package):
     #: directories, sanity checks will fail.
     sanity_check_is_dir = []
 
+    #: List of glob expressions. Each expression must either be
+    #: absolute or relative to the package source path.
+    #: Matching artifacts found at the end of the build process will
+    #: be copied in the same directory tree as build.env and build.out.
+    archive_files = []
+
     #
     # Set default licensing information
     #
@@ -1647,8 +1654,53 @@ def log(self):
             # FIXME : this potentially catches too many things...
             pass
 
+        # Archive the whole stdout + stderr for the package
         install(self.log_path, log_install_path)
+        # Archive the environment used for the build
         install(self.env_path, env_install_path)
+        # Finally, archive files that are specific to each package
+        with working_dir(self.stage.source_path):
+            errors = StringIO()
+            target_dir = os.path.join(
+                spack.store.layout.metadata_path(self.spec), 'archived-files'
+            )
+            for glob_expr in self.archive_files:
+                # Check that we are trying to copy things that are
+                # in the source_path tree (not arbitrary files)
+                abs_expr = os.path.realpath(glob_expr)
+                if os.path.realpath(self.stage.source_path) not in abs_expr:
+                    errors.write(
+                        '[OUTSIDE SOURCE PATH]: {0}\n'.format(glob_expr)
+                    )
+                    continue
+                # Now that we are sure that the path is within the correct
+                # folder, make it relative and check for matches
+                if os.path.isabs(glob_expr):
+                    glob_expr = os.path.relpath(
+                        glob_expr, self.stage.source_path
+                    )
+                files = glob.glob(glob_expr)
+                for f in files:
+                    try:
+                        target = os.path.join(target_dir, f)
+                        # We must ensure that the directory exists before
+                        # copying a file in
+                        mkdirp(os.path.dirname(target))
+                        install(f, target)
+                    except Exception:
+                        # Here try to be conservative, and avoid discarding
+                        # the whole install procedure because of copying a
+                        # single file failed
+                        errors.write('[FAILED TO ARCHIVE]: {0}'.format(f))
+
+            if errors.getvalue():
+                error_file = os.path.join(target_dir, 'errors.txt')
+                mkdirp(target_dir)
+                with open(error_file, 'w') as err:
+                    err.write(errors.getvalue())
+                tty.warn('Errors occurred when archiving files.\n\t'
+                         'See: {0}'.format(error_file))
+
         dump_packages(self.spec, packages_dir)
 
     def sanity_check_prefix(self):

lib/spack/spack/test/cmd/install.py

@@ -376,3 +376,22 @@ def test_install_mix_cli_and_files(clispecs, filespecs, tmpdir):
 
     install(*args, fail_on_error=False)
     assert install.returncode == 0
+
+
+@pytest.mark.usefixtures(
+    'builtin_mock', 'mock_archive', 'mock_fetch', 'config', 'install_mockery'
+)
+def test_extra_files_are_archived():
+    s = Spec('archive-files')
+    s.concretize()
+
+    install('archive-files')
+
+    archive_dir = os.path.join(
+        spack.store.layout.metadata_path(s), 'archived-files'
+    )
+    config_log = os.path.join(archive_dir, 'config.log')
+    assert os.path.exists(config_log)
+
+    errors_txt = os.path.join(archive_dir, 'errors.txt')
+    assert os.path.exists(errors_txt)

var/spack/repos/builtin.mock/packages/archive-files/package.py

@@ -0,0 +1,53 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+
+class ArchiveFiles(AutotoolsPackage):
+    """Simple package with one optional dependency"""
+
+    homepage = "http://www.example.com"
+    url = "http://www.example.com/a-1.0.tar.gz"
+
+    version('1.0', '0123456789abcdef0123456789abcdef')
+    version('2.0', '2.0_a_hash')
+
+    @property
+    def archive_files(self):
+        return super(ArchiveFiles, self).archive_files + ['../../outside.log']
+
+    def autoreconf(self, spec, prefix):
+        pass
+
+    def configure(self, spec, prefix):
+        pass
+
+    def build(self, spec, prefix):
+        mkdirp(self.build_directory)
+        config_log = join_path(self.build_directory, 'config.log')
+        touch(config_log)
+
+    def install(self, spec, prefix):
+        touch(join_path(prefix, 'deleteme'))