Merge pull request #475 from LLNL/features/env-and-package-provenance

Features/env and package provenance

lib/spack/llnl/util/filesystem.py

@@ -152,15 +152,20 @@ def set_install_permissions(path):
 def copy_mode(src, dest):
     src_mode = os.stat(src).st_mode
     dest_mode = os.stat(dest).st_mode
-    if src_mode | stat.S_IXUSR: dest_mode |= stat.S_IXUSR
+    if src_mode & stat.S_IXUSR: dest_mode |= stat.S_IXUSR
-    if src_mode | stat.S_IXGRP: dest_mode |= stat.S_IXGRP
+    if src_mode & stat.S_IXGRP: dest_mode |= stat.S_IXGRP
-    if src_mode | stat.S_IXOTH: dest_mode |= stat.S_IXOTH
+    if src_mode & stat.S_IXOTH: dest_mode |= stat.S_IXOTH
     os.chmod(dest, dest_mode)
 
 
 def install(src, dest):
     """Manually install a file to a particular location."""
     tty.debug("Installing %s to %s" % (src, dest))
+
+    # Expand dsst to its eventual full path if it is a directory.
+    if os.path.isdir(dest):
+        dest = join_path(dest, os.path.basename(src))
+
     shutil.copy(src, dest)
     set_install_permissions(dest)
     copy_mode(src, dest)

lib/spack/spack/cmd/repo.py

@@ -74,51 +74,7 @@ def setup_parser(subparser):
 
 def repo_create(args):
     """Create a new package repository."""
-    root = canonicalize_path(args.directory)
+    full_path, namespace = create_repo(args.directory, args.namespace)
-    namespace = args.namespace
-
-    if not args.namespace:
-        namespace = os.path.basename(root)
-
-    if not re.match(r'\w[\.\w-]*', namespace):
-        tty.die("'%s' is not a valid namespace." % namespace)
-
-    existed = False
-    if os.path.exists(root):
-        if os.path.isfile(root):
-            tty.die('File %s already exists and is not a directory' % root)
-        elif os.path.isdir(root):
-            if not os.access(root, os.R_OK | os.W_OK):
-                tty.die('Cannot create new repo in %s: cannot access directory.' % root)
-            if os.listdir(root):
-                tty.die('Cannot create new repo in %s: directory is not empty.' % root)
-        existed = True
-
-    full_path = os.path.realpath(root)
-    parent = os.path.dirname(full_path)
-    if not os.access(parent, os.R_OK | os.W_OK):
-        tty.die("Cannot create repository in %s: can't access parent!" % root)
-
-    try:
-        config_path = os.path.join(root, repo_config_name)
-        packages_path = os.path.join(root, packages_dir_name)
-
-        mkdirp(packages_path)
-        with open(config_path, 'w') as config:
-            config.write("repo:\n")
-            config.write("  namespace: '%s'\n" % namespace)
-
-    except (IOError, OSError) as e:
-        tty.die('Failed to create new repository in %s.' % root,
-                "Caused by %s: %s" % (type(e), e))
-
-        # try to clean up.
-        if existed:
-            shutil.rmtree(config_path, ignore_errors=True)
-            shutil.rmtree(packages_path, ignore_errors=True)
-        else:
-            shutil.rmtree(root, ignore_errors=True)
-
     tty.msg("Created repo with namespace '%s'." % namespace)
     tty.msg("To register it with spack, run this command:",
             'spack repo add %s' % full_path)

lib/spack/spack/directory_layout.py

@@ -173,7 +173,9 @@ def __init__(self, root, **kwargs):
 
         self.spec_file_name      = 'spec.yaml'
         self.extension_file_name = 'extensions.yaml'
-        self.build_log_name      = 'build.out' # TODO: use config file.
+        self.build_log_name      = 'build.out'  # build log.
+        self.build_env_name      = 'build.env'  # build environment
+        self.packages_dir        = 'repos'      # archive of package.py files
 
         # Cache of already written/read extension maps.
         self._extension_maps = {}
@@ -231,6 +233,16 @@ def build_log_path(self, spec):
                          self.build_log_name)
 
 
+    def build_env_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.build_env_name)
+
+
+    def build_packages_path(self, spec):
+        return join_path(self.path_for_spec(spec), self.metadata_dir,
+                         self.packages_dir)
+
+
     def create_install_directory(self, spec):
         _check_concrete(spec)
 

lib/spack/spack/package.py

@@ -58,6 +58,7 @@
 import spack.mirror
 import spack.hooks
 import spack.directives
+import spack.repository
 import spack.build_environment
 import spack.url
 import spack.util.web
@@ -66,6 +67,7 @@
 from spack.stage import Stage, ResourceStage, StageComposite
 from spack.util.compression import allowed_archive, extension
 from spack.util.executable import ProcessError
+from spack.util.environment import dump_environment
 
 """Allowed URL schemes for spack packages."""
 _ALLOWED_URL_SCHEMES = ["http", "https", "ftp", "file", "git"]
@@ -501,6 +503,7 @@ def fetcher(self):
             self._fetcher = self._make_fetcher()
         return self._fetcher
 
+
     @fetcher.setter
     def fetcher(self, f):
         self._fetcher = f
@@ -884,10 +887,14 @@ def real_work():
                     # Do the real install in the source directory.
                     self.stage.chdir_to_source()
 
+                    # Save the build environment in a file before building.
+                    env_path = join_path(os.getcwd(), 'spack-build.env')
+
                     # This redirects I/O to a build log (and optionally to the terminal)
                     log_path = join_path(os.getcwd(), 'spack-build.out')
                     log_file = open(log_path, 'w')
                     with log_output(log_file, verbose, sys.stdout.isatty(), True):
+                        dump_environment(env_path)
                         self.install(self.spec, self.prefix)
 
                 # Ensure that something was actually installed.
@@ -896,7 +903,12 @@ def real_work():
                 # Move build log into install directory on success
                 if not fake:
                     log_install_path = spack.install_layout.build_log_path(self.spec)
+                    env_install_path = spack.install_layout.build_env_path(self.spec)
                     install(log_path, log_install_path)
+                    install(env_path, env_install_path)
+
+                packages_dir = spack.install_layout.build_packages_path(self.spec)
+                dump_packages(self.spec, packages_dir)
 
                 # On successful install, remove the stage.
                 if not keep_stage:
@@ -1212,6 +1224,52 @@ def validate_package_url(url_string):
         tty.die("Invalid file type in URL: '%s'" % url_string)
 
 
+def dump_packages(spec, path):
+    """Dump all package information for a spec and its dependencies.
+
+       This creates a package repository within path for every
+       namespace in the spec DAG, and fills the repos wtih package
+       files and patch files for every node in the DAG.
+    """
+    mkdirp(path)
+
+    # Copy in package.py files from any dependencies.
+    # Note that we copy them in as they are in the *install* directory
+    # NOT as they are in the repository, because we want a snapshot of
+    # how *this* particular build was done.
+    for node in spec.traverse():
+        if node is not spec:
+            # Locate the dependency package in the install tree and find
+            # its provenance information.
+            source = spack.install_layout.build_packages_path(node)
+            source_repo_root = join_path(source, node.namespace)
+
+            # There's no provenance installed for the source package.  Skip it.
+            # User can always get something current from the builtin repo.
+            if not os.path.isdir(source_repo_root):
+                continue
+
+            # Create a source repo and get the pkg directory out of it.
+            try:
+                source_repo = spack.repository.Repo(source_repo_root)
+                source_pkg_dir = source_repo.dirname_for_package_name(node.name)
+            except RepoError as e:
+                tty.warn("Warning: Couldn't copy in provenance for %s" % node.name)
+
+        # Create a destination repository
+        dest_repo_root = join_path(path, node.namespace)
+        if not os.path.exists(dest_repo_root):
+            spack.repository.create_repo(dest_repo_root)
+        repo = spack.repository.Repo(dest_repo_root)
+
+        # Get the location of the package in the dest repo.
+        dest_pkg_dir = repo.dirname_for_package_name(node.name)
+        if node is not spec:
+            install_tree(source_pkg_dir, dest_pkg_dir)
+        else:
+            spack.repo.dump_provenance(node, dest_pkg_dir)
+
+
 def print_pkg(message):
     """Outputs a message with a package icon."""
     from llnl.util.tty.color import cwrite

lib/spack/spack/repository.py

@@ -33,7 +33,7 @@
 from external import yaml
 
 import llnl.util.tty as tty
-from llnl.util.filesystem import join_path
+from llnl.util.filesystem import *
 
 import spack.error
 import spack.config
@@ -316,6 +316,16 @@ def get(self, spec, new=False):
         return self.repo_for_pkg(spec).get(spec)
 
 
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        return self.repo_for_pkg(spec).dump_provenance(spec, path)
+
+
     def dirname_for_package_name(self, pkg_name):
         return self.repo_for_pkg(pkg_name).dirname_for_package_name(pkg_name)
 
@@ -552,6 +562,35 @@ def get(self, spec, new=False):
         return self._instances[key]
 
 
+    @_autospec
+    def dump_provenance(self, spec, path):
+        """Dump provenance information for a spec to a particular path.
+
+           This dumps the package file and any associated patch files.
+           Raises UnknownPackageError if not found.
+        """
+        # Some preliminary checks.
+        if spec.virtual:
+            raise UnknownPackageError(spec.name)
+
+        if spec.namespace and spec.namespace != self.namespace:
+            raise UnknownPackageError("Repository %s does not contain package %s."
+                                      % (self.namespace, spec.fullname))
+
+        # Install any patch files needed by packages.
+        mkdirp(path)
+        for spec, patches in spec.package.patches.items():
+            for patch in patches:
+                if patch.path:
+                    if os.path.exists(patch.path):
+                        install(patch.path, path)
+                    else:
+                        tty.warn("Patch file did not exist: %s" % patch.path)
+
+        # Install the package.py file itself.
+        install(self.filename_for_package_name(spec), path)
+
+
     def purge(self):
         """Clear entire package instance cache."""
         self._instances.clear()
@@ -705,6 +744,58 @@ def __contains__(self, pkg_name):
         return self.exists(pkg_name)
 
 
+def create_repo(root, namespace=None):
+    """Create a new repository in root with the specified namespace.
+
+       If the namespace is not provided, use basename of root.
+       Return the canonicalized path and the namespace of the created repository.
+    """
+    root = canonicalize_path(root)
+    if not namespace:
+        namespace = os.path.basename(root)
+
+    if not re.match(r'\w[\.\w-]*', namespace):
+        raise InvalidNamespaceError("'%s' is not a valid namespace." % namespace)
+
+    existed = False
+    if os.path.exists(root):
+        if os.path.isfile(root):
+            raise BadRepoError('File %s already exists and is not a directory' % root)
+        elif os.path.isdir(root):
+            if not os.access(root, os.R_OK | os.W_OK):
+                raise BadRepoError('Cannot create new repo in %s: cannot access directory.' % root)
+            if os.listdir(root):
+                raise BadRepoError('Cannot create new repo in %s: directory is not empty.' % root)
+        existed = True
+
+    full_path = os.path.realpath(root)
+    parent = os.path.dirname(full_path)
+    if not os.access(parent, os.R_OK | os.W_OK):
+        raise BadRepoError("Cannot create repository in %s: can't access parent!" % root)
+
+    try:
+        config_path = os.path.join(root, repo_config_name)
+        packages_path = os.path.join(root, packages_dir_name)
+
+        mkdirp(packages_path)
+        with open(config_path, 'w') as config:
+            config.write("repo:\n")
+            config.write("  namespace: '%s'\n" % namespace)
+
+    except (IOError, OSError) as e:
+        raise BadRepoError('Failed to create new repository in %s.' % root,
+                           "Caused by %s: %s" % (type(e), e))
+
+        # try to clean up.
+        if existed:
+            shutil.rmtree(config_path, ignore_errors=True)
+            shutil.rmtree(packages_path, ignore_errors=True)
+        else:
+            shutil.rmtree(root, ignore_errors=True)
+
+    return full_path, namespace
+
+
 class RepoError(spack.error.SpackError):
     """Superclass for repository-related errors."""
 
@@ -713,6 +804,10 @@ class NoRepoConfiguredError(RepoError):
     """Raised when there are no repositories configured."""
 
 
+class InvalidNamespaceError(RepoError):
+    """Raised when an invalid namespace is encountered."""
+
+
 class BadRepoError(RepoError):
     """Raised when repo layout is invalid."""
 

lib/spack/spack/util/environment.py

@@ -63,3 +63,10 @@ def pop_keys(dictionary, *keys):
     for key in keys:
         if key in dictionary:
             dictionary.pop(key)
+
+
+def dump_environment(path):
+    """Dump the current environment out to a file."""
+    with open(path, 'w') as env_file:
+        for key,val in sorted(os.environ.items()):
+            env_file.write("%s=%s\n" % (key, val))