filesystem_view: add a class to handle a view via a Yaml description

lib/spack/spack/filesystem_view.py

@@ -0,0 +1,524 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import functools as ft
+import os
+import re
+import shutil
+import sys
+
+from llnl.util.filesystem import join_path
+from llnl.util.link_tree import LinkTree
+from llnl.util import tty
+
+import spack
+import spack.spec
+import spack.store
+from spack.directory_layout import ExtensionAlreadyInstalledError
+from spack.directory_layout import YamlViewExtensionsLayout
+
+# compatability
+if sys.version_info < (3, 0):
+    from itertools import imap as map
+    from itertools import ifilter as filter
+    from itertools import izip as zip
+
+__all__ = ["FilesystemView", "YamlFilesystemView"]
+
+
+class FilesystemView(object):
+    """
+        Governs a filesystem view that is located at certain root-directory.
+
+        Packages are linked from their install directories into a common file
+        hierachy.
+
+        In distributed filesystems, loading each installed package seperately
+        can lead to slow-downs due to too many directories being traversed.
+        This can be circumvented by loading all needed modules into a common
+        directory structure.
+    """
+
+    def __init__(self, root, layout, **kwargs):
+        """
+            Initialize a filesystem view under the given `root` directory with
+            corresponding directory `layout`.
+
+            Files are linked by method `link` (os.symlink by default).
+        """
+        self.root = root
+        self.layout = layout
+
+        self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
+        self.link = kwargs.get("link", os.symlink)
+        self.verbose = kwargs.get("verbose", False)
+
+    def add_specs(self, *specs, **kwargs):
+        """
+            Add given specs to view.
+
+            The supplied specs might be standalone packages or extensions of
+            other packages.
+
+            Should accept `with_dependencies` as keyword argument (default
+            True) to indicate wether or not dependencies should be activated as
+            well.
+
+            Should except an `exclude` keyword argument containing a list of
+            regexps that filter out matching spec names.
+
+            This method should make use of `activate_{extension,standalone}`.
+        """
+        raise NotImplementedError
+
+    def add_extension(self, spec):
+        """
+            Add (link) an extension in this view.
+        """
+        raise NotImplementedError
+
+    def add_standalone(self, spec):
+        """
+            Add (link) a standalone package into this view.
+        """
+        raise NotImplementedError
+
+    def check_added(self, spec):
+        """
+            Check if the given concrete spec is active in this view.
+        """
+        raise NotImplementedError
+
+    def remove_specs(self, *specs, **kwargs):
+        """
+            Removes given specs from view.
+
+            The supplied spec might be a standalone package or an extension of
+            another package.
+
+            Should accept `with_dependencies` as keyword argument (default
+            True) to indicate wether or not dependencies should be deactivated
+            as well.
+
+            Should accept `with_dependents` as keyword argument (default True)
+            to indicate wether or not dependents on the deactivated specs
+            should be removed as well.
+
+            Should except an `exclude` keyword argument containing a list of
+            regexps that filter out matching spec names.
+
+            This method should make use of `deactivate_{extension,standalone}`.
+        """
+        raise NotImplementedError
+
+    def remove_extension(self, spec):
+        """
+            Remove (unlink) an extension from this view.
+        """
+        raise NotImplementedError
+
+    def remove_standalone(self, spec):
+        """
+            Remove (unlink) a standalone package from this view.
+        """
+        raise NotImplementedError
+
+    def get_all_specs(self):
+        """
+            Get all specs currently active in this view.
+        """
+        raise NotImplementedError
+
+    def get_spec(self, spec):
+        """
+            Return the actual spec linked in this view (i.e. do not look it up
+            in the database by name).
+
+            `spec` can be a name or a spec from which the name is extracted.
+
+            As there can only be a single version active for any spec the name
+            is enough to identify the spec in the view.
+
+            If no spec is present, returns None.
+        """
+        raise NotImplementedError
+
+    def print_status(self, *specs, **kwargs):
+        """
+            Print a short summary about the given specs, detailing whether..
+                * ..they are active in the view.
+                * ..they are active but the activated version differs.
+                * ..they are not activte in the view.
+
+            Takes `with_dependencies` keyword argument so that the status of
+            dependencies is printed as well.
+        """
+        raise NotImplementedError
+
+
+class YamlFilesystemView(FilesystemView):
+    """
+        Filesystem view to work with a yaml based directory layout.
+    """
+
+    def __init__(self, root, layout, **kwargs):
+        super(YamlFilesystemView, self).__init__(root, layout, **kwargs)
+
+        self.extensions_layout = YamlViewExtensionsLayout(root, layout)
+
+        self._croot = colorize_root(self.root) + " "
+
+    def add_specs(self, *specs, **kwargs):
+        assert all((s.concrete for s in specs))
+        specs = set(specs)
+
+        if kwargs.get("with_dependencies", True):
+            specs.update(get_dependencies(specs))
+
+        if kwargs.get("exclude", None):
+            specs = set(filter_exclude(specs, kwargs["exclude"]))
+
+        conflicts = self.get_conflicts(*specs)
+
+        if conflicts:
+            for s, v in conflicts:
+                self.print_conflict(v, s)
+            return
+
+        extensions = set(filter(lambda s: s.package.is_extension, specs))
+        standalones = specs - extensions
+
+        set(map(self._check_no_ext_conflicts, extensions))
+        # fail on first error, otherwise link extensions as well
+        if all(map(self.add_standalone, standalones)):
+            all(map(self.add_extension, extensions))
+
+    def add_extension(self, spec):
+        if not spec.package.is_extension:
+            tty.error(self._croot + 'Package %s is not an extension.'
+                      % spec.name)
+            return False
+
+        try:
+            if not spec.package.is_activated(self.extensions_layout):
+                spec.package.do_activate(
+                    verbose=self.verbose,
+                    extensions_layout=self.extensions_layout)
+
+        except ExtensionAlreadyInstalledError:
+            # As we use sets in add_specs(), the order in which packages get
+            # activated is essentially random. So this spec might have already
+            # been activated as dependency of another package -> fail silently
+            pass
+
+        # make sure the meta folder is linked as well (this is not done by the
+        # extension-activation mechnism)
+        if not self.check_added(spec):
+            self.link_meta_folder(spec)
+
+        return True
+
+    def add_standalone(self, spec):
+        if spec.package.is_extension:
+            tty.error(self._croot + 'Package %s is an extension.'
+                      % spec.name)
+            return False
+
+        if self.check_added(spec):
+            tty.warn(self._croot + 'Skipping already linked package: %s'
+                     % colorize_spec(spec))
+            return True
+
+        tree = LinkTree(spec.prefix)
+
+        if not self.ignore_conflicts:
+            conflict = tree.find_conflict(self.root)
+            if conflict is not None:
+                tty.error(self._croot +
+                          "Cannot link package %s, file already exists: %s"
+                          % (spec.name, conflict))
+                return False
+
+        conflicts = tree.merge(self.root, link=self.link,
+                               ignore=ignore_metadata_dir,
+                               ignore_conflicts=self.ignore_conflicts)
+        self.link_meta_folder(spec)
+
+        if self.ignore_conflicts:
+            for c in conflicts:
+                tty.warn(self._croot + "Could not link: %s" % c)
+
+        if self.verbose:
+            tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec))
+        return True
+
+    def check_added(self, spec):
+        assert spec.concrete
+        return spec == self.get_spec(spec)
+
+    def remove_specs(self, *specs, **kwargs):
+        assert all((s.concrete for s in specs))
+        with_dependents = kwargs.get("with_dependents", True)
+        with_dependencies = kwargs.get("with_dependencies", False)
+
+        specs = set(specs)
+
+        if with_dependencies:
+            specs = get_dependencies(specs)
+
+        if kwargs.get("exclude", None):
+            specs = set(filter_exclude(specs, kwargs["exclude"]))
+
+        all_specs = set(self.get_all_specs())
+
+        to_deactivate = specs
+        to_keep = all_specs - to_deactivate
+
+        dependents = find_dependents(to_keep, to_deactivate)
+
+        if with_dependents:
+            # remove all packages depending on the ones to remove
+            if len(dependents) > 0:
+                tty.warn(self._croot +
+                         "The following dependents will be removed: %s"
+                         % ", ".join((s.name for s in dependents)))
+                to_deactivate.update(dependents)
+        elif len(dependents) > 0:
+            tty.warn(self._croot +
+                     "The following packages will be unusable: %s"
+                     % ", ".join((s.name for s in dependents)))
+
+        extensions = set(filter(lambda s: s.package.is_extension,
+                         to_deactivate))
+        standalones = to_deactivate - extensions
+
+        # Please note that a traversal of the DAG in post-order and then
+        # forcibly removing each package should remove the need to specify
+        # with_dependents for deactivating extensions/allow removal without
+        # additional checks (force=True). If removal performance becomes
+        # unbearable for whatever reason, this should be the first point of
+        # attack.
+        #
+        # see: https://github.com/LLNL/spack/pull/3227#discussion_r117147475
+        remove_extension = ft.partial(self.remove_extension,
+                                      with_dependents=with_dependents)
+
+        set(map(remove_extension, extensions))
+        set(map(self.remove_standalone, standalones))
+
+        self.purge_empty_directories()
+
+    def remove_extension(self, spec, with_dependents=True):
+        """
+            Remove (unlink) an extension from this view.
+        """
+        if not self.check_added(spec):
+            tty.warn(self._croot +
+                     'Skipping package not linked in view: %s' % spec.name)
+            return
+
+        # The spec might have been deactivated as depdency of another package
+        # already
+        if spec.package.is_activated(self.extensions_layout):
+            spec.package.do_deactivate(
+                verbose=self.verbose,
+                remove_dependents=with_dependents,
+                extensions_layout=self.extensions_layout)
+        self.unlink_meta_folder(spec)
+
+    def remove_standalone(self, spec):
+        """
+            Remove (unlink) a standalone package from this view.
+        """
+        if not self.check_added(spec):
+            tty.warn(self._croot +
+                     'Skipping package not linked in view: %s' % spec.name)
+            return
+
+        tree = LinkTree(spec.prefix)
+        tree.unmerge(self.root, ignore=ignore_metadata_dir)
+        self.unlink_meta_folder(spec)
+
+        if self.verbose:
+            tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec))
+
+    def get_all_specs(self):
+        dotspack = join_path(self.root, spack.store.layout.metadata_dir)
+        if os.path.exists(dotspack):
+            return list(filter(None, map(self.get_spec, os.listdir(dotspack))))
+        else:
+            return []
+
+    def get_conflicts(self, *specs):
+        """
+            Return list of tuples (<spec>, <spec in view>) where the spec
+            active in the view differs from the one to be activated.
+        """
+        in_view = map(self.get_spec, specs)
+        return [(s, v) for s, v in zip(specs, in_view)
+                if v is not None and s != v]
+
+    def get_path_meta_folder(self, spec):
+        "Get path to meta folder for either spec or spec name."
+        return join_path(self.root, spack.store.layout.metadata_dir,
+                         getattr(spec, "name", spec))
+
+    def get_spec(self, spec):
+        dotspack = self.get_path_meta_folder(spec)
+        filename = join_path(dotspack, spack.store.layout.spec_file_name)
+
+        try:
+            with open(filename, "r") as f:
+                return spack.spec.Spec.from_yaml(f)
+        except IOError:
+            return None
+
+    def link_meta_folder(self, spec):
+        src = spack.store.layout.metadata_path(spec)
+        tgt = self.get_path_meta_folder(spec)
+
+        tree = LinkTree(src)
+        # there should be no conflicts when linking the meta folder
+        tree.merge(tgt, link=self.link)
+
+    def print_conflict(self, spec_active, spec_specified, level="error"):
+        "Singular print function for spec conflicts."
+        cprint = getattr(tty, level)
+        color = sys.stdout.isatty()
+        linked    = tty.color.colorize("   (@gLinked@.)", color=color)
+        specified = tty.color.colorize("(@rSpecified@.)", color=color)
+        cprint(self._croot + "Package conflict detected:\n"
+               "%s %s\n" % (linked, colorize_spec(spec_active)) +
+               "%s %s" % (specified, colorize_spec(spec_specified)))
+
+    def print_status(self, *specs, **kwargs):
+        if kwargs.get("with_dependencies", False):
+            specs = set(get_dependencies(specs))
+
+        specs = sorted(specs, key=lambda s: s.name)
+        in_view = list(map(self.get_spec, specs))
+
+        for s, v in zip(specs, in_view):
+            if not v:
+                tty.error(self._croot +
+                          'Package not linked: %s' % s.name)
+            elif s != v:
+                self.print_conflict(v, s, level="warn")
+
+        in_view = list(filter(None, in_view))
+
+        if len(specs) > 0:
+            tty.msg("Packages linked in %s:" % self._croot[:-1])
+
+            # avoid circular dependency
+            import spack.cmd
+            spack.cmd.display_specs(in_view, flags=True, variants=True,
+                                    long=self.verbose)
+        else:
+            tty.warn(self._croot + "No packages found.")
+
+    def purge_empty_directories(self):
+        """
+            Ascend up from the leaves accessible from `path`
+            and remove empty directories.
+        """
+        for dirpath, subdirs, files in os.walk(self.root, topdown=False):
+            for sd in subdirs:
+                sdp = os.path.join(dirpath, sd)
+                try:
+                    os.rmdir(sdp)
+                except OSError:
+                    pass
+
+    def unlink_meta_folder(self, spec):
+        path = self.get_path_meta_folder(spec)
+        assert os.path.exists(path)
+        shutil.rmtree(path)
+
+    def _check_no_ext_conflicts(self, spec):
+        """
+            Check that there is no extension conflict for specs.
+        """
+        extendee = spec.package.extendee_spec
+        try:
+            self.extensions_layout.check_extension_conflict(extendee, spec)
+        except ExtensionAlreadyInstalledError:
+            # we print the warning here because later on the order in which
+            # packages get activated is not clear (set-sorting)
+            tty.warn(self._croot +
+                     'Skipping already activated package: %s' % spec.name)
+
+
+#####################
+# utility functions #
+#####################
+
+def colorize_root(root):
+    colorize = ft.partial(tty.color.colorize, color=sys.stdout.isatty())
+    pre, post = map(colorize, "@M[@. @M]@.".split())
+    return "".join([pre, root, post])
+
+
+def colorize_spec(spec):
+    "Colorize spec output if in TTY."
+    if sys.stdout.isatty():
+        return spec.cshort_spec
+    else:
+        return spec.short_spec
+
+
+def find_dependents(all_specs, providers, deptype='run'):
+    """
+        Return a set containing all those specs from all_specs that depend on
+        providers at the given dependency type.
+    """
+    dependents = set()
+    for s in all_specs:
+        for dep in s.traverse(deptype=deptype):
+            if dep in providers:
+                dependents.add(s)
+    return dependents
+
+
+def filter_exclude(specs, exclude):
+    "Filter specs given sequence of exclude regex"
+    to_exclude = [re.compile(e) for e in exclude]
+
+    def keep(spec):
+        for e in to_exclude:
+            if e.match(spec.name):
+                return False
+        return True
+    return filter(keep, specs)
+
+
+def get_dependencies(specs):
+    "Get set of dependencies (includes specs)"
+    retval = set()
+    set(map(retval.update, (set(s.traverse()) for s in specs)))
+    return retval
+
+
+def ignore_metadata_dir(f):
+    return f in spack.store.layout.hidden_file_paths