environment.py: allow link:run (#29336)

* environment.py: allow link:run

Some users want minimal views, excluding run-type dependencies, since
those type of dependencies are covered by rpaths and the symlinked
libraries in the view aren't used anyways.

With this change, an environment like this:

```
spack:
  specs: ['py-flake8']
  view:
    default:
      root: view
      link: run
```

includes python packages and python, but no link type deps of python.

lib/spack/docs/environments.rst

@@ -740,9 +740,10 @@ file snippet we define a view named ``mpis``, rooted at
 version, and compiler name to determine the path for a given
 package. This view selects all packages that depend on MPI, and
 excludes those built with the PGI compiler at version 18.5.
-All the dependencies of each root spec in the environment will be linked
+The root specs with their (transitive) link and run type dependencies
-in the view due to the command ``link: all`` and the files in the view will
+will be put in the view due to the  ``link: all`` option,
-be symlinks to the spack install directories.
+and the files in the view will be symlinks to the spack install
+directories.
 
 .. code-block:: yaml
 
@@ -762,9 +763,22 @@ For more information on using view projections, see the section on
 :ref:`adding_projections_to_views`. The default for the ``select`` and
 ``exclude`` values is to select everything and exclude nothing. The
 default projection is the default view projection (``{}``). The ``link``
-defaults to ``all`` but can also be ``roots`` when only the root specs
+attribute allows the following values:
-in the environment are desired in the view. The ``link_type`` defaults
+
-to ``symlink`` but can also take the value of ``hardlink`` or ``copy``.
+#. ``link: all`` include root specs with their transitive run and link type
+   dependencies (default);
+#. ``link: run`` include root specs with their transitive run type dependencies;
+#. ``link: roots`` include root specs without their dependencies.
+
+The ``link_type`` defaults to ``symlink`` but can also take the value
+of ``hardlink`` or ``copy``.
+
+.. tip::
+
+   The option ``link: run`` can be used to create small environment views for
+   Python packages. Python will be able to import packages *inside* of the view even
+   when the environment is not activated, and linked libraries will be located
+   *outside* of the view thanks to rpaths.
 
 Any number of views may be defined under the ``view`` heading in a
 Spack Environment.

lib/spack/llnl/util/lang.py

@@ -589,20 +589,31 @@ def match(string):
     return match
 
 
-def dedupe(sequence):
+def dedupe(sequence, key=None):
-    """Yields a stable de-duplication of an hashable sequence
+    """Yields a stable de-duplication of an hashable sequence by key
 
     Args:
         sequence: hashable sequence to be de-duplicated
+        key: callable applied on values before uniqueness test; identity
+            by default.
 
     Returns:
         stable de-duplication of the sequence
+
+    Examples:
+
+        Dedupe a list of integers:
+
+            [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]
+
+            [x for x in llnl.util.lang.dedupe([1,-2,1,3,2], key=abs)] == [1, -2, 3]
     """
     seen = set()
     for x in sequence:
-        if x not in seen:
+        x_key = x if key is None else key(x)
+        if x_key not in seen:
             yield x
-            seen.add(x)
+            seen.add(x_key)
 
 
 def pretty_date(time, now=None):

lib/spack/spack/environment/environment.py

@@ -16,6 +16,7 @@
 
 import llnl.util.filesystem as fs
 import llnl.util.tty as tty
+from llnl.util.lang import dedupe
 
 import spack.bootstrap
 import spack.compilers
@@ -471,80 +472,80 @@ def __contains__(self, spec):
 
         return True
 
-    def specs_for_view(self, all_specs, roots):
+    def specs_for_view(self, concretized_specs):
-        specs_for_view = []
+        """
-        specs = all_specs if self.link == 'all' else roots
+        From the list of concretized user specs in the environment, flatten
+        the dags, and filter selected, installed specs, remove duplicates on dag hash.
+        """
+        specs = []
 
-        for spec in specs:
+        for (_, s) in concretized_specs:
-            # The view does not store build deps, so if we want it to
+            if self.link == 'all':
-            # recognize environment specs (which do store build deps),
+                specs.extend(s.traverse(deptype=('link', 'run')))
-            # then they need to be stripped.
+            elif self.link == 'run':
-            if spec.concrete:  # Do not link unconcretized roots
+                specs.extend(s.traverse(deptype=('run')))
-                # We preserve _hash _normal to avoid recomputing DAG
+            else:
-                # hashes (DAG hashes don't consider build deps)
+                specs.append(s)
-                spec_copy = spec.copy(deps=('link', 'run'))
-                spec_copy._hash = spec._hash
-                spec_copy._normal = spec._normal
-                specs_for_view.append(spec_copy)
-        return specs_for_view
 
-    def regenerate(self, all_specs, roots):
+        # De-dupe by dag hash
-        specs_for_view = self.specs_for_view(all_specs, roots)
+        specs = dedupe(specs, key=lambda s: s.dag_hash())
 
-        # regeneration queries the database quite a bit; this read
+        # Filter selected, installed specs
-        # transaction ensures that we don't repeatedly lock/unlock.
         with spack.store.db.read_transaction():
-            installed_specs_for_view = set(
+            specs = [s for s in specs if s in self and s.package.installed]
-                s for s in specs_for_view if s in self and s.package.installed)
 
-            # To ensure there are no conflicts with packages being installed
+        return specs
-            # that cannot be resolved or have repos that have been removed
-            # we always regenerate the view from scratch.
-            # We will do this by hashing the view contents and putting the view
-            # in a directory by hash, and then having a symlink to the real
-            # view in the root. The real root for a view at /dirname/basename
-            # will be /dirname/._basename_<hash>.
-            # This allows for atomic swaps when we update the view
 
-            # cache the roots because the way we determine which is which does
+    def regenerate(self, concretized_specs):
-            # not work while we are updating
+        specs = self.specs_for_view(concretized_specs)
-            new_root = self._next_root(installed_specs_for_view)
-            old_root = self._current_root
 
-            if new_root == old_root:
+        # To ensure there are no conflicts with packages being installed
-                tty.debug("View at %s does not need regeneration." % self.root)
+        # that cannot be resolved or have repos that have been removed
-                return
+        # we always regenerate the view from scratch.
+        # We will do this by hashing the view contents and putting the view
+        # in a directory by hash, and then having a symlink to the real
+        # view in the root. The real root for a view at /dirname/basename
+        # will be /dirname/._basename_<hash>.
+        # This allows for atomic swaps when we update the view
 
-            # construct view at new_root
+        # cache the roots because the way we determine which is which does
-            tty.msg("Updating view at {0}".format(self.root))
+        # not work while we are updating
+        new_root = self._next_root(specs)
+        old_root = self._current_root
 
-            view = self.view(new=new_root)
+        if new_root == old_root:
-            fs.mkdirp(new_root)
+            tty.debug("View at %s does not need regeneration." % self.root)
-            view.add_specs(*installed_specs_for_view,
+            return
-                           with_dependencies=False)
 
-            # create symlink from tmpname to new_root
+        # construct view at new_root
-            root_dirname = os.path.dirname(self.root)
+        tty.msg("Updating view at {0}".format(self.root))
-            tmp_symlink_name = os.path.join(root_dirname, '._view_link')
-            if os.path.exists(tmp_symlink_name):
-                os.unlink(tmp_symlink_name)
-            os.symlink(new_root, tmp_symlink_name)
 
-            # mv symlink atomically over root symlink to old_root
+        view = self.view(new=new_root)
-            if os.path.exists(self.root) and not os.path.islink(self.root):
+        fs.mkdirp(new_root)
-                msg = "Cannot create view: "
+        view.add_specs(*specs, with_dependencies=False)
-                msg += "file already exists and is not a link: %s" % self.root
-                raise SpackEnvironmentViewError(msg)
-            os.rename(tmp_symlink_name, self.root)
 
-            # remove old_root
+        # create symlink from tmpname to new_root
-            if old_root and os.path.exists(old_root):
+        root_dirname = os.path.dirname(self.root)
-                try:
+        tmp_symlink_name = os.path.join(root_dirname, '._view_link')
-                    shutil.rmtree(old_root)
+        if os.path.exists(tmp_symlink_name):
-                except (IOError, OSError) as e:
+            os.unlink(tmp_symlink_name)
-                    msg = "Failed to remove old view at %s\n" % old_root
+        os.symlink(new_root, tmp_symlink_name)
-                    msg += str(e)
+
-                    tty.warn(msg)
+        # mv symlink atomically over root symlink to old_root
+        if os.path.exists(self.root) and not os.path.islink(self.root):
+            msg = "Cannot create view: "
+            msg += "file already exists and is not a link: %s" % self.root
+            raise SpackEnvironmentViewError(msg)
+        os.rename(tmp_symlink_name, self.root)
+
+        # remove old_root
+        if old_root and os.path.exists(old_root):
+            try:
+                shutil.rmtree(old_root)
+            except (IOError, OSError) as e:
+                msg = "Failed to remove old view at %s\n" % old_root
+                msg += str(e)
+                tty.warn(msg)
 
 
 def _create_environment(*args, **kwargs):
@@ -1303,9 +1304,8 @@ def regenerate_views(self):
                       " maintain a view")
             return
 
-        specs = self._get_environment_specs()
         for view in self.views.values():
-            view.regenerate(specs, self.roots())
+            view.regenerate(self.concretized_specs())
 
     def check_views(self):
         """Checks if the environments default view can be activated."""

lib/spack/spack/schema/env.py

@@ -124,7 +124,7 @@
                                             },
                                             'link': {
                                                 'type': 'string',
-                                                'pattern': '(roots|all)',
+                                                'pattern': '(roots|all|run)',
                                             },
                                             'link_type': {
                                                 'type': 'string'

lib/spack/spack/test/cmd/env.py

@@ -386,23 +386,23 @@ def test_environment_status(capsys, tmpdir):
 
 def test_env_status_broken_view(
     mutable_mock_env_path, mock_archive, mock_fetch, mock_packages,
-    install_mockery
+    install_mockery, tmpdir
 ):
-    with ev.create('test'):
+    env_dir = str(tmpdir)
+    with ev.Environment(env_dir):
         install('trivial-install-test-package')
 
-        # switch to a new repo that doesn't include the installed package
+    # switch to a new repo that doesn't include the installed package
-        # test that Spack detects the missing package and warns the user
+    # test that Spack detects the missing package and warns the user
-        new_repo = MockPackageMultiRepo()
+    with spack.repo.use_repositories(MockPackageMultiRepo()):
-        with spack.repo.use_repositories(new_repo):
+        with ev.Environment(env_dir):
             output = env('status')
-            assert 'In environment test' in output
+            assert 'includes out of date packages or repos' in output
-            assert 'Environment test includes out of date' in output
 
-        # Test that the warning goes away when it's fixed
+    # Test that the warning goes away when it's fixed
+    with ev.Environment(env_dir):
         output = env('status')
-        assert 'In environment test' in output
+        assert 'includes out of date packages or repos' not in output
-        assert 'Environment test includes out of date' not in output
 
 
 def test_env_activate_broken_view(
@@ -1962,6 +1962,37 @@ def test_view_link_roots(tmpdir, mock_fetch, mock_packages, mock_archive,
                                  (spec.version, spec.compiler.name)))
 
 
+def test_view_link_run(tmpdir, mock_fetch, mock_packages, mock_archive,
+                       install_mockery):
+    yaml = str(tmpdir.join('spack.yaml'))
+    viewdir = str(tmpdir.join('view'))
+    envdir = str(tmpdir)
+    with open(yaml, 'w') as f:
+        f.write("""
+spack:
+  specs:
+  - dttop
+
+  view:
+    combinatorial:
+      root: %s
+      link: run
+      projections:
+        all: '{name}'""" % viewdir)
+
+    with ev.Environment(envdir):
+        install()
+
+    # make sure transitive run type deps are in the view
+    for pkg in ('dtrun1', 'dtrun3'):
+        assert os.path.exists(os.path.join(viewdir, pkg))
+
+    # and non-run-type deps are not.
+    for pkg in ('dtlink1', 'dtlink2', 'dtlink3', 'dtlink4', 'dtlink5'
+                'dtbuild1', 'dtbuild2', 'dtbuild3'):
+        assert not os.path.exists(os.path.join(viewdir, pkg))
+
+
 @pytest.mark.parametrize('link_type', ['hardlink', 'copy', 'symlink'])
 def test_view_link_type(link_type, tmpdir, mock_fetch, mock_packages, mock_archive,
                         install_mockery):

lib/spack/spack/test/llnl/util/lang.py

@@ -10,7 +10,7 @@
 import pytest
 
 import llnl.util.lang
-from llnl.util.lang import match_predicate, memoized, pretty_date, stable_args
+from llnl.util.lang import dedupe, match_predicate, memoized, pretty_date, stable_args
 
 
 @pytest.fixture()
@@ -265,3 +265,8 @@ def f(*args, **kwargs):
     key = stable_args(*args, **kwargs)
     assert str(key) in exc_msg
     assert "function 'f'" in exc_msg
+
+
+def test_dedupe():
+    assert [x for x in dedupe([1, 2, 1, 3, 2])] == [1, 2, 3]
+    assert [x for x in dedupe([1, -2, 1, 3, 2], key=abs)] == [1, -2, 3]