stacks: add option to link only roots or all specs, default to all

lib/spack/spack/environment.py

@@ -78,8 +78,12 @@
 env_schema_keys = ('spack', 'env')
 
 # Magic names
+# The name of the standalone spec list in the manifest yaml
 user_speclist_name = 'specs'
-def_view_name = 'default'
+# The name of the default view (the view loaded on env.activate)
+default_view_name = 'default'
+# Default behavior to link all packages into views (vs. only root packages)
+default_view_link = 'all'
 
 
 def valid_env_name(name):
@@ -145,7 +149,7 @@ def activate(
             cmds += 'export SPACK_OLD_PS1="${PS1}"; fi;\n'
             cmds += 'export PS1="%s ${PS1}";\n' % prompt
 
-    if add_view and def_view_name in env.views:
+    if add_view and default_view_name in env.views:
         cmds += env.add_default_view_to_shell(shell)
 
     return cmds
@@ -187,7 +191,7 @@ def deactivate(shell='sh'):
         cmds += 'unset SPACK_OLD_PS1; export SPACK_OLD_PS1;\n'
         cmds += 'fi;\n'
 
-    if def_view_name in _active_environment.views:
+    if default_view_name in _active_environment.views:
         cmds += _active_environment.rm_default_view_from_shell(shell)
 
     tty.debug("Deactivated environmennt '%s'" % _active_environment.name)
@@ -414,7 +418,8 @@ def _eval_conditional(string):
 
 
 class ViewDescriptor(object):
-    def __init__(self, root, projections={}, select=[], exclude=[]):
+    def __init__(self, root, projections={}, select=[], exclude=[],
+                 link=default_view_link):
         self.root = root
         self.projections = projections
         self.select = select
@@ -422,6 +427,7 @@ def __init__(self, root, projections={}, select=[], exclude=[]):
         self.exclude = exclude
         self.exclude_fn = lambda x: not any(x.satisfies(e)
                                             for e in self.exclude)
+        self.link = link
 
     def to_dict(self):
         ret = {'root': self.root}
@@ -431,6 +437,8 @@ def to_dict(self):
             ret['select'] = self.select
         if self.exclude:
             ret['exclude'] = self.exclude
+        if self.link != default_view_link:
+            ret['link'] = self.link
         return ret
 
     @staticmethod
@@ -438,19 +446,22 @@ def from_dict(d):
         return ViewDescriptor(d['root'],
                               d.get('projections', {}),
                               d.get('select', []),
-                              d.get('exclude', []))
+                              d.get('exclude', []),
+                              d.get('link', default_view_link))
 
     def view(self):
         return YamlFilesystemView(self.root, spack.store.layout,
                                   ignore_conflicts=True,
                                   projections=self.projections)
 
-    def regenerate(self, specs):
+    def regenerate(self, all_specs, roots):
         specs_for_view = []
+        specs = all_specs if self.link == 'all' else roots
         for spec in specs:
             # The view does not store build deps, so if we want it to
             # recognize environment specs (which do store build deps), then
             # they need to be stripped
+            if spec.concrete:  # Do not link unconcretized roots
                 specs_for_view.append(spack.spec.Spec.from_dict(
                     spec.to_dict(all_deps=False)
                 ))
@@ -523,9 +534,9 @@ def __init__(self, path, init_file=None, with_view=None):
             self.views = {}
         elif with_view is True:
             self.views = {
-                def_view_name: ViewDescriptor(self.view_path_default)}
+                default_view_name: ViewDescriptor(self.view_path_default)}
         elif isinstance(with_view, six.string_types):
-            self.views = {def_view_name: ViewDescriptor(with_view)}
+            self.views = {default_view_name: ViewDescriptor(with_view)}
         # If with_view is None, then defer to the view settings determined by
         # the manifest file
 
@@ -556,9 +567,9 @@ def _read_manifest(self, f):
         # enable_view can be boolean, string, or None
         if enable_view is True or enable_view is None:
             self.views = {
-                def_view_name: ViewDescriptor(self.view_path_default)}
+                default_view_name: ViewDescriptor(self.view_path_default)}
         elif isinstance(enable_view, six.string_types):
-            self.views = {def_view_name: ViewDescriptor(enable_view)}
+            self.views = {default_view_name: ViewDescriptor(enable_view)}
         elif enable_view:
             self.views = dict((name, ViewDescriptor.from_dict(values))
                               for name, values in enable_view.items())
@@ -881,23 +892,24 @@ def default_view(self):
             raise SpackEnvironmentError(
                 "{0} does not have a view enabled".format(self.name))
 
-        if def_view_name not in self.views:
+        if default_view_name not in self.views:
             raise SpackEnvironmentError(
                 "{0} does not have a default view enabled".format(self.name))
 
-        return self.views[def_view_name]
+        return self.views[default_view_name]
 
     def update_default_view(self, viewpath):
-        if def_view_name in self.views and self.default_view.root != viewpath:
+        name = default_view_name
+        if name in self.views and self.default_view.root != viewpath:
             shutil.rmtree(self.default_view.root)
 
         if viewpath:
-            if def_view_name in self.views:
+            if name in self.views:
                 self.default_view.root = viewpath
             else:
-                self.views[def_view_name] = ViewDescriptor(viewpath)
+                self.views[name] = ViewDescriptor(viewpath)
         else:
-            self.views.pop(def_view_name, None)
+            self.views.pop(name, None)
 
     def regenerate_views(self):
         if not self.views:
@@ -907,7 +919,7 @@ def regenerate_views(self):
 
         specs = self._get_environment_specs()
         for view in self.views.values():
-            view.regenerate(specs)
+            view.regenerate(specs, self.roots())
 
     def _shell_vars(self):
         updates = [
@@ -921,7 +933,7 @@ def _shell_vars(self):
             ('CMAKE_PREFIX_PATH', ['']),
         ]
         path_updates = list()
-        if def_view_name in self.views:
+        if default_view_name in self.views:
             for var, subdirs in updates:
                 paths = filter(lambda x: os.path.exists(x),
                                list(os.path.join(self.default_view.root, x)
@@ -1198,7 +1210,8 @@ def write(self):
                                                            [])
         yaml_spec_list[:] = self.user_specs.yaml_list
 
-        if self.views and len(self.views) == 1 and def_view_name in self.views:
+        default_name = default_view_name
+        if self.views and len(self.views) == 1 and default_name in self.views:
             path = self.default_view.root
             if self.default_view == ViewDescriptor(self.view_path_default):
                 view = True

lib/spack/spack/schema/env.py

@@ -96,6 +96,10 @@
                                           'root': {
                                               'type': 'string'
                                           },
+                                          'link': {
+                                              'type': 'string',
+                                              'pattern': '(roots|all)',
+                                          },
                                           'select': {
                                               'type': 'array',
                                               'items': {

lib/spack/spack/test/cmd/env.py

@@ -32,10 +32,9 @@
 find       = SpackCommand('find')
 
 
-def check_mpileaks_install(viewdir):
+def check_mpileaks_and_deps_in_view(viewdir):
     """Check that the expected install directories exist."""
     assert os.path.exists(str(viewdir.join('.spack', 'mpileaks')))
-    # Check that dependencies got in too
     assert os.path.exists(str(viewdir.join('.spack', 'libdwarf')))
 
 
@@ -634,7 +633,7 @@ def test_env_updates_view_install(
         add('mpileaks')
         install('--fake')
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
 
 def test_env_without_view_install(
@@ -656,7 +655,7 @@ def test_env_without_view_install(
 
     # After enabling the view, the specs should be linked into the environment
     # view dir
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
 
 def test_env_config_view_default(
@@ -686,7 +685,7 @@ def test_env_updates_view_install_package(
     with ev.read('test'):
         install('--fake', 'mpileaks')
 
-    check_mpileaks_install(view_dir)
+    assert os.path.exists(str(view_dir.join('.spack/mpileaks')))
 
 
 def test_env_updates_view_add_concretize(
@@ -698,7 +697,7 @@ def test_env_updates_view_add_concretize(
         add('mpileaks')
         concretize()
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
 
 def test_env_updates_view_uninstall(
@@ -708,7 +707,7 @@ def test_env_updates_view_uninstall(
     with ev.read('test'):
         install('--fake', 'mpileaks')
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
     with ev.read('test'):
         uninstall('-ay')
@@ -725,7 +724,7 @@ def test_env_updates_view_uninstall_referenced_elsewhere(
         add('mpileaks')
         concretize()
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
     with ev.read('test'):
         uninstall('-ay')
@@ -742,7 +741,7 @@ def test_env_updates_view_remove_concretize(
         add('mpileaks')
         concretize()
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
     with ev.read('test'):
         remove('mpileaks')
@@ -758,7 +757,7 @@ def test_env_updates_view_force_remove(
     with ev.read('test'):
         install('--fake', 'mpileaks')
 
-    check_mpileaks_install(view_dir)
+    check_mpileaks_and_deps_in_view(view_dir)
 
     with ev.read('test'):
         remove('-f', 'mpileaks')
@@ -1123,7 +1122,7 @@ def test_stack_combinatorial_view(tmpdir, mock_fetch, mock_packages,
             install()
 
         test = ev.read('test')
-        for _, spec in test.concretized_specs():
+        for spec in test._get_environment_specs():
             assert os.path.exists(
                 os.path.join(viewdir, spec.name, '%s-%s' %
                              (spec.version, spec.compiler.name)))
@@ -1156,7 +1155,7 @@ def test_stack_view_select(tmpdir, mock_fetch, mock_packages,
             install()
 
         test = ev.read('test')
-        for _, spec in test.concretized_specs():
+        for spec in test._get_environment_specs():
             if spec.satisfies('%gcc'):
                 assert os.path.exists(
                     os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1194,7 +1193,7 @@ def test_stack_view_exclude(tmpdir, mock_fetch, mock_packages,
             install()
 
         test = ev.read('test')
-        for _, spec in test.concretized_specs():
+        for spec in test._get_environment_specs():
             if not spec.satisfies('callpath'):
                 assert os.path.exists(
                     os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1233,7 +1232,88 @@ def test_stack_view_select_and_exclude(tmpdir, mock_fetch, mock_packages,
             install()
 
         test = ev.read('test')
-        for _, spec in test.concretized_specs():
+        for spec in test._get_environment_specs():
+            if spec.satisfies('%gcc') and not spec.satisfies('callpath'):
+                assert os.path.exists(
+                    os.path.join(viewdir, spec.name, '%s-%s' %
+                                 (spec.version, spec.compiler.name)))
+            else:
+                assert not os.path.exists(
+                    os.path.join(viewdir, spec.name, '%s-%s' %
+                                 (spec.version, spec.compiler.name)))
+
+
+def test_view_link_roots(tmpdir, mock_fetch, mock_packages, mock_archive,
+                         install_mockery):
+    filename = str(tmpdir.join('spack.yaml'))
+    viewdir = str(tmpdir.join('view'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+    - compilers: ['%%gcc', '%%clang']
+  specs:
+    - matrix:
+        - [$packages]
+        - [$compilers]
+
+  view:
+    combinatorial:
+      root: %s
+      select: ['%%gcc']
+      exclude: [callpath]
+      link: 'roots'
+      projections:
+        'all': '{name}/{version}-{compiler.name}'""" % viewdir)
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            install()
+
+        test = ev.read('test')
+        for spec in test._get_environment_specs():
+            if spec in test.roots() and (spec.satisfies('%gcc') and
+                                         not spec.satisfies('callpath')):
+                assert os.path.exists(
+                    os.path.join(viewdir, spec.name, '%s-%s' %
+                                 (spec.version, spec.compiler.name)))
+            else:
+                assert not os.path.exists(
+                    os.path.join(viewdir, spec.name, '%s-%s' %
+                                 (spec.version, spec.compiler.name)))
+
+
+def test_view_link_all(tmpdir, mock_fetch, mock_packages, mock_archive,
+                       install_mockery):
+    filename = str(tmpdir.join('spack.yaml'))
+    viewdir = str(tmpdir.join('view'))
+    with open(filename, 'w') as f:
+        f.write("""\
+env:
+  definitions:
+    - packages: [mpileaks, callpath]
+    - compilers: ['%%gcc', '%%clang']
+  specs:
+    - matrix:
+        - [$packages]
+        - [$compilers]
+
+  view:
+    combinatorial:
+      root: %s
+      select: ['%%gcc']
+      exclude: [callpath]
+      link: 'all'
+      projections:
+        'all': '{name}/{version}-{compiler.name}'""" % viewdir)
+    with tmpdir.as_cwd():
+        env('create', 'test', './spack.yaml')
+        with ev.read('test'):
+            install()
+
+        test = ev.read('test')
+        for spec in test._get_environment_specs():
             if spec.satisfies('%gcc') and not spec.satisfies('callpath'):
                 assert os.path.exists(
                     os.path.join(viewdir, spec.name, '%s-%s' %
@@ -1342,7 +1422,7 @@ def test_stack_view_multiple_views(tmpdir, mock_fetch, mock_packages,
         assert os.path.join(default_viewdir, 'bin') in shell
 
         test = ev.read('test')
-        for _, spec in test.concretized_specs():
+        for spec in test._get_environment_specs():
             if not spec.satisfies('callpath%gcc'):
                 assert os.path.exists(
                     os.path.join(combin_viewdir, spec.name, '%s-%s' %