Allow combinatorial projections in views (#9679)

Allow customizing views with Spec-formatted directory structure

Allow views to specify projections that are more complicated than
merging every package into a single shared prefix. This will allow
sites to configure a view for the way they want to present packages
to their users; for example this can be used to create a prefix for
each package but omit the DAG hash from the path.

This includes a new YAML format file for specifying the simplified
prefix for a spec in a view. This configuration allows the use of
different prefix formats for different specs (i.e. specs depending
on MPI can include the MPI implementation in the prefix).
Documentation on usage of the view projection configuration is
included.

Depending on the projection configuration, paths are not guaranteed
to be unique and it may not be possible to add multiple installs of
a package to a view.

var/spack/repos/builtin.mock/packages/perl-extension/package.py

@@ -0,0 +1,28 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack import *
+import os.path
+
+
+class PerlExtension(PerlPackage):
+    """A package which extends perl"""
+
+    homepage = "http://www.example.com"
+    url      = "http://www.example.com/extension1-1.0.tar.gz"
+
+    version('1.0', 'hash-extension-1.0')
+    version('2.0', 'hash-extension-2.0')
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        with open(os.path.join(prefix.bin, 'perl-extension'), 'w+') as fout:
+            fout.write(str(spec.version))
+
+    # Give the package a hook to set the extendee spec
+    extends_spec = 'perl'
+
+    @property
+    def extendee_spec(self):
+        return self.extends_spec

var/spack/repos/builtin.mock/packages/perl/package.py

@@ -0,0 +1,18 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack import *
+
+
+class Perl(Package):
+    """Dummy Perl package to allow a dummy perl-extension in repo."""
+    homepage = "http://www.python.org"
+    url      = "http://www.python.org/ftp/python/2.7.8/Python-2.7.8.tgz"
+
+    extendable = True
+
+    version('0.0.0', 'hash')
+
+    def install(self, spec, prefix):
+        pass

var/spack/repos/builtin.mock/packages/py-extension1/package.py

@@ -0,0 +1,28 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack import *
+import os.path
+
+
+class PyExtension1(PythonPackage):
+    """A package which extends python"""
+
+    homepage = "http://www.example.com"
+    url      = "http://www.example.com/extension1-1.0.tar.gz"
+
+    version('1.0', 'hash-extension1-1.0')
+    version('2.0', 'hash-extension1-2.0')
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        with open(os.path.join(prefix.bin, 'py-extension1'), 'w+') as fout:
+            fout.write(str(spec.version))
+
+    # Give the package a hook to set the extendee spec
+    extends_spec = 'python'
+
+    @property
+    def extendee_spec(self):
+        return self.extends_spec

var/spack/repos/builtin.mock/packages/py-extension2/package.py

@@ -0,0 +1,30 @@
+# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+from spack import *
+import os.path
+
+
+class PyExtension2(PythonPackage):
+    """A package which extends python. It also depends on another
+       package which extends the same package."""
+
+    homepage = "http://www.example.com"
+    url      = "http://www.example.com/extension2-1.0.tar.gz"
+
+    depends_on('py-extension1', type=('build', 'run'))
+
+    version('1.0', 'hash-extension2-1.0')
+
+    def install(self, spec, prefix):
+        mkdirp(prefix.bin)
+        with open(os.path.join(prefix.bin, 'py-extension2'), 'w+') as fout:
+            fout.write(str(spec.version))
+
+    # Give the package a hook to set the extendee spec
+    extends_spec = 'python'
+
+    @property
+    def extendee_spec(self):
+        return self.extends_spec

var/spack/repos/builtin/packages/python/package.py

@@ -694,7 +694,9 @@ def activate(self, ext_pkg, view, **args):
         exts = extensions_layout.extension_map(self.spec)
         exts[ext_pkg.name] = ext_pkg.spec
 
-        self.write_easy_install_pth(exts, prefix=view.root)
+        self.write_easy_install_pth(exts, prefix=view.get_projection_for_spec(
+            self.spec
+        ))
 
     def deactivate(self, ext_pkg, view, **args):
         args.update(ignore=self.python_ignore(ext_pkg, args))
@@ -706,7 +708,10 @@ def deactivate(self, ext_pkg, view, **args):
         # Make deactivate idempotent
         if ext_pkg.name in exts:
             del exts[ext_pkg.name]
-            self.write_easy_install_pth(exts, prefix=view.root)
+            self.write_easy_install_pth(exts,
+                                        prefix=view.get_projection_for_spec(
+                                            self.spec
+                                        ))
 
     def add_files_to_view(self, view, merge_map):
         bin_dir = self.spec.prefix.bin
@@ -717,7 +722,13 @@ def add_files_to_view(self, view, merge_map):
                 copy(src, dst)
                 if 'script' in get_filetype(src):
                     filter_file(
-                        self.spec.prefix, os.path.abspath(view.root), dst)
+                        self.spec.prefix,
+                        os.path.abspath(
+                            view.get_projection_for_spec(self.spec)
+                        ),
+                        dst,
+                        backup=False
+                    )
             else:
                 orig_link_target = os.path.realpath(src)
                 new_link_target = os.path.abspath(merge_map[orig_link_target])