Feature: add option to create view by copying/relocating files (#16480)

* add subcommand `spack view copy/relocate`

* update bash completions

* add copy/relocate commands to view tests

* allow copied views to be removed

lib/spack/spack/cmd/view.py

@@ -33,8 +33,6 @@
 YamlFilesystemView.
 
 '''
-import os
-
 import llnl.util.tty as tty
 from llnl.util.link_tree import MergeConflictError
 from llnl.util.tty.color import colorize
@@ -45,13 +43,15 @@
 import spack.schema.projections
 from spack.config import validate
 from spack.filesystem_view import YamlFilesystemView
+from spack.filesystem_view import view_symlink, view_hardlink, view_copy
 from spack.util import spack_yaml as s_yaml
 
 description = "project packages to a compact naming scheme on the filesystem."
 section = "environments"
 level = "short"
 
-actions_link = ["symlink", "add", "soft", "hardlink", "hard"]
+actions_link = ["symlink", "add", "soft", "hardlink", "hard", "copy",
+                "relocate"]
 actions_remove = ["remove", "rm"]
 actions_status = ["statlink", "status", "check"]
 
@@ -112,6 +112,9 @@ def setup_parser(sp):
         "hardlink": ssp.add_parser(
             'hardlink', aliases=['hard'],
             help='add packages files to a filesystem view via hard links'),
+        "copy": ssp.add_parser(
+            'copy', aliases=['relocate'],
+            help='add package files to a filesystem view via copy/relocate'),
         "remove": ssp.add_parser(
             'remove', aliases=['rm'],
             help='remove packages from a filesystem view'),
@@ -125,7 +128,7 @@ def setup_parser(sp):
         act.add_argument('path', nargs=1,
                          help="path to file system view directory")
 
-        if cmd in ("symlink", "hardlink"):
+        if cmd in ("symlink", "hardlink", "copy"):
             # invalid for remove/statlink, for those commands the view needs to
             # already know its own projections.
             help_msg = "Initialize view using projections from file."
@@ -157,7 +160,7 @@ def setup_parser(sp):
             so["nargs"] = "+"
             act.add_argument('specs', **so)
 
-    for cmd in ["symlink", "hardlink"]:
+    for cmd in ["symlink", "hardlink", "copy"]:
         act = file_system_view_actions[cmd]
         act.add_argument("-i", "--ignore-conflicts", action='store_true')
 
@@ -179,11 +182,19 @@ def view(parser, args):
     else:
         ordered_projections = {}
 
+    # What method are we using for this view
+    if args.action in ("hardlink", "hard"):
+        link_fn = view_hardlink
+    elif args.action in ("copy", "relocate"):
+        link_fn = view_copy
+    else:
+        link_fn = view_symlink
+
     view = YamlFilesystemView(
         path, spack.store.layout,
         projections=ordered_projections,
         ignore_conflicts=getattr(args, "ignore_conflicts", False),
-        link=os.link if args.action in ["hardlink", "hard"] else os.symlink,
+        link=link_fn,
         verbose=args.verbose)
 
     # Process common args and specs

lib/spack/spack/filesystem_view.py

@@ -24,6 +24,7 @@
 import spack.schema.projections
 import spack.projections
 import spack.config
+import spack.relocate
 from spack.error import SpackError
 from spack.directory_layout import ExtensionAlreadyInstalledError
 from spack.directory_layout import YamlViewExtensionsLayout
@@ -41,6 +42,58 @@
 _projections_path = '.spack/projections.yaml'
 
 
+def view_symlink(src, dst, **kwargs):
+    # keyword arguments are irrelevant
+    # here to fit required call signature
+    os.symlink(src, dst)
+
+
+def view_hardlink(src, dst, **kwargs):
+    # keyword arguments are irrelevant
+    # here to fit required call signature
+    os.link(src, dst)
+
+
+def view_copy(src, dst, view, spec=None):
+    """
+    Copy a file from src to dst.
+
+    Use spec and view to generate relocations
+    """
+    shutil.copyfile(src, dst)
+    if spec:
+        # Not metadata, we have to relocate it
+
+        # Get information on where to relocate from/to
+        prefix_to_projection = dict(
+            (dep.prefix, view.get_projection_for_spec(dep))
+            for dep in spec.traverse()
+        )
+
+        if spack.relocate.is_binary(dst):
+            # relocate binaries
+            spack.relocate.relocate_text_bin(
+                binaries=[dst],
+                orig_install_prefix=spec.prefix,
+                new_install_prefix=view.get_projection_for_spec(spec),
+                orig_spack=spack.paths.spack_root,
+                new_spack=view._root,
+                new_prefixes=prefix_to_projection
+            )
+        else:
+            # relocate text
+            spack.relocate.relocate_text(
+                files=[dst],
+                orig_layout_root=spack.store.layout.root,
+                new_layout_root=view._root,
+                orig_install_prefix=spec.prefix,
+                new_install_prefix=view.get_projection_for_spec(spec),
+                orig_spack=spack.paths.spack_root,
+                new_spack=view._root,
+                new_prefixes=prefix_to_projection
+            )
+
+
 class FilesystemView(object):
     """
         Governs a filesystem view that is located at certain root-directory.
@@ -67,9 +120,12 @@ def __init__(self, root, layout, **kwargs):
         self.projections = kwargs.get('projections', {})
 
         self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
-        self.link = kwargs.get("link", os.symlink)
         self.verbose = kwargs.get("verbose", False)
 
+        # Setup link function to include view
+        link_func = kwargs.get("link", view_symlink)
+        self.link = ft.partial(link_func, view=self)
+
     def add_specs(self, *specs, **kwargs):
         """
             Add given specs to view.
@@ -355,8 +411,6 @@ def remove_file(self, src, dest):
         if not os.path.lexists(dest):
             tty.warn("Tried to remove %s which does not exist" % dest)
             return
-        if not os.path.islink(dest):
-            raise ValueError("%s is not a link tree!" % dest)
         # remove if dest is a hardlink/symlink to src; this will only
         # be false if two packages are merged into a prefix and have a
         # conflicting file

lib/spack/spack/package.py

@@ -332,7 +332,7 @@ def add_files_to_view(self, view, merge_map):
         """
         for src, dst in merge_map.items():
             if not os.path.exists(dst):
-                view.link(src, dst)
+                view.link(src, dst, spec=self.spec)
 
     def remove_files_from_view(self, view, merge_map):
         """Given a map of package files to files currently linked in the view,

lib/spack/spack/test/cmd/view.py

@@ -24,7 +24,8 @@ def create_projection_file(tmpdir, projection):
     return projection_file
 
 
-@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
+@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add',
+                                 'copy', 'relocate'])
 def test_view_link_type(
         tmpdir, mock_packages, mock_archive, mock_fetch, config,
         install_mockery, cmd):
@@ -33,10 +34,14 @@ def test_view_link_type(
     view(cmd, viewpath, 'libdwarf')
     package_prefix = os.path.join(viewpath, 'libdwarf')
     assert os.path.exists(package_prefix)
-    assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
+
+    # Check that we use symlinks for and only for the appropriate subcommands
+    is_link_cmd = cmd in ('symlink', 'add')
+    assert os.path.islink(package_prefix) == is_link_cmd
 
 
-@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
+@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add',
+                                 'copy', 'relocate'])
 def test_view_projections(
         tmpdir, mock_packages, mock_archive, mock_fetch, config,
         install_mockery, cmd):
@@ -54,7 +59,10 @@ def test_view_projections(
 
     package_prefix = os.path.join(viewpath, 'libdwarf-20130207/libdwarf')
     assert os.path.exists(package_prefix)
-    assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
+
+    # Check that we use symlinks for and only for the appropriate subcommands
+    is_symlink_cmd = cmd in ('symlink', 'add')
+    assert os.path.islink(package_prefix) == is_symlink_cmd
 
 
 def test_view_multiple_projections(

share/spack/spack-completion.bash

@@ -1521,7 +1521,7 @@ _spack_view() {
     then
         SPACK_COMPREPLY="-h --help -v --verbose -e --exclude -d --dependencies"
     else
-        SPACK_COMPREPLY="symlink add soft hardlink hard remove rm statlink status check"
+        SPACK_COMPREPLY="symlink add soft hardlink hard copy relocate remove rm statlink status check"
     fi
 }
 
@@ -1570,6 +1570,24 @@ _spack_view_hard() {
     fi
 }
 
+_spack_view_copy() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --projection-file -i --ignore-conflicts"
+    else
+        _all_packages
+    fi
+}
+
+_spack_view_relocate() {
+    if $list_options
+    then
+        SPACK_COMPREPLY="-h --help --projection-file -i --ignore-conflicts"
+    else
+        _all_packages
+    fi
+}
+
 _spack_view_remove() {
     if $list_options
     then

var/spack/repos/builtin/packages/python/package.py

@@ -962,7 +962,7 @@ def add_files_to_view(self, view, merge_map):
         bin_dir = self.spec.prefix.bin
         for src, dst in merge_map.items():
             if not path_contains_subdirectory(src, bin_dir):
-                view.link(src, dst)
+                view.link(src, dst, spec=self.spec)
             elif not os.path.islink(src):
                 copy(src, dst)
                 if 'script' in get_filetype(src):
@@ -988,7 +988,7 @@ def add_files_to_view(self, view, merge_map):
                 orig_link_target = os.path.join(self.spec.prefix, realpath_rel)
 
                 new_link_target = os.path.abspath(merge_map[orig_link_target])
-                view.link(new_link_target, dst)
+                view.link(new_link_target, dst, spec=self.spec)
 
     def remove_files_from_view(self, view, merge_map):
         bin_dir = self.spec.prefix.bin