Feature: add option to create view by copying/relocating files (#16480)
* add subcommand `spack view copy/relocate` * update bash completions * add copy/relocate commands to view tests * allow copied views to be removed
This commit is contained in:
Greg Becker
GitHub
parent
7aa9cb0f7a
commit
3347ef2de4
@ -33,8 +33,6 @@
|
||||
YamlFilesystemView.
|
||||
|
||||
'''
|
||||
import os
|
||||
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.link_tree import MergeConflictError
|
||||
from llnl.util.tty.color import colorize
|
||||
@ -45,13 +43,15 @@
|
||||
import spack.schema.projections
|
||||
from spack.config import validate
|
||||
from spack.filesystem_view import YamlFilesystemView
|
||||
from spack.filesystem_view import view_symlink, view_hardlink, view_copy
|
||||
from spack.util import spack_yaml as s_yaml
|
||||
|
||||
description = "project packages to a compact naming scheme on the filesystem."
|
||||
section = "environments"
|
||||
level = "short"
|
||||
|
||||
actions_link = ["symlink", "add", "soft", "hardlink", "hard"]
|
||||
actions_link = ["symlink", "add", "soft", "hardlink", "hard", "copy",
|
||||
"relocate"]
|
||||
actions_remove = ["remove", "rm"]
|
||||
actions_status = ["statlink", "status", "check"]
|
||||
|
||||
@ -112,6 +112,9 @@ def setup_parser(sp):
|
||||
"hardlink": ssp.add_parser(
|
||||
'hardlink', aliases=['hard'],
|
||||
help='add packages files to a filesystem view via hard links'),
|
||||
"copy": ssp.add_parser(
|
||||
'copy', aliases=['relocate'],
|
||||
help='add package files to a filesystem view via copy/relocate'),
|
||||
"remove": ssp.add_parser(
|
||||
'remove', aliases=['rm'],
|
||||
help='remove packages from a filesystem view'),
|
||||
@ -125,7 +128,7 @@ def setup_parser(sp):
|
||||
act.add_argument('path', nargs=1,
|
||||
help="path to file system view directory")
|
||||
|
||||
if cmd in ("symlink", "hardlink"):
|
||||
if cmd in ("symlink", "hardlink", "copy"):
|
||||
# invalid for remove/statlink, for those commands the view needs to
|
||||
# already know its own projections.
|
||||
help_msg = "Initialize view using projections from file."
|
||||
@ -157,7 +160,7 @@ def setup_parser(sp):
|
||||
so["nargs"] = "+"
|
||||
act.add_argument('specs', **so)
|
||||
|
||||
for cmd in ["symlink", "hardlink"]:
|
||||
for cmd in ["symlink", "hardlink", "copy"]:
|
||||
act = file_system_view_actions[cmd]
|
||||
act.add_argument("-i", "--ignore-conflicts", action='store_true')
|
||||
|
||||
@ -179,11 +182,19 @@ def view(parser, args):
|
||||
else:
|
||||
ordered_projections = {}
|
||||
|
||||
# What method are we using for this view
|
||||
if args.action in ("hardlink", "hard"):
|
||||
link_fn = view_hardlink
|
||||
elif args.action in ("copy", "relocate"):
|
||||
link_fn = view_copy
|
||||
else:
|
||||
link_fn = view_symlink
|
||||
|
||||
view = YamlFilesystemView(
|
||||
path, spack.store.layout,
|
||||
projections=ordered_projections,
|
||||
ignore_conflicts=getattr(args, "ignore_conflicts", False),
|
||||
link=os.link if args.action in ["hardlink", "hard"] else os.symlink,
|
||||
link=link_fn,
|
||||
verbose=args.verbose)
|
||||
|
||||
# Process common args and specs
|
||||
|
||||
@ -24,6 +24,7 @@
|
||||
import spack.schema.projections
|
||||
import spack.projections
|
||||
import spack.config
|
||||
import spack.relocate
|
||||
from spack.error import SpackError
|
||||
from spack.directory_layout import ExtensionAlreadyInstalledError
|
||||
from spack.directory_layout import YamlViewExtensionsLayout
|
||||
@ -41,6 +42,58 @@
|
||||
_projections_path = '.spack/projections.yaml'
|
||||
|
||||
|
||||
def view_symlink(src, dst, **kwargs):
|
||||
# keyword arguments are irrelevant
|
||||
# here to fit required call signature
|
||||
os.symlink(src, dst)
|
||||
|
||||
|
||||
def view_hardlink(src, dst, **kwargs):
|
||||
# keyword arguments are irrelevant
|
||||
# here to fit required call signature
|
||||
os.link(src, dst)
|
||||
|
||||
|
||||
def view_copy(src, dst, view, spec=None):
|
||||
"""
|
||||
Copy a file from src to dst.
|
||||
|
||||
Use spec and view to generate relocations
|
||||
"""
|
||||
shutil.copyfile(src, dst)
|
||||
if spec:
|
||||
# Not metadata, we have to relocate it
|
||||
|
||||
# Get information on where to relocate from/to
|
||||
prefix_to_projection = dict(
|
||||
(dep.prefix, view.get_projection_for_spec(dep))
|
||||
for dep in spec.traverse()
|
||||
)
|
||||
|
||||
if spack.relocate.is_binary(dst):
|
||||
# relocate binaries
|
||||
spack.relocate.relocate_text_bin(
|
||||
binaries=[dst],
|
||||
orig_install_prefix=spec.prefix,
|
||||
new_install_prefix=view.get_projection_for_spec(spec),
|
||||
orig_spack=spack.paths.spack_root,
|
||||
new_spack=view._root,
|
||||
new_prefixes=prefix_to_projection
|
||||
)
|
||||
else:
|
||||
# relocate text
|
||||
spack.relocate.relocate_text(
|
||||
files=[dst],
|
||||
orig_layout_root=spack.store.layout.root,
|
||||
new_layout_root=view._root,
|
||||
orig_install_prefix=spec.prefix,
|
||||
new_install_prefix=view.get_projection_for_spec(spec),
|
||||
orig_spack=spack.paths.spack_root,
|
||||
new_spack=view._root,
|
||||
new_prefixes=prefix_to_projection
|
||||
)
|
||||
|
||||
|
||||
class FilesystemView(object):
|
||||
"""
|
||||
Governs a filesystem view that is located at certain root-directory.
|
||||
@ -67,9 +120,12 @@ def __init__(self, root, layout, **kwargs):
|
||||
self.projections = kwargs.get('projections', {})
|
||||
|
||||
self.ignore_conflicts = kwargs.get("ignore_conflicts", False)
|
||||
self.link = kwargs.get("link", os.symlink)
|
||||
self.verbose = kwargs.get("verbose", False)
|
||||
|
||||
# Setup link function to include view
|
||||
link_func = kwargs.get("link", view_symlink)
|
||||
self.link = ft.partial(link_func, view=self)
|
||||
|
||||
def add_specs(self, *specs, **kwargs):
|
||||
"""
|
||||
Add given specs to view.
|
||||
@ -355,8 +411,6 @@ def remove_file(self, src, dest):
|
||||
if not os.path.lexists(dest):
|
||||
tty.warn("Tried to remove %s which does not exist" % dest)
|
||||
return
|
||||
if not os.path.islink(dest):
|
||||
raise ValueError("%s is not a link tree!" % dest)
|
||||
# remove if dest is a hardlink/symlink to src; this will only
|
||||
# be false if two packages are merged into a prefix and have a
|
||||
# conflicting file
|
||||
|
||||
@ -332,7 +332,7 @@ def add_files_to_view(self, view, merge_map):
|
||||
"""
|
||||
for src, dst in merge_map.items():
|
||||
if not os.path.exists(dst):
|
||||
view.link(src, dst)
|
||||
view.link(src, dst, spec=self.spec)
|
||||
|
||||
def remove_files_from_view(self, view, merge_map):
|
||||
"""Given a map of package files to files currently linked in the view,
|
||||
|
||||
@ -24,7 +24,8 @@ def create_projection_file(tmpdir, projection):
|
||||
return projection_file
|
||||
|
||||
|
||||
@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
|
||||
@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add',
|
||||
'copy', 'relocate'])
|
||||
def test_view_link_type(
|
||||
tmpdir, mock_packages, mock_archive, mock_fetch, config,
|
||||
install_mockery, cmd):
|
||||
@ -33,10 +34,14 @@ def test_view_link_type(
|
||||
view(cmd, viewpath, 'libdwarf')
|
||||
package_prefix = os.path.join(viewpath, 'libdwarf')
|
||||
assert os.path.exists(package_prefix)
|
||||
assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
|
||||
|
||||
# Check that we use symlinks for and only for the appropriate subcommands
|
||||
is_link_cmd = cmd in ('symlink', 'add')
|
||||
assert os.path.islink(package_prefix) == is_link_cmd
|
||||
|
||||
|
||||
@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add'])
|
||||
@pytest.mark.parametrize('cmd', ['hardlink', 'symlink', 'hard', 'add',
|
||||
'copy', 'relocate'])
|
||||
def test_view_projections(
|
||||
tmpdir, mock_packages, mock_archive, mock_fetch, config,
|
||||
install_mockery, cmd):
|
||||
@ -54,7 +59,10 @@ def test_view_projections(
|
||||
|
||||
package_prefix = os.path.join(viewpath, 'libdwarf-20130207/libdwarf')
|
||||
assert os.path.exists(package_prefix)
|
||||
assert os.path.islink(package_prefix) == (not cmd.startswith('hard'))
|
||||
|
||||
# Check that we use symlinks for and only for the appropriate subcommands
|
||||
is_symlink_cmd = cmd in ('symlink', 'add')
|
||||
assert os.path.islink(package_prefix) == is_symlink_cmd
|
||||
|
||||
|
||||
def test_view_multiple_projections(
|
||||
|
||||
@ -1521,7 +1521,7 @@ _spack_view() {
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help -v --verbose -e --exclude -d --dependencies"
|
||||
else
|
||||
SPACK_COMPREPLY="symlink add soft hardlink hard remove rm statlink status check"
|
||||
SPACK_COMPREPLY="symlink add soft hardlink hard copy relocate remove rm statlink status check"
|
||||
fi
|
||||
}
|
||||
|
||||
@ -1570,6 +1570,24 @@ _spack_view_hard() {
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_view_copy() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help --projection-file -i --ignore-conflicts"
|
||||
else
|
||||
_all_packages
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_view_relocate() {
|
||||
if $list_options
|
||||
then
|
||||
SPACK_COMPREPLY="-h --help --projection-file -i --ignore-conflicts"
|
||||
else
|
||||
_all_packages
|
||||
fi
|
||||
}
|
||||
|
||||
_spack_view_remove() {
|
||||
if $list_options
|
||||
then
|
||||
|
||||
@ -962,7 +962,7 @@ def add_files_to_view(self, view, merge_map):
|
||||
bin_dir = self.spec.prefix.bin
|
||||
for src, dst in merge_map.items():
|
||||
if not path_contains_subdirectory(src, bin_dir):
|
||||
view.link(src, dst)
|
||||
view.link(src, dst, spec=self.spec)
|
||||
elif not os.path.islink(src):
|
||||
copy(src, dst)
|
||||
if 'script' in get_filetype(src):
|
||||
@ -988,7 +988,7 @@ def add_files_to_view(self, view, merge_map):
|
||||
orig_link_target = os.path.join(self.spec.prefix, realpath_rel)
|
||||
|
||||
new_link_target = os.path.abspath(merge_map[orig_link_target])
|
||||
view.link(new_link_target, dst)
|
||||
view.link(new_link_target, dst, spec=self.spec)
|
||||
|
||||
def remove_files_from_view(self, view, merge_map):
|
||||
bin_dir = self.spec.prefix.bin
|
||||
|
||||
Loading…
Reference in New Issue
Block a user