zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

views: packages can customize how they're added to views (#7152)

Browse Source 
Functional updates:

- `python` now creates a copy of the `python` binaries when it is added
  to a view

- Python extensions (packages which subclass `PythonPackage`) rewrite
  their shebang lines to refer to python in the view

- Python packages in the same namespace will not generate conflicts if
  both have `...lib/site-packages/namespace-example/__init__.py`

  - These `__init__` files will also remain when removing any package in
    the namespace until the last package in the namespace is removed


Generally (Updated 2/16):

- Any package can define `add_files_to_view` to customize how it is added
  to a view (and at the moment custom definitions are included for
  `python` and `PythonPackage`)

  - Likewise any package can define `remove_files_from_view` to customize
    which files are removed (e.g. you don't always want to remove the
    namespace `__init__`)

- Any package can define `view_file_conflicts` to customize what it
  considers a merge conflict

- Global activations are handled like views (where the view root is the
  spec prefix of the extendee)

  - Benefit: filesystem-management aspects of activating extensions are
    now placed in views (e.g. now one can hardlink a global activation)

  - Benefit: overriding `Package.activate` is more straightforward (see
    `Python.activate`)

  - Complication: extension packages which have special-purpose logic
    *only* when activated outside of the extendee prefix must check for
    this in their `add_files_to_view` method (see `PythonPackage`)

- `LinkTree` is refactored to have separate methods for copying a
  directory structure and for copying files (since it was found that
  generally packages may want to alter how files are copied but still
  wanted to copy directories in the same way)


TODOs (updated 2/20):

- [x] additional testing (there is some unit testing added at this point
  but more would be useful)

- [x] refactor or reorganize `LinkTree` methods: currently there is a
  separate set of methods for replicating just the directory structure
  without the files, and a set for replicating everything

- [x] Right now external views (i.e. those not used for global
  activations) call `view.add_extension`, but global activations do not
  to avoid some extra work that goes into maintaining external views. I'm
  not sure if addressing that needs to be done here but I'd like to
  clarify it in the comments (UPDATE: for now I have added a TODO and in
  my opinion this can be merged now and the refactor handled later)

- [x] Several method descriptions (e.g. for `Package.activate`) are out
  of date and reference a distinction between global activations and
  views, they need to be updated

- [x] Update aspell package activations
This commit is contained in:
scheibelp
2018-06-26 16:14:05 -07:00
committed by Todd Gamblin
parent 1276ce0585
commit 3560f6dbe9
26 changed files with 837 additions and 380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
lib/spack/llnl/util/filesystem.py
View File

@@ -79,6 +79,18 @@
]
def path_contains_subdirectory(path, root):
norm_root = os.path.abspath(root).rstrip(os.path.sep) + os.path.sep
norm_path = os.path.abspath(path).rstrip(os.path.sep) + os.path.sep
return norm_path.startswith(norm_root)
def same_path(path1, path2):
norm1 = os.path.abspath(path1).rstrip(os.path.sep)
norm2 = os.path.abspath(path2).rstrip(os.path.sep)
return norm1 == norm2
def filter_file(regex, repl, *filenames, **kwargs):
r"""Like sed, but uses python regular expressions.
@@ -281,6 +293,17 @@ def is_exe(path):
return os.path.isfile(path) and os.access(path, os.X_OK)
def get_filetype(path_name):
"""
Return the output of file path_name as a string to identify file type.
"""
file = Executable('file')
file.add_default_env('LC_ALL', 'C')
output = file('-b', '-h', '%s' % path_name,
output=str, error=str)
return output.strip()
def mkdirp(*paths):
"""Creates a directory, as well as parent directories if needed."""
for path in paths:

141
lib/spack/llnl/util/link_tree.py
View File

@@ -29,6 +29,7 @@
import filecmp
from llnl.util.filesystem import traverse_tree, mkdirp, touch
import llnl.util.tty as tty
__all__ = ['LinkTree']
@@ -44,37 +45,49 @@ class LinkTree(object):
Trees comprise symlinks only to files; directries are never
symlinked to, to prevent the source directory from ever being
modified.
"""
def __init__(self, source_root):
if not os.path.exists(source_root):
raise IOError("No such file or directory: '%s'", source_root)
self._root = source_root
def find_conflict(self, dest_root, **kwargs):
def find_conflict(self, dest_root, ignore=None,
ignore_file_conflicts=False):
"""Returns the first file in dest that conflicts with src"""
kwargs['follow_nonexisting'] = False
ignore = ignore or (lambda x: False)
conflicts = self.find_dir_conflicts(dest_root, ignore)
if not ignore_file_conflicts:
conflicts.extend(
dst for src, dst
in self.get_file_map(dest_root, ignore).items()
if os.path.exists(dst))
if conflicts:
return conflicts[0]
def find_dir_conflicts(self, dest_root, ignore):
conflicts = []
kwargs = {'follow_nonexisting': False, 'ignore': ignore}
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if os.path.isdir(src):
if os.path.exists(dest) and not os.path.isdir(dest):
return dest
elif os.path.exists(dest):
return dest
return None
conflicts.append("File blocks directory: %s" % dest)
elif os.path.exists(dest) and os.path.isdir(dest):
conflicts.append("Directory blocks directory: %s" % dest)
return conflicts
def merge(self, dest_root, link=os.symlink, **kwargs):
"""Link all files in src into dest, creating directories
if necessary.
If ignore_conflicts is True, do not break when the target exists but
rather return a list of files that could not be linked.
Note that files blocking directories will still cause an error.
"""
kwargs['order'] = 'pre'
ignore_conflicts = kwargs.get("ignore_conflicts", False)
existing = []
def get_file_map(self, dest_root, ignore):
merge_map = {}
kwargs = {'follow_nonexisting': True, 'ignore': ignore}
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
if not os.path.isdir(src):
merge_map[src] = dest
return merge_map
def merge_directories(self, dest_root, ignore):
for src, dest in traverse_tree(self._root, dest_root, ignore=ignore):
if os.path.isdir(src):
if not os.path.exists(dest):
mkdirp(dest)
@@ -88,31 +101,13 @@ def merge(self, dest_root, link=os.symlink, **kwargs):
marker = os.path.join(dest, empty_file_name)
touch(marker)
else:
if os.path.exists(dest):
if ignore_conflicts:
existing.append(src)
else:
raise AssertionError("File already exists: %s" % dest)
else:
link(src, dest)
if ignore_conflicts:
return existing
def unmerge(self, dest_root, **kwargs):
"""Unlink all files in dest that exist in src.
Unlinks directories in dest if they are empty.
"""
kwargs['order'] = 'post'
for src, dest in traverse_tree(self._root, dest_root, **kwargs):
def unmerge_directories(self, dest_root, ignore):
for src, dest in traverse_tree(
self._root, dest_root, ignore=ignore, order='post'):
if os.path.isdir(src):
# Skip non-existing links.
if not os.path.exists(dest):
continue
if not os.path.isdir(dest):
elif not os.path.isdir(dest):
raise ValueError("File blocks directory: %s" % dest)
# remove directory if it is empty.
@@ -124,11 +119,61 @@ def unmerge(self, dest_root, **kwargs):
if os.path.exists(marker):
os.remove(marker)
elif os.path.exists(dest):
if not os.path.islink(dest):
raise ValueError("%s is not a link tree!" % dest)
# remove if dest is a hardlink/symlink to src; this will only
# be false if two packages are merged into a prefix and have a
# conflicting file
if filecmp.cmp(src, dest, shallow=True):
os.remove(dest)
def merge(self, dest_root, **kwargs):
"""Link all files in src into dest, creating directories
if necessary.
If ignore_conflicts is True, do not break when the target exists but
rather return a list of files that could not be linked.
Note that files blocking directories will still cause an error.
"""
ignore_conflicts = kwargs.get("ignore_conflicts", False)
ignore = kwargs.get('ignore', lambda x: False)
conflict = self.find_conflict(
dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts)
if conflict:
raise MergeConflictError(conflict)
self.merge_directories(dest_root, ignore)
existing = []
merge_file = kwargs.get('merge_file', merge_link)
for src, dst in self.get_file_map(dest_root, ignore).items():
if os.path.exists(dst):
existing.append(dst)
else:
merge_file(src, dst)
for c in existing:
tty.warn("Could not merge: %s" % c)
def unmerge(self, dest_root, **kwargs):
"""Unlink all files in dest that exist in src.
Unlinks directories in dest if they are empty.
"""
remove_file = kwargs.get('remove_file', remove_link)
ignore = kwargs.get('ignore', lambda x: False)
for src, dst in self.get_file_map(dest_root, ignore).items():
remove_file(src, dst)
self.unmerge_directories(dest_root, ignore)
def merge_link(src, dest):
os.symlink(src, dest)
def remove_link(src, dest):
if not os.path.islink(dest):
raise ValueError("%s is not a link tree!" % dest)
# remove if dest is a hardlink/symlink to src; this will only
# be false if two packages are merged into a prefix and have a
# conflicting file
if filecmp.cmp(src, dest, shallow=True):
os.remove(dest)
class MergeConflictError(Exception):
def __init__(self, path):
super(MergeConflictError, self).__init__(
"Package merge blocked by file: %s" % path)