zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

install_tree: symlink handling and add 'ignore' option (#9019)

Browse Source 
Fixes #9001

#8289 added support for install_tree and copy_tree to merge into an existing
directory structure. However, it did not properly handle relative symlinks and
also removed support for the 'ignore' keyword. Additionally, some of the tests
were overly-strict when checking the permissions on the copied files.

This updates the install_tree/copy_tree methods and their tests:

* copy_tree/install_tree now preserve relative link targets (if the symlink in the
  source directory structure is relative, the symlink created in the destination
  will be relative)
* Added support for 'ignore' argument back to copy_tree/install_tree (removed
  in #8289). It is no longer the object output by shutil.ignore_patterns: you pass a
  function that accepts a path relative to the source and returns whether that
  path should be copied.
* The openfoam packages (currently the only ones making use of the 'ignore'
  argument) are updated for the new API
* When a symlink target is absolute, copy_tree and install_tree now rewrite the
  source prefix to be the destination prefix
* copy_tree tests no longer check permissions: copy_tree doesn't enforce
  anything about permissions so its tests don't check for that
* install_tree tests no longer check for exact permission matching since it can add
  file permissions
This commit is contained in:
scheibelp 2018-08-17 22:08:38 -04:00 committed by GitHub
parent a7a6745120
commit 638cc64571
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 98 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
lib/spack/llnl/util/filesystem.py
View File

@ -306,7 +306,21 @@ def install(src, dest):
copy(src, dest, _permissions=True)
def copy_tree(src, dest, symlinks=True, _permissions=False):
def resolve_link_target_relative_to_the_link(l):
"""
os.path.isdir uses os.path.exists, which for links will check
the existence of the link target. If the link target is relative to
the link, we need to construct a pathname that is valid from
our cwd (which may not be the same as the link's directory)
"""
target = os.readlink(l)
if os.path.isabs(target):
return target
link_dir = os.path.dirname(os.path.abspath(l))
return os.path.join(link_dir, target)
def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
"""Recursively copy an entire directory tree rooted at *src*.
If the destination directory *dest* does not already exist, it will
@ -317,10 +331,14 @@ def copy_tree(src, dest, symlinks=True, _permissions=False):
will be copied as far as the platform allows; if false, the contents and
metadata of the linked files are copied to the new tree.
If *ignore* is set, then each path relative to *src* will be passed to
this function; the function returns whether that path should be skipped.
Parameters:
src (str): the directory to copy
dest (str): the destination directory
symlinks (bool): whether or not to preserve symlinks
ignore (function): function indicating which files to ignore
_permissions (bool): for internal use only
"""
if _permissions:
@ -330,13 +348,31 @@ def copy_tree(src, dest, symlinks=True, _permissions=False):
mkdirp(dest)
for s, d in traverse_tree(src, dest, order='pre', follow_nonexisting=True):
if symlinks and os.path.islink(s):
# Note that this won't rewrite absolute links into the old
# root to point at the new root. Should we handle that case?
src = os.path.abspath(src)
dest = os.path.abspath(dest)
for s, d in traverse_tree(src, dest, order='pre',
follow_symlinks=not symlinks,
ignore=ignore,
follow_nonexisting=True):
if os.path.islink(s):
link_target = resolve_link_target_relative_to_the_link(s)
if symlinks:
target = os.readlink(s)
os.symlink(os.path.abspath(target), d)
elif os.path.isdir(s):
if os.path.isabs(target):
new_target = re.sub(src, dest, target)
if new_target != target:
tty.debug("Redirecting link {0} to {1}"
.format(target, new_target))
target = new_target
os.symlink(target, d)
elif os.path.isdir(link_target):
mkdirp(d)
else:
shutil.copyfile(s, d)
else:
if os.path.isdir(s):
mkdirp(d)
else:
shutil.copyfile(s, d)
@ -346,7 +382,7 @@ def copy_tree(src, dest, symlinks=True, _permissions=False):
copy_mode(s, d)
def install_tree(src, dest, symlinks=True):
def install_tree(src, dest, symlinks=True, ignore=None):
"""Recursively install an entire directory tree rooted at *src*.
Same as :py:func:`copy_tree` with the addition of setting proper
@ -356,8 +392,9 @@ def install_tree(src, dest, symlinks=True):
src (str): the directory to install
dest (str): the destination directory
symlinks (bool): whether or not to preserve symlinks
ignore (function): function indicating which files to ignore
"""
copy_tree(src, dest, symlinks, _permissions=True)
copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)
def is_exe(path):
@ -564,7 +601,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
Keyword Arguments:
order (str): Whether to do pre- or post-order traversal. Accepted
values are 'pre' and 'post'
ignore (str): Predicate indicating which files to ignore
ignore (function): function indicating which files to ignore
follow_nonexisting (bool): Whether to descend into directories in
``src`` that do not exit in ``dest``. Default is True
follow_links (bool): Whether to descend into symlinks in ``src``
@ -578,7 +615,7 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
raise ValueError("Order must be 'pre' or 'post'.")
# List of relative paths to ignore under the src root.
ignore = kwargs.get('ignore', lambda filename: False)
ignore = kwargs.get('ignore', None) or (lambda filename: False)
# Don't descend into ignored directories
if ignore(rel_path):
@ -597,6 +634,9 @@ def traverse_tree(source_root, dest_root, rel_path='', **kwargs):
rel_child = os.path.join(rel_path, f)
# Treat as a directory
# TODO: for symlinks, os.path.isdir looks for the link target. If the
# target is relative to the link, then that may not resolve properly
# relative to our cwd - see resolve_link_target_relative_to_the_link
if os.path.isdir(source_child) and (
follow_links or not os.path.islink(source_child)):

37
lib/spack/spack/test/llnl/util/filesystem.py
View File

@ -26,6 +26,7 @@
import llnl.util.filesystem as fs
import os
import stat
import pytest
@ -45,8 +46,10 @@ def stage(tmpdir_factory):
fs.touchp('source/c/d/6')
fs.touchp('source/c/d/e/7')
# Create symlink
# Create symlinks
os.symlink(os.path.abspath('source/1'), 'source/2')
os.symlink('b/2', 'source/a/b2')
os.symlink('a/b', 'source/f')
# Create destination directory
fs.mkdirp('dest')
@ -64,7 +67,6 @@ def test_file_dest(self, stage):
fs.copy('source/1', 'dest/1')
assert os.path.exists('dest/1')
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
def test_dir_dest(self, stage):
"""Test using a directory as the destination."""
@ -73,7 +75,14 @@ def test_dir_dest(self, stage):
fs.copy('source/1', 'dest')
assert os.path.exists('dest/1')
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
def check_added_exe_permissions(src, dst):
src_mode = os.stat(src).st_mode
dst_mode = os.stat(dst).st_mode
for perm in [stat.S_IXUSR, stat.S_IXGRP, stat.S_IXOTH]:
if src_mode & perm:
assert dst_mode & perm
class TestInstall:
@ -86,7 +95,7 @@ def test_file_dest(self, stage):
fs.install('source/1', 'dest/1')
assert os.path.exists('dest/1')
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
check_added_exe_permissions('source/1', 'dest/1')
def test_dir_dest(self, stage):
"""Test using a directory as the destination."""
@ -95,7 +104,7 @@ def test_dir_dest(self, stage):
fs.install('source/1', 'dest')
assert os.path.exists('dest/1')
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
check_added_exe_permissions('source/1', 'dest/1')
class TestCopyTree:
@ -126,6 +135,24 @@ def test_symlinks_true(self, stage):
assert os.path.exists('dest/2')
assert os.path.islink('dest/2')
assert os.path.exists('dest/a/b2')
with fs.working_dir('dest/a'):
assert os.path.exists(os.readlink('b2'))
assert (os.path.realpath('dest/f/2') ==
os.path.abspath('dest/a/b/2'))
assert os.path.realpath('dest/2') == os.path.abspath('dest/1')
def test_symlinks_true_ignore(self, stage):
"""Test copying when specifying relative paths that should be ignored
"""
with fs.working_dir(str(stage)):
ignore = lambda p: p in ['c/d/e', 'a']
fs.copy_tree('source', 'dest', symlinks=True, ignore=ignore)
assert not os.path.exists('dest/a')
assert os.path.exists('dest/c/d')
assert not os.path.exists('dest/c/d/e')
def test_symlinks_false(self, stage):
"""Test copying without symlink preservation."""

11
var/spack/repos/builtin/packages/foam-extend/package.py
View File

@ -54,7 +54,6 @@
##############################################################################
import glob
import re
import shutil
import os
from spack import *
@ -382,7 +381,6 @@ def build(self, spec, prefix):
def install(self, spec, prefix):
"""Install under the projectdir"""
opts = str(self.foam_arch)
# Fairly ugly since intermediate targets are scattered inside sources
appdir = 'applications'
@ -419,19 +417,22 @@ def install(self, spec, prefix):
subitem = join_path(appdir, 'Allwmake')
install(subitem, join_path(self.projectdir, subitem))
ignored = [opts] # Ignore intermediate targets
foam_arch_str = str(self.foam_arch)
# Ignore intermediate targets
ignore = lambda p: os.path.basename(p) == foam_arch_str
for d in ['src', 'tutorials']:
install_tree(
d,
join_path(self.projectdir, d),
ignore=shutil.ignore_patterns(*ignored),
ignore=ignore,
symlinks=True)
for d in ['solvers', 'utilities']:
install_tree(
join_path(appdir, d),
join_path(self.projectdir, appdir, d),
ignore=shutil.ignore_patterns(*ignored),
ignore=ignore,
symlinks=True)
etc_dir = join_path(self.projectdir, 'etc')

6
var/spack/repos/builtin/packages/openfoam-com/package.py
View File

@ -60,7 +60,6 @@
##############################################################################
import glob
import re
import shutil
import os
from spack import *
@ -692,12 +691,13 @@ def install(self, spec, prefix):
dirs.extend(['doc'])
# Install platforms (and doc) skipping intermediate targets
ignored = ['src', 'applications', 'html', 'Guides']
relative_ignore_paths = ['src', 'applications', 'html', 'Guides']
ignore = lambda p: p in relative_ignore_paths
for d in dirs:
install_tree(
d,
join_path(self.projectdir, d),
ignore=shutil.ignore_patterns(*ignored),
ignore=ignore,
symlinks=True)
etc_dir = join_path(self.projectdir, 'etc')

6
var/spack/repos/builtin/packages/openfoam-org/package.py
View File

@ -55,7 +55,6 @@
##############################################################################
import glob
import re
import shutil
import os
import llnl.util.tty as tty
@ -345,12 +344,13 @@ def install(self, spec, prefix):
dirs.extend(['doc'])
# Install platforms (and doc) skipping intermediate targets
ignored = ['src', 'applications', 'html', 'Guides']
relative_ignore_paths = ['src', 'applications', 'html', 'Guides']
ignore = lambda p: p in relative_ignore_paths
for d in dirs:
install_tree(
d,
join_path(self.projectdir, d),
ignore=shutil.ignore_patterns(*ignored),
ignore=ignore,
symlinks=True)
etc_dir = join_path(self.projectdir, 'etc')