zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

install/install_tree: glob support (#18376)

Browse Source 
* install/install_tree: glob support

* Add unit tests

* Update existing packages

* Raise error if glob finds no files, document function raises
This commit is contained in:
Adam J. Stewart
2020-09-03 12:47:19 -05:00
committed by GitHub
parent 098beee295
commit 741bb9bafe
48 changed files with 276 additions and 370 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
lib/spack/llnl/util/filesystem.py
View File

@@ -337,41 +337,63 @@ def unset_executable_mode(path):
def copy(src, dest, _permissions=False):
"""Copies the file *src* to the file or directory *dest*.
"""Copy the file(s) *src* to the file or directory *dest*.
If *dest* specifies a directory, the file will be copied into *dest*
using the base filename from *src*.
*src* may contain glob characters.
Parameters:
src (str): the file to copy
src (str): the file(s) to copy
dest (str): the destination file or directory
_permissions (bool): for internal use only
Raises:
IOError: if *src* does not match any files or directories
ValueError: if *src* matches multiple files but *dest* is
not a directory
"""
if _permissions:
tty.debug('Installing {0} to {1}'.format(src, dest))
else:
tty.debug('Copying {0} to {1}'.format(src, dest))
# Expand dest to its eventual full path if it is a directory.
if os.path.isdir(dest):
dest = join_path(dest, os.path.basename(src))
files = glob.glob(src)
if not files:
raise IOError("No such file or directory: '{0}'".format(src))
if len(files) > 1 and not os.path.isdir(dest):
raise ValueError(
"'{0}' matches multiple files but '{1}' is not a directory".format(
src, dest))
shutil.copy(src, dest)
for src in files:
# Expand dest to its eventual full path if it is a directory.
dst = dest
if os.path.isdir(dest):
dst = join_path(dest, os.path.basename(src))
if _permissions:
set_install_permissions(dest)
copy_mode(src, dest)
shutil.copy(src, dst)
if _permissions:
set_install_permissions(dst)
copy_mode(src, dst)
def install(src, dest):
"""Installs the file *src* to the file or directory *dest*.
"""Install the file(s) *src* to the file or directory *dest*.
Same as :py:func:`copy` with the addition of setting proper
permissions on the installed file.
Parameters:
src (str): the file to install
src (str): the file(s) to install
dest (str): the destination file or directory
Raises:
IOError: if *src* does not match any files or directories
ValueError: if *src* matches multiple files but *dest* is
not a directory
"""
copy(src, dest, _permissions=True)
@@ -396,6 +418,8 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
If the destination directory *dest* does not already exist, it will
be created as well as missing parent directories.
*src* may contain glob characters.
If *symlinks* is true, symbolic links in the source tree are represented
as symbolic links in the new tree and the metadata of the original links
will be copied as far as the platform allows; if false, the contents and
@@ -410,56 +434,66 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
symlinks (bool): whether or not to preserve symlinks
ignore (function): function indicating which files to ignore
_permissions (bool): for internal use only
Raises:
IOError: if *src* does not match any files or directories
ValueError: if *src* is a parent directory of *dest*
"""
if _permissions:
tty.debug('Installing {0} to {1}'.format(src, dest))
else:
tty.debug('Copying {0} to {1}'.format(src, dest))
abs_src = os.path.abspath(src)
if not abs_src.endswith(os.path.sep):
abs_src += os.path.sep
abs_dest = os.path.abspath(dest)
if not abs_dest.endswith(os.path.sep):
abs_dest += os.path.sep
# Stop early to avoid unnecessary recursion if being asked to copy from a
# parent directory.
if abs_dest.startswith(abs_src):
raise ValueError('Cannot copy ancestor directory {0} into {1}'.
format(abs_src, abs_dest))
files = glob.glob(src)
if not files:
raise IOError("No such file or directory: '{0}'".format(src))
mkdirp(dest)
for src in files:
abs_src = os.path.abspath(src)
if not abs_src.endswith(os.path.sep):
abs_src += os.path.sep
for s, d in traverse_tree(abs_src, abs_dest, order='pre',
follow_symlinks=not symlinks,
ignore=ignore,
follow_nonexisting=True):
if os.path.islink(s):
link_target = resolve_link_target_relative_to_the_link(s)
if symlinks:
target = os.readlink(s)
if os.path.isabs(target):
new_target = re.sub(abs_src, abs_dest, target)
if new_target != target:
tty.debug("Redirecting link {0} to {1}"
.format(target, new_target))
target = new_target
# Stop early to avoid unnecessary recursion if being asked to copy
# from a parent directory.
if abs_dest.startswith(abs_src):
raise ValueError('Cannot copy ancestor directory {0} into {1}'.
format(abs_src, abs_dest))
os.symlink(target, d)
elif os.path.isdir(link_target):
mkdirp(d)
mkdirp(abs_dest)
for s, d in traverse_tree(abs_src, abs_dest, order='pre',
follow_symlinks=not symlinks,
ignore=ignore,
follow_nonexisting=True):
if os.path.islink(s):
link_target = resolve_link_target_relative_to_the_link(s)
if symlinks:
target = os.readlink(s)
if os.path.isabs(target):
new_target = re.sub(abs_src, abs_dest, target)
if new_target != target:
tty.debug("Redirecting link {0} to {1}"
.format(target, new_target))
target = new_target
os.symlink(target, d)
elif os.path.isdir(link_target):
mkdirp(d)
else:
shutil.copyfile(s, d)
else:
shutil.copyfile(s, d)
else:
if os.path.isdir(s):
mkdirp(d)
else:
shutil.copy2(s, d)
if os.path.isdir(s):
mkdirp(d)
else:
shutil.copy2(s, d)
if _permissions:
set_install_permissions(d)
copy_mode(s, d)
if _permissions:
set_install_permissions(d)
copy_mode(s, d)
def install_tree(src, dest, symlinks=True, ignore=None):
@@ -473,6 +507,10 @@ def install_tree(src, dest, symlinks=True, ignore=None):
dest (str): the destination directory
symlinks (bool): whether or not to preserve symlinks
ignore (function): function indicating which files to ignore
Raises:
IOError: if *src* does not match any files or directories
ValueError: if *src* is a parent directory of *dest*
"""
copy_tree(src, dest, symlinks=symlinks, ignore=ignore, _permissions=True)

128
lib/spack/spack/test/llnl/util/filesystem.py
View File

@@ -30,6 +30,9 @@ def stage(tmpdir_factory):
fs.touchp('source/c/d/5')
fs.touchp('source/c/d/6')
fs.touchp('source/c/d/e/7')
fs.touchp('source/g/h/i/8')
fs.touchp('source/g/h/i/9')
fs.touchp('source/g/i/j/10')
# Create symlinks
os.symlink(os.path.abspath('source/1'), 'source/2')
@@ -61,6 +64,31 @@ def test_dir_dest(self, stage):
assert os.path.exists('dest/1')
def test_glob_src(self, stage):
"""Test using a glob as the source."""
with fs.working_dir(str(stage)):
fs.copy('source/a/*/*', 'dest')
assert os.path.exists('dest/2')
assert os.path.exists('dest/3')
def test_non_existing_src(self, stage):
"""Test using a non-existing source."""
with fs.working_dir(str(stage)):
with pytest.raises(IOError, match='No such file or directory'):
fs.copy('source/none', 'dest')
def test_multiple_src_file_dest(self, stage):
"""Test a glob that matches multiple source files and a dest
that is not a directory."""
with fs.working_dir(str(stage)):
match = '.* matches multiple files but .* is not a directory'
with pytest.raises(ValueError, match=match):
fs.copy('source/a/*/*', 'dest/1')
def check_added_exe_permissions(src, dst):
src_mode = os.stat(src).st_mode
@@ -91,6 +119,33 @@ def test_dir_dest(self, stage):
assert os.path.exists('dest/1')
check_added_exe_permissions('source/1', 'dest/1')
def test_glob_src(self, stage):
"""Test using a glob as the source."""
with fs.working_dir(str(stage)):
fs.install('source/a/*/*', 'dest')
assert os.path.exists('dest/2')
assert os.path.exists('dest/3')
check_added_exe_permissions('source/a/b/2', 'dest/2')
check_added_exe_permissions('source/a/b/3', 'dest/3')
def test_non_existing_src(self, stage):
"""Test using a non-existing source."""
with fs.working_dir(str(stage)):
with pytest.raises(IOError, match='No such file or directory'):
fs.install('source/none', 'dest')
def test_multiple_src_file_dest(self, stage):
"""Test a glob that matches multiple source files and a dest
that is not a directory."""
with fs.working_dir(str(stage)):
match = '.* matches multiple files but .* is not a directory'
with pytest.raises(ValueError, match=match):
fs.install('source/a/*/*', 'dest/1')
class TestCopyTree:
"""Tests for ``filesystem.copy_tree``"""
@@ -111,21 +166,6 @@ def test_non_existing_dir(self, stage):
assert os.path.exists('dest/sub/directory/a/b/2')
def test_parent_dir(self, stage):
"""Test copying to from a parent directory."""
# Make sure we get the right error if we try to copy a parent into
# a descendent directory.
with pytest.raises(ValueError, match="Cannot copy"):
with fs.working_dir(str(stage)):
fs.copy_tree('source', 'source/sub/directory')
# Only point with this check is to make sure we don't try to perform
# the copy.
with pytest.raises(IOError, match="No such file or directory"):
with fs.working_dir(str(stage)):
fs.copy_tree('foo/ba', 'foo/bar')
def test_symlinks_true(self, stage):
"""Test copying with symlink preservation."""
@@ -162,6 +202,31 @@ def test_symlinks_false(self, stage):
assert os.path.exists('dest/2')
assert not os.path.islink('dest/2')
def test_glob_src(self, stage):
"""Test using a glob as the source."""
with fs.working_dir(str(stage)):
fs.copy_tree('source/g/*', 'dest')
assert os.path.exists('dest/i/8')
assert os.path.exists('dest/i/9')
assert os.path.exists('dest/j/10')
def test_non_existing_src(self, stage):
"""Test using a non-existing source."""
with fs.working_dir(str(stage)):
with pytest.raises(IOError, match='No such file or directory'):
fs.copy_tree('source/none', 'dest')
def test_parent_dir(self, stage):
"""Test source as a parent directory of destination."""
with fs.working_dir(str(stage)):
match = 'Cannot copy ancestor directory'
with pytest.raises(ValueError, match=match):
fs.copy_tree('source', 'source/sub/directory')
class TestInstallTree:
"""Tests for ``filesystem.install_tree``"""
@@ -173,6 +238,7 @@ def test_existing_dir(self, stage):
fs.install_tree('source', 'dest')
assert os.path.exists('dest/a/b/2')
check_added_exe_permissions('source/a/b/2', 'dest/a/b/2')
def test_non_existing_dir(self, stage):
"""Test installing to a non-existing directory."""
@@ -181,6 +247,8 @@ def test_non_existing_dir(self, stage):
fs.install_tree('source', 'dest/sub/directory')
assert os.path.exists('dest/sub/directory/a/b/2')
check_added_exe_permissions(
'source/a/b/2', 'dest/sub/directory/a/b/2')
def test_symlinks_true(self, stage):
"""Test installing with symlink preservation."""
@@ -190,6 +258,7 @@ def test_symlinks_true(self, stage):
assert os.path.exists('dest/2')
assert os.path.islink('dest/2')
check_added_exe_permissions('source/2', 'dest/2')
def test_symlinks_false(self, stage):
"""Test installing without symlink preservation."""
@@ -199,6 +268,35 @@ def test_symlinks_false(self, stage):
assert os.path.exists('dest/2')
assert not os.path.islink('dest/2')
check_added_exe_permissions('source/2', 'dest/2')
def test_glob_src(self, stage):
"""Test using a glob as the source."""
with fs.working_dir(str(stage)):
fs.install_tree('source/g/*', 'dest')
assert os.path.exists('dest/i/8')
assert os.path.exists('dest/i/9')
assert os.path.exists('dest/j/10')
check_added_exe_permissions('source/g/h/i/8', 'dest/i/8')
check_added_exe_permissions('source/g/h/i/9', 'dest/i/9')
check_added_exe_permissions('source/g/i/j/10', 'dest/j/10')
def test_non_existing_src(self, stage):
"""Test using a non-existing source."""
with fs.working_dir(str(stage)):
with pytest.raises(IOError, match='No such file or directory'):
fs.install_tree('source/none', 'dest')
def test_parent_dir(self, stage):
"""Test source as a parent directory of destination."""
with fs.working_dir(str(stage)):
match = 'Cannot copy ancestor directory'
with pytest.raises(ValueError, match=match):
fs.install_tree('source', 'source/sub/directory')
def test_paths_containing_libs(dirs_with_libfiles):