install_tree, copy_tree can install into existing directory structures (#8289)
Replace use of `shutil.copytree` with `copy_tree` and `install_tree` functions in `llnl.util.filesystem`. - `copy_tree` copies without setting permissions. It should be used to copy files around in the build directory. - `install_tree` copies files and sets permissions. It should be used to copy files into the installation directory. - `install` and `copy` are analogous single-file functions. - add more extensive tests for these functions - update packages to use these functions.
This commit is contained in:
Adam J. Stewart
committed by
Todd Gamblin
Todd Gamblin
parent
c0699539d5
commit
73c978ddd9
@@ -60,7 +60,9 @@
|
||||
'fix_darwin_install_name',
|
||||
'force_remove',
|
||||
'force_symlink',
|
||||
'copy',
|
||||
'install',
|
||||
'copy_tree',
|
||||
'install_tree',
|
||||
'is_exe',
|
||||
'join_path',
|
||||
@@ -264,27 +266,98 @@ def unset_executable_mode(path):
|
||||
os.chmod(path, mode)
|
||||
|
||||
|
||||
def install(src, dest):
|
||||
"""Manually install a file to a particular location."""
|
||||
tty.debug("Installing %s to %s" % (src, dest))
|
||||
def copy(src, dest, _permissions=False):
|
||||
"""Copies the file *src* to the file or directory *dest*.
|
||||
|
||||
If *dest* specifies a directory, the file will be copied into *dest*
|
||||
using the base filename from *src*.
|
||||
|
||||
Parameters:
|
||||
src (str): the file to copy
|
||||
dest (str): the destination file or directory
|
||||
_permissions (bool): for internal use only
|
||||
"""
|
||||
if _permissions:
|
||||
tty.debug('Installing {0} to {1}'.format(src, dest))
|
||||
else:
|
||||
tty.debug('Copying {0} to {1}'.format(src, dest))
|
||||
|
||||
# Expand dest to its eventual full path if it is a directory.
|
||||
if os.path.isdir(dest):
|
||||
dest = join_path(dest, os.path.basename(src))
|
||||
|
||||
shutil.copy(src, dest)
|
||||
set_install_permissions(dest)
|
||||
copy_mode(src, dest)
|
||||
|
||||
if _permissions:
|
||||
set_install_permissions(dest)
|
||||
copy_mode(src, dest)
|
||||
|
||||
|
||||
def install_tree(src, dest, **kwargs):
|
||||
"""Manually install a directory tree to a particular location."""
|
||||
tty.debug("Installing %s to %s" % (src, dest))
|
||||
shutil.copytree(src, dest, **kwargs)
|
||||
def install(src, dest):
|
||||
"""Installs the file *src* to the file or directory *dest*.
|
||||
|
||||
for s, d in traverse_tree(src, dest, follow_nonexisting=False):
|
||||
set_install_permissions(d)
|
||||
copy_mode(s, d)
|
||||
Same as :py:func:`copy` with the addition of setting proper
|
||||
permissions on the installed file.
|
||||
|
||||
Parameters:
|
||||
src (str): the file to install
|
||||
dest (str): the destination file or directory
|
||||
"""
|
||||
copy(src, dest, _permissions=True)
|
||||
|
||||
|
||||
def copy_tree(src, dest, symlinks=True, _permissions=False):
|
||||
"""Recursively copy an entire directory tree rooted at *src*.
|
||||
|
||||
If the destination directory *dest* does not already exist, it will
|
||||
be created as well as missing parent directories.
|
||||
|
||||
If *symlinks* is true, symbolic links in the source tree are represented
|
||||
as symbolic links in the new tree and the metadata of the original links
|
||||
will be copied as far as the platform allows; if false, the contents and
|
||||
metadata of the linked files are copied to the new tree.
|
||||
|
||||
Parameters:
|
||||
src (str): the directory to copy
|
||||
dest (str): the destination directory
|
||||
symlinks (bool): whether or not to preserve symlinks
|
||||
_permissions (bool): for internal use only
|
||||
"""
|
||||
if _permissions:
|
||||
tty.debug('Installing {0} to {1}'.format(src, dest))
|
||||
else:
|
||||
tty.debug('Copying {0} to {1}'.format(src, dest))
|
||||
|
||||
mkdirp(dest)
|
||||
|
||||
for s, d in traverse_tree(src, dest, order='pre', follow_nonexisting=True):
|
||||
if symlinks and os.path.islink(s):
|
||||
# Note that this won't rewrite absolute links into the old
|
||||
# root to point at the new root. Should we handle that case?
|
||||
target = os.readlink(s)
|
||||
os.symlink(os.path.abspath(target), d)
|
||||
elif os.path.isdir(s):
|
||||
mkdirp(d)
|
||||
else:
|
||||
shutil.copyfile(s, d)
|
||||
|
||||
if _permissions:
|
||||
set_install_permissions(d)
|
||||
copy_mode(s, d)
|
||||
|
||||
|
||||
def install_tree(src, dest, symlinks=True):
|
||||
"""Recursively install an entire directory tree rooted at *src*.
|
||||
|
||||
Same as :py:func:`copy_tree` with the addition of setting proper
|
||||
permissions on the installed files and directories.
|
||||
|
||||
Parameters:
|
||||
src (str): the directory to install
|
||||
dest (str): the destination directory
|
||||
symlinks (bool): whether or not to preserve symlinks
|
||||
"""
|
||||
copy_tree(src, dest, symlinks, _permissions=True)
|
||||
|
||||
|
||||
def is_exe(path):
|
||||
|
||||
209
lib/spack/spack/test/llnl/util/filesystem.py
Normal file
209
lib/spack/spack/test/llnl/util/filesystem.py
Normal file
@@ -0,0 +1,209 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://github.com/spack/spack
|
||||
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License (as
|
||||
# published by the Free Software Foundation) version 2.1, February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
"""Tests for ``llnl/util/filesystem.py``"""
|
||||
|
||||
import llnl.util.filesystem as fs
|
||||
import os
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def stage(tmpdir_factory):
|
||||
"""Creates a stage with the directory structure for the tests."""
|
||||
|
||||
s = tmpdir_factory.mktemp('filesystem_test')
|
||||
|
||||
with s.as_cwd():
|
||||
# Create source file hierarchy
|
||||
fs.touchp('source/1')
|
||||
fs.touchp('source/a/b/2')
|
||||
fs.touchp('source/a/b/3')
|
||||
fs.touchp('source/c/4')
|
||||
fs.touchp('source/c/d/5')
|
||||
fs.touchp('source/c/d/6')
|
||||
fs.touchp('source/c/d/e/7')
|
||||
|
||||
# Create symlink
|
||||
os.symlink(os.path.abspath('source/1'), 'source/2')
|
||||
|
||||
# Create destination directory
|
||||
fs.mkdirp('dest')
|
||||
|
||||
yield s
|
||||
|
||||
|
||||
class TestCopy:
|
||||
"""Tests for ``filesystem.copy``"""
|
||||
|
||||
def test_file_dest(self, stage):
|
||||
"""Test using a filename as the destination."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy('source/1', 'dest/1')
|
||||
|
||||
assert os.path.exists('dest/1')
|
||||
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
|
||||
|
||||
def test_dir_dest(self, stage):
|
||||
"""Test using a directory as the destination."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy('source/1', 'dest')
|
||||
|
||||
assert os.path.exists('dest/1')
|
||||
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
|
||||
|
||||
|
||||
class TestInstall:
|
||||
"""Tests for ``filesystem.install``"""
|
||||
|
||||
def test_file_dest(self, stage):
|
||||
"""Test using a filename as the destination."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install('source/1', 'dest/1')
|
||||
|
||||
assert os.path.exists('dest/1')
|
||||
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
|
||||
|
||||
def test_dir_dest(self, stage):
|
||||
"""Test using a directory as the destination."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install('source/1', 'dest')
|
||||
|
||||
assert os.path.exists('dest/1')
|
||||
assert os.stat('source/1').st_mode == os.stat('dest/1').st_mode
|
||||
|
||||
|
||||
class TestCopyTree:
|
||||
"""Tests for ``filesystem.copy_tree``"""
|
||||
|
||||
def test_existing_dir(self, stage):
|
||||
"""Test copying to an existing directory."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy_tree('source', 'dest')
|
||||
|
||||
assert os.path.exists('dest/a/b/2')
|
||||
|
||||
def test_non_existing_dir(self, stage):
|
||||
"""Test copying to a non-existing directory."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy_tree('source', 'dest/sub/directory')
|
||||
|
||||
assert os.path.exists('dest/sub/directory/a/b/2')
|
||||
|
||||
def test_symlinks_true(self, stage):
|
||||
"""Test copying with symlink preservation."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy_tree('source', 'dest', symlinks=True)
|
||||
|
||||
assert os.path.exists('dest/2')
|
||||
assert os.path.islink('dest/2')
|
||||
|
||||
def test_symlinks_false(self, stage):
|
||||
"""Test copying without symlink preservation."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.copy_tree('source', 'dest', symlinks=False)
|
||||
|
||||
assert os.path.exists('dest/2')
|
||||
assert not os.path.islink('dest/2')
|
||||
|
||||
|
||||
class TestInstallTree:
|
||||
"""Tests for ``filesystem.install_tree``"""
|
||||
|
||||
def test_existing_dir(self, stage):
|
||||
"""Test installing to an existing directory."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install_tree('source', 'dest')
|
||||
|
||||
assert os.path.exists('dest/a/b/2')
|
||||
|
||||
def test_non_existing_dir(self, stage):
|
||||
"""Test installing to a non-existing directory."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install_tree('source', 'dest/sub/directory')
|
||||
|
||||
assert os.path.exists('dest/sub/directory/a/b/2')
|
||||
|
||||
def test_symlinks_true(self, stage):
|
||||
"""Test installing with symlink preservation."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install_tree('source', 'dest', symlinks=True)
|
||||
|
||||
assert os.path.exists('dest/2')
|
||||
assert os.path.islink('dest/2')
|
||||
|
||||
def test_symlinks_false(self, stage):
|
||||
"""Test installing without symlink preservation."""
|
||||
|
||||
with fs.working_dir(str(stage)):
|
||||
fs.install_tree('source', 'dest', symlinks=False)
|
||||
|
||||
assert os.path.exists('dest/2')
|
||||
assert not os.path.islink('dest/2')
|
||||
|
||||
|
||||
def test_move_transaction_commit(tmpdir):
|
||||
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Just some fake content.')
|
||||
|
||||
old_md5 = fs.hash_directory(str(tmpdir))
|
||||
|
||||
with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Other content.')
|
||||
new_md5 = fs.hash_directory(str(tmpdir))
|
||||
|
||||
assert old_md5 != fs.hash_directory(str(tmpdir))
|
||||
assert new_md5 == fs.hash_directory(str(tmpdir))
|
||||
|
||||
|
||||
def test_move_transaction_rollback(tmpdir):
|
||||
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Just some fake content.')
|
||||
|
||||
h = fs.hash_directory(str(tmpdir))
|
||||
|
||||
try:
|
||||
with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
|
||||
assert h != fs.hash_directory(str(tmpdir))
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Other content.')
|
||||
raise RuntimeError('')
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
assert h == fs.hash_directory(str(tmpdir))
|
||||
@@ -1,61 +0,0 @@
|
||||
##############################################################################
|
||||
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
|
||||
# Produced at the Lawrence Livermore National Laboratory.
|
||||
#
|
||||
# This file is part of Spack.
|
||||
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
|
||||
# LLNL-CODE-647188
|
||||
#
|
||||
# For details, see https://github.com/spack/spack
|
||||
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
|
||||
#
|
||||
# This program is free software; you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Lesser General Public License (as
|
||||
# published by the Free Software Foundation) version 2.1, February 1999.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but
|
||||
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
|
||||
# conditions of the GNU Lesser General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU Lesser General Public
|
||||
# License along with this program; if not, write to the Free Software
|
||||
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
|
||||
##############################################################################
|
||||
|
||||
import llnl.util.filesystem as fs
|
||||
|
||||
|
||||
def test_move_transaction_commit(tmpdir):
|
||||
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Just some fake content.')
|
||||
|
||||
old_md5 = fs.hash_directory(str(tmpdir))
|
||||
|
||||
with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Other content.')
|
||||
new_md5 = fs.hash_directory(str(tmpdir))
|
||||
|
||||
assert old_md5 != fs.hash_directory(str(tmpdir))
|
||||
assert new_md5 == fs.hash_directory(str(tmpdir))
|
||||
|
||||
|
||||
def test_move_transaction_rollback(tmpdir):
|
||||
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Just some fake content.')
|
||||
|
||||
h = fs.hash_directory(str(tmpdir))
|
||||
|
||||
try:
|
||||
with fs.replace_directory_transaction(str(tmpdir.join('lib'))):
|
||||
assert h != fs.hash_directory(str(tmpdir))
|
||||
fake_library = tmpdir.mkdir('lib').join('libfoo.so')
|
||||
fake_library.write('Other content.')
|
||||
raise RuntimeError('')
|
||||
except RuntimeError:
|
||||
pass
|
||||
|
||||
assert h == fs.hash_directory(str(tmpdir))
|
||||
Reference in New Issue
Block a user