Buildcache creation/extraction use temp tarfile to preserve hardlinks during copy to/from prefix. (#15003)
* Buildcache creation change the way prefix is copied to workdir. * install_tree copies hardlinked files * tarfile creates hardlinked files on extraction. * create a temporary tarfile from prefix and extract it to workdir * Use temp tarfile to move workdir to prefix to preserve hardlinks instead of copying
This commit is contained in:
Patrick Gartung
GitHub
parent
3c28e72d2f
commit
baafa7ec84
@ -18,7 +18,7 @@
|
||||
from six.moves.urllib.error import URLError
|
||||
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.filesystem import mkdirp, install_tree
|
||||
from llnl.util.filesystem import mkdirp
|
||||
|
||||
import spack.cmd
|
||||
import spack.config as config
|
||||
@ -308,7 +308,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
cache_prefix = build_cache_prefix(tmpdir)
|
||||
|
||||
tarfile_name = tarball_name(spec, '.tar.gz')
|
||||
tarfile_name = tarball_name(spec, '.tar.bz2')
|
||||
tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec))
|
||||
tarfile_path = os.path.join(tarfile_dir, tarfile_name)
|
||||
spackfile_path = os.path.join(
|
||||
@ -342,8 +342,18 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
|
||||
raise NoOverwriteException(url_util.format(remote_specfile_path))
|
||||
|
||||
# make a copy of the install directory to work with
|
||||
workdir = os.path.join(tempfile.mkdtemp(), os.path.basename(spec.prefix))
|
||||
install_tree(spec.prefix, workdir, symlinks=True)
|
||||
workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
|
||||
# install_tree copies hardlinks
|
||||
# create a temporary tarfile from prefix and exract it to workdir
|
||||
# tarfile preserves hardlinks
|
||||
temp_tarfile_name = tarball_name(spec, '.tar')
|
||||
temp_tarfile_path = os.path.join(tarfile_dir, temp_tarfile_name)
|
||||
with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
|
||||
tar.add(name='%s' % spec.prefix,
|
||||
arcname='.')
|
||||
with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
|
||||
tar.extractall(workdir)
|
||||
os.remove(temp_tarfile_path)
|
||||
|
||||
# create info for later relocation and create tar
|
||||
write_buildinfo_file(spec.prefix, workdir, rel=rel)
|
||||
@ -368,7 +378,7 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
|
||||
tty.die(e)
|
||||
|
||||
# create compressed tarball of the install prefix
|
||||
with closing(tarfile.open(tarfile_path, 'w:gz')) as tar:
|
||||
with closing(tarfile.open(tarfile_path, 'w:bz2')) as tar:
|
||||
tar.add(name='%s' % workdir,
|
||||
arcname='%s' % os.path.basename(spec.prefix))
|
||||
# remove copy of install directory
|
||||
@ -407,8 +417,8 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
|
||||
sign_tarball(key, force, specfile_path)
|
||||
# put tarball, spec and signature files in .spack archive
|
||||
with closing(tarfile.open(spackfile_path, 'w')) as tar:
|
||||
tar.add(name='%s' % tarfile_path, arcname='%s' % tarfile_name)
|
||||
tar.add(name='%s' % specfile_path, arcname='%s' % specfile_name)
|
||||
tar.add(name=tarfile_path, arcname='%s' % tarfile_name)
|
||||
tar.add(name=specfile_path, arcname='%s' % specfile_name)
|
||||
if not unsigned:
|
||||
tar.add(name='%s.asc' % specfile_path,
|
||||
arcname='%s.asc' % specfile_name)
|
||||
@ -579,7 +589,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
|
||||
stagepath = os.path.dirname(filename)
|
||||
spackfile_name = tarball_name(spec, '.spack')
|
||||
spackfile_path = os.path.join(stagepath, spackfile_name)
|
||||
tarfile_name = tarball_name(spec, '.tar.gz')
|
||||
tarfile_name = tarball_name(spec, '.tar.bz2')
|
||||
tarfile_path = os.path.join(tmpdir, tarfile_name)
|
||||
specfile_name = tarball_name(spec, '.spec.yaml')
|
||||
specfile_path = os.path.join(tmpdir, specfile_name)
|
||||
@ -638,7 +648,17 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
|
||||
# so the pathname should be the same now that the directory layout
|
||||
# is confirmed
|
||||
workdir = os.path.join(tmpdir, os.path.basename(spec.prefix))
|
||||
install_tree(workdir, spec.prefix, symlinks=True)
|
||||
# install_tree copies hardlinks
|
||||
# create a temporary tarfile from prefix and exract it to workdir
|
||||
# tarfile preserves hardlinks
|
||||
temp_tarfile_name = tarball_name(spec, '.tar')
|
||||
temp_tarfile_path = os.path.join(tmpdir, temp_tarfile_name)
|
||||
with closing(tarfile.open(temp_tarfile_path, 'w')) as tar:
|
||||
tar.add(name='%s' % workdir,
|
||||
arcname='.')
|
||||
with closing(tarfile.open(temp_tarfile_path, 'r')) as tar:
|
||||
tar.extractall(spec.prefix)
|
||||
os.remove(temp_tarfile_path)
|
||||
|
||||
# cleanup
|
||||
os.remove(tarfile_path)
|
||||
|
||||
@ -107,11 +107,6 @@ def test_buildcache(mock_archive, tmpdir):
|
||||
buildcache.buildcache(parser, args)
|
||||
|
||||
files = os.listdir(spec.prefix)
|
||||
assert 'link_to_dummy.txt' in files
|
||||
assert 'dummy.txt' in files
|
||||
assert os.path.realpath(
|
||||
os.path.join(spec.prefix, 'link_to_dummy.txt')
|
||||
) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
|
||||
|
||||
# create build cache with relative path and signing
|
||||
args = parser.parse_args(
|
||||
@ -129,13 +124,6 @@ def test_buildcache(mock_archive, tmpdir):
|
||||
args = parser.parse_args(['install', '-f', str(pkghash)])
|
||||
buildcache.buildcache(parser, args)
|
||||
|
||||
files = os.listdir(spec.prefix)
|
||||
assert 'link_to_dummy.txt' in files
|
||||
assert 'dummy.txt' in files
|
||||
assert os.path.realpath(
|
||||
os.path.join(spec.prefix, 'link_to_dummy.txt')
|
||||
) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
|
||||
|
||||
else:
|
||||
# create build cache without signing
|
||||
args = parser.parse_args(
|
||||
@ -152,10 +140,6 @@ def test_buildcache(mock_archive, tmpdir):
|
||||
files = os.listdir(spec.prefix)
|
||||
assert 'link_to_dummy.txt' in files
|
||||
assert 'dummy.txt' in files
|
||||
assert os.path.realpath(
|
||||
os.path.join(spec.prefix, 'link_to_dummy.txt')
|
||||
) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
|
||||
|
||||
# test overwrite install without verification
|
||||
args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
|
||||
buildcache.buildcache(parser, args)
|
||||
@ -242,7 +226,7 @@ def test_relocate_links(tmpdir):
|
||||
old_src = os.path.join(old_dir, filename)
|
||||
os.symlink(old_src, filename)
|
||||
filenames = [filename]
|
||||
new_dir = '/opt/rh/devtoolset/'
|
||||
new_dir = '/opt/rh/devtoolset'
|
||||
relocate_links(filenames, old_dir, new_dir)
|
||||
assert os.path.realpath(filename) == os.path.join(new_dir, filename)
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user