rewiring of spliced specs (#26873)
* tests for rewiring pure specs to spliced specs * relocate text, binaries, and links * using llnl.util.symlink for windows compat. Note: This does not include CLI hooks for relocation. Co-authored-by: Nathan Hanford <hanford1@llnl.gov>
This commit is contained in:
Nathan Hanford
GitHub
parent
8ddaa08ed2
commit
88d8ca9b65
128
lib/spack/spack/rewiring.py
Normal file
128
lib/spack/spack/rewiring.py
Normal file
@ -0,0 +1,128 @@
|
||||
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
from collections import OrderedDict
|
||||
|
||||
from llnl.util.symlink import symlink
|
||||
|
||||
import spack.binary_distribution as bindist
|
||||
import spack.error
|
||||
import spack.hooks
|
||||
import spack.paths
|
||||
import spack.relocate as relocate
|
||||
import spack.stage
|
||||
import spack.store
|
||||
|
||||
|
||||
def _relocate_spliced_links(links, orig_prefix, new_prefix):
|
||||
"""Re-linking function which differs from `relocate.relocate_links` by
|
||||
reading the old link rather than the new link, since the latter wasn't moved
|
||||
in our case. This still needs to be called after the copy to destination
|
||||
because it expects the new directory structure to be in place."""
|
||||
for link in links:
|
||||
link_target = os.readlink(os.path.join(orig_prefix, link))
|
||||
link_target = re.sub('^' + orig_prefix, new_prefix, link_target)
|
||||
new_link_path = os.path.join(new_prefix, link)
|
||||
os.unlink(new_link_path)
|
||||
symlink(link_target, new_link_path)
|
||||
|
||||
|
||||
def rewire(spliced_spec):
|
||||
"""Given a spliced spec, this function conducts all the rewiring on all
|
||||
nodes in the DAG of that spec."""
|
||||
assert spliced_spec.spliced
|
||||
for spec in spliced_spec.traverse(order='post', root=True):
|
||||
if not spec.build_spec.package.installed:
|
||||
# TODO: May want to change this at least for the root spec...
|
||||
# spec.build_spec.package.do_install(force=True)
|
||||
raise PackageNotInstalledError(spliced_spec,
|
||||
spec.build_spec,
|
||||
spec)
|
||||
if spec.build_spec is not spec and not spec.package.installed:
|
||||
explicit = spec is spliced_spec
|
||||
rewire_node(spec, explicit)
|
||||
|
||||
|
||||
def rewire_node(spec, explicit):
|
||||
"""This function rewires a single node, worrying only about references to
|
||||
its subgraph. Binaries, text, and links are all changed in accordance with
|
||||
the splice. The resulting package is then 'installed.'"""
|
||||
tempdir = tempfile.mkdtemp()
|
||||
# copy anything installed to a temporary directory
|
||||
shutil.copytree(spec.build_spec.prefix,
|
||||
os.path.join(tempdir, spec.dag_hash()))
|
||||
|
||||
spack.hooks.pre_install(spec)
|
||||
# compute prefix-to-prefix for every node from the build spec to the spliced
|
||||
# spec
|
||||
prefix_to_prefix = OrderedDict({spec.build_spec.prefix: spec.prefix})
|
||||
for build_dep in spec.build_spec.traverse(root=False):
|
||||
prefix_to_prefix[build_dep.prefix] = spec[build_dep.name].prefix
|
||||
|
||||
manifest = bindist.get_buildfile_manifest(spec.build_spec)
|
||||
platform = spack.platforms.by_name(spec.platform)
|
||||
|
||||
text_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path)
|
||||
for rel_path in manifest.get('text_to_relocate', [])]
|
||||
if text_to_relocate:
|
||||
relocate.relocate_text(files=text_to_relocate,
|
||||
prefixes=prefix_to_prefix)
|
||||
|
||||
bins_to_relocate = [os.path.join(tempdir, spec.dag_hash(), rel_path)
|
||||
for rel_path in manifest.get('binary_to_relocate', [])]
|
||||
if bins_to_relocate:
|
||||
if 'macho' in platform.binary_formats:
|
||||
relocate.relocate_macho_binaries(bins_to_relocate,
|
||||
str(spack.store.layout.root),
|
||||
str(spack.store.layout.root),
|
||||
prefix_to_prefix,
|
||||
False,
|
||||
spec.build_spec.prefix,
|
||||
spec.prefix)
|
||||
if 'elf' in platform.binary_formats:
|
||||
relocate.relocate_elf_binaries(bins_to_relocate,
|
||||
str(spack.store.layout.root),
|
||||
str(spack.store.layout.root),
|
||||
prefix_to_prefix,
|
||||
False,
|
||||
spec.build_spec.prefix,
|
||||
spec.prefix)
|
||||
relocate.relocate_text_bin(binaries=bins_to_relocate,
|
||||
prefixes=prefix_to_prefix)
|
||||
# copy package into place (shutil.copytree)
|
||||
shutil.copytree(os.path.join(tempdir, spec.dag_hash()), spec.prefix,
|
||||
ignore=shutil.ignore_patterns('spec.json',
|
||||
'install_manifest.json'))
|
||||
if manifest.get('link_to_relocate'):
|
||||
_relocate_spliced_links(manifest.get('link_to_relocate'),
|
||||
spec.build_spec.prefix,
|
||||
spec.prefix)
|
||||
shutil.rmtree(tempdir)
|
||||
# handle all metadata changes; don't copy over spec.json file in .spack/
|
||||
spack.store.layout.write_spec(spec, spack.store.layout.spec_file_path(spec))
|
||||
# add to database, not sure about explicit
|
||||
spack.store.db.add(spec, spack.store.layout, explicit=explicit)
|
||||
|
||||
# run post install hooks
|
||||
spack.hooks.post_install(spec)
|
||||
|
||||
|
||||
class RewireError(spack.error.SpackError):
|
||||
"""Raised when something goes wrong with rewiring."""
|
||||
def __init__(self, message, long_msg=None):
|
||||
super(RewireError, self).__init__(message, long_msg)
|
||||
|
||||
|
||||
class PackageNotInstalledError(RewireError):
|
||||
"""Raised when the build_spec for a splice was not installed."""
|
||||
def __init__(self, spliced_spec, build_spec, dep):
|
||||
super(PackageNotInstalledError, self).__init__(
|
||||
"""Rewire of {0}
|
||||
failed due to missing install of build spec {1}
|
||||
for spec {2}""".format(spliced_spec, build_spec, dep))
|
||||
142
lib/spack/spack/test/rewiring.py
Normal file
142
lib/spack/spack/test/rewiring.py
Normal file
@ -0,0 +1,142 @@
|
||||
# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import filecmp
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
import spack.rewiring
|
||||
import spack.store
|
||||
from spack.spec import Spec
|
||||
from spack.test.relocate import text_in_bin
|
||||
|
||||
args = ['strings', 'file']
|
||||
if sys.platform == 'darwin':
|
||||
args.extend(['/usr/bin/clang++', 'install_name_tool'])
|
||||
else:
|
||||
args.extend(['/usr/bin/g++', 'patchelf'])
|
||||
|
||||
|
||||
@pytest.mark.requires_executables(*args)
|
||||
@pytest.mark.parametrize('transitive', [True, False])
|
||||
def test_rewire(mock_fetch, install_mockery, transitive):
|
||||
spec = Spec('splice-t^splice-h~foo').concretized()
|
||||
dep = Spec('splice-h+foo').concretized()
|
||||
spec.package.do_install()
|
||||
dep.package.do_install()
|
||||
spliced_spec = spec.splice(dep, transitive=transitive)
|
||||
assert spec.dag_hash() != spliced_spec.dag_hash()
|
||||
|
||||
spack.rewiring.rewire(spliced_spec)
|
||||
|
||||
# check that the prefix exists
|
||||
assert os.path.exists(spliced_spec.prefix)
|
||||
|
||||
# test that it made it into the database
|
||||
rec = spack.store.db.get_record(spliced_spec)
|
||||
installed_in_db = rec.installed if rec else False
|
||||
assert installed_in_db
|
||||
|
||||
# check the file in the prefix has the correct paths
|
||||
for node in spliced_spec.traverse(root=True):
|
||||
text_file_path = os.path.join(node.prefix, node.name)
|
||||
with open(text_file_path, 'r') as f:
|
||||
text = f.read()
|
||||
for modded_spec in node.traverse(root=True):
|
||||
assert modded_spec.prefix in text
|
||||
|
||||
|
||||
@pytest.mark.requires_executables(*args)
|
||||
@pytest.mark.parametrize('transitive', [True, False])
|
||||
def test_rewire_bin(mock_fetch, install_mockery, transitive):
|
||||
spec = Spec('quux').concretized()
|
||||
dep = Spec('garply cflags=-g').concretized()
|
||||
spec.package.do_install()
|
||||
dep.package.do_install()
|
||||
spliced_spec = spec.splice(dep, transitive=transitive)
|
||||
assert spec.dag_hash() != spliced_spec.dag_hash()
|
||||
|
||||
spack.rewiring.rewire(spliced_spec)
|
||||
|
||||
# check that the prefix exists
|
||||
assert os.path.exists(spliced_spec.prefix)
|
||||
|
||||
# test that it made it into the database
|
||||
rec = spack.store.db.get_record(spliced_spec)
|
||||
installed_in_db = rec.installed if rec else False
|
||||
assert installed_in_db
|
||||
|
||||
# check the file in the prefix has the correct paths
|
||||
bin_names = {'garply': 'garplinator',
|
||||
'corge': 'corgegator',
|
||||
'quux': 'quuxifier'}
|
||||
for node in spliced_spec.traverse(root=True):
|
||||
for dep in node.traverse(root=True):
|
||||
bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name])
|
||||
assert text_in_bin(dep.prefix, bin_file_path)
|
||||
|
||||
|
||||
@pytest.mark.requires_executables(*args)
|
||||
def test_rewire_writes_new_metadata(mock_fetch, install_mockery):
|
||||
# check for spec.json and install_manifest.json and that they are new
|
||||
# for a simple case.
|
||||
spec = Spec('quux').concretized()
|
||||
dep = Spec('garply cflags=-g').concretized()
|
||||
spec.package.do_install()
|
||||
dep.package.do_install()
|
||||
spliced_spec = spec.splice(dep, transitive=True)
|
||||
spack.rewiring.rewire(spliced_spec)
|
||||
|
||||
# test install manifests
|
||||
for node in spliced_spec.traverse(root=True):
|
||||
spack.store.layout.ensure_installed(node)
|
||||
manifest_file_path = os.path.join(node.prefix,
|
||||
spack.store.layout.metadata_dir,
|
||||
spack.store.layout.manifest_file_name)
|
||||
assert os.path.exists(manifest_file_path)
|
||||
orig_node = spec[node.name]
|
||||
orig_manifest_file_path = os.path.join(orig_node.prefix,
|
||||
spack.store.layout.metadata_dir,
|
||||
spack.store.layout.manifest_file_name)
|
||||
assert os.path.exists(orig_manifest_file_path)
|
||||
assert not filecmp.cmp(orig_manifest_file_path, manifest_file_path,
|
||||
shallow=False)
|
||||
specfile_path = os.path.join(node.prefix,
|
||||
spack.store.layout.metadata_dir,
|
||||
spack.store.layout.spec_file_name)
|
||||
assert os.path.exists(specfile_path)
|
||||
orig_specfile_path = os.path.join(orig_node.prefix,
|
||||
spack.store.layout.metadata_dir,
|
||||
spack.store.layout.spec_file_name)
|
||||
assert os.path.exists(orig_specfile_path)
|
||||
assert not filecmp.cmp(orig_specfile_path, specfile_path,
|
||||
shallow=False)
|
||||
|
||||
|
||||
@pytest.mark.requires_executables(*args)
|
||||
@pytest.mark.parametrize('transitive', [True, False])
|
||||
def test_uninstall_rewired_spec(mock_fetch, install_mockery, transitive):
|
||||
# Test that rewired packages can be uninstalled as normal.
|
||||
spec = Spec('quux').concretized()
|
||||
dep = Spec('garply cflags=-g').concretized()
|
||||
spec.package.do_install()
|
||||
dep.package.do_install()
|
||||
spliced_spec = spec.splice(dep, transitive=transitive)
|
||||
spack.rewiring.rewire(spliced_spec)
|
||||
spliced_spec.package.do_uninstall()
|
||||
assert len(spack.store.db.query(spliced_spec)) == 0
|
||||
assert not os.path.exists(spliced_spec.prefix)
|
||||
|
||||
|
||||
@pytest.mark.requires_executables(*args)
|
||||
def test_rewire_not_installed_fails(mock_fetch, install_mockery):
|
||||
spec = Spec('quux').concretized()
|
||||
dep = Spec('garply cflags=-g').concretized()
|
||||
spliced_spec = spec.splice(dep, False)
|
||||
with pytest.raises(spack.rewiring.PackageNotInstalledError,
|
||||
match="failed due to missing install of build spec"):
|
||||
spack.rewiring.rewire(spliced_spec)
|
||||
@ -6,7 +6,7 @@
|
||||
from spack import *
|
||||
|
||||
|
||||
class SpliceH(AutotoolsPackage):
|
||||
class SpliceH(Package):
|
||||
"""Simple package with one optional dependency"""
|
||||
|
||||
homepage = "http://www.example.com"
|
||||
@ -20,3 +20,8 @@ class SpliceH(AutotoolsPackage):
|
||||
|
||||
depends_on('splice-z')
|
||||
depends_on('splice-z+foo', when='+foo')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
with open(prefix.join('splice-h'), 'w') as f:
|
||||
f.write('splice-h: {0}'.format(prefix))
|
||||
f.write('splice-z: {0}'.format(spec['splice-z'].prefix))
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
from spack import *
|
||||
|
||||
|
||||
class SpliceT(AutotoolsPackage):
|
||||
class SpliceT(Package):
|
||||
"""Simple package with one optional dependency"""
|
||||
|
||||
homepage = "http://www.example.com"
|
||||
@ -16,3 +16,9 @@ class SpliceT(AutotoolsPackage):
|
||||
|
||||
depends_on('splice-h')
|
||||
depends_on('splice-z')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
with open(prefix.join('splice-t'), 'w') as f:
|
||||
f.write('splice-t: {0}'.format(prefix))
|
||||
f.write('splice-h: {0}'.format(spec['splice-h'].prefix))
|
||||
f.write('splice-z: {0}'.format(spec['splice-z'].prefix))
|
||||
|
||||
@ -6,7 +6,7 @@
|
||||
from spack import *
|
||||
|
||||
|
||||
class SpliceZ(AutotoolsPackage):
|
||||
class SpliceZ(Package):
|
||||
"""Simple package with one optional dependency"""
|
||||
|
||||
homepage = "http://www.example.com"
|
||||
@ -16,3 +16,7 @@ class SpliceZ(AutotoolsPackage):
|
||||
|
||||
variant('foo', default=False, description='nope')
|
||||
variant('bar', default=False, description='nope')
|
||||
|
||||
def install(self, spec, prefix):
|
||||
with open(prefix.join('splice-z'), 'w') as f:
|
||||
f.write('splice-z: {0}'.format(prefix))
|
||||
|
||||
Loading…
Reference in New Issue
Block a user