zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bootstrap patchelf like GnuPG (#27532)

Browse Source 
Remove a custom bootstrapping procedure to
use spack.bootstrap instead

Modifications:
* Reference count the bootstrap context manager
* Avoid SpackCommand to make the bootstrapping
  procedure more transparent
* Put back requirement on patchelf being in PATH for unit tests
* Add an e2e test to check bootstrapping patchelf
This commit is contained in:
Massimiliano Culpo 2021-11-26 15:32:13 +01:00 committed by GitHub
parent 2cdc758860
commit a96f2f603b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 98 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
.github/workflows/bootstrap.yml vendored
View File

@ -78,6 +78,34 @@ jobs:
spack -d solve zlib
tree ~/.spack/bootstrap/store/
ubuntu-clingo-binaries-and-patchelf:
runs-on: ubuntu-latest
container: "ubuntu:latest"
steps:
- name: Install dependencies
env:
DEBIAN_FRONTEND: noninteractive
run: |
apt-get update -y && apt-get upgrade -y
apt-get install -y \
bzip2 curl file g++ gcc gfortran git gnupg2 gzip \
make patch unzip xz-utils python3 python3-dev tree
- uses: actions/checkout@ec3a7ce113134d7a93b817d10a8272cb61118579 # @v2
- name: Setup repo and non-root user
run: |
git --version
git fetch --unshallow
. .github/workflows/setup_git.sh
useradd -m spack-test
chown -R spack-test .
- name: Bootstrap clingo
shell: runuser -u spack-test -- bash {0}
run: |
source share/spack/setup-env.sh
spack -d solve zlib
tree ~/.spack/bootstrap/store/
opensuse-clingo-sources:
runs-on: ubuntu-latest
container: "opensuse/leap:latest"

62
lib/spack/spack/bootstrap.py
View File

@ -4,6 +4,7 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from __future__ import print_function
import argparse
import contextlib
import fnmatch
import functools
@ -25,7 +26,6 @@
import spack.config
import spack.detection
import spack.environment
import spack.main
import spack.modules
import spack.paths
import spack.platforms
@ -36,9 +36,6 @@
import spack.util.executable
import spack.util.path
#: "spack buildcache" command, initialized lazily
_buildcache_cmd = None
#: Map a bootstrapper type to the corresponding class
_bootstrap_methods = {}
@ -258,10 +255,10 @@ def _read_metadata(self, package_name):
return data
def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform):
global _buildcache_cmd
if _buildcache_cmd is None:
_buildcache_cmd = spack.main.SpackCommand('buildcache')
# TODO: The local import is due to a circular import error. The
# TODO: correct fix for this is a refactor of the API used for
# TODO: binary relocation
import spack.cmd.buildcache
index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
# Reconstruct the compiler that we need to use for bootstrapping
@ -282,13 +279,16 @@ def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform):
'compilers', [{'compiler': compiler_entry}]
):
spec_str = '/' + pkg_hash
parser = argparse.ArgumentParser()
spack.cmd.buildcache.setup_parser(parser)
install_args = [
'install',
'--sha256', pkg_sha256,
'--only-root',
'-a', '-u', '-o', '-f', spec_str
]
_buildcache_cmd(*install_args, fail_on_error=False)
args = parser.parse_args(install_args)
spack.cmd.buildcache.installtarball(args)
def _install_and_test(
self, abstract_spec, bincache_platform, bincache_data, test_fn
@ -421,9 +421,12 @@ def try_search_path(self, executables, abstract_spec_str):
abstract_spec_str += ' os=fe'
concrete_spec = spack.spec.Spec(abstract_spec_str)
concrete_spec.concretize()
if concrete_spec.name == 'patchelf':
concrete_spec._old_concretize(deprecation_warning=False)
else:
concrete_spec.concretize()
msg = "[BOOTSTRAP GnuPG] Try installing '{0}' from sources"
msg = "[BOOTSTRAP] Try installing '{0}' from sources"
tty.debug(msg.format(abstract_spec_str))
concrete_spec.package.do_install()
if _executables_in_store(executables, concrete_spec, query_info=info):
@ -644,8 +647,30 @@ def _add_externals_if_missing():
spack.detection.update_configuration(detected_packages, scope='bootstrap')
#: Reference counter for the bootstrapping configuration context manager
_REF_COUNT = 0
@contextlib.contextmanager
def ensure_bootstrap_configuration():
# The context manager is reference counted to ensure we don't swap multiple
# times if there's nested use of it in the stack. One compelling use case
# is bootstrapping patchelf during the bootstrap of clingo.
global _REF_COUNT
already_swapped = bool(_REF_COUNT)
_REF_COUNT += 1
try:
if already_swapped:
yield
else:
with _ensure_bootstrap_configuration():
yield
finally:
_REF_COUNT -= 1
@contextlib.contextmanager
def _ensure_bootstrap_configuration():
bootstrap_store_path = store_path()
user_configuration = _read_and_sanitize_configuration()
with spack.environment.no_active_environment():
@ -753,10 +778,23 @@ def gnupg_root_spec():
def ensure_gpg_in_path_or_raise():
"""Ensure gpg or gpg2 are in the PATH or raise."""
ensure_executables_in_path_or_raise(
return ensure_executables_in_path_or_raise(
executables=['gpg2', 'gpg'], abstract_spec=gnupg_root_spec()
)
def patchelf_root_spec():
"""Return the root spec used to bootstrap patchelf"""
return _root_spec('patchelf@0.13:')
def ensure_patchelf_in_path_or_raise():
"""Ensure patchelf is in the PATH or raise."""
return ensure_executables_in_path_or_raise(
executables=['patchelf'], abstract_spec=patchelf_root_spec()
)
###
# Development dependencies
###

31
lib/spack/spack/relocate.py
View File

@ -14,6 +14,7 @@
import llnl.util.lang
import llnl.util.tty as tty
import spack.bootstrap
import spack.platforms
import spack.repo
import spack.spec
@ -75,32 +76,16 @@ def __init__(self, old_path, new_path):
def _patchelf():
"""Return the full path to the patchelf binary, if available, else None.
Search first the current PATH for patchelf. If not found, try to look
if the default patchelf spec is installed and if not install it.
Return None on Darwin or if patchelf cannot be found.
"""
# Check if patchelf is already in the PATH
patchelf = executable.which('patchelf')
if patchelf is not None:
return patchelf.path
# Check if patchelf spec is installed
spec = spack.spec.Spec('patchelf')
spec._old_concretize(deprecation_warning=False)
exe_path = os.path.join(spec.prefix.bin, "patchelf")
if spec.package.installed and os.path.exists(exe_path):
return exe_path
# Skip darwin
"""Return the full path to the patchelf binary, if available, else None."""
if is_macos:
return None
# Install the spec and return its path
spec.package.do_install()
return exe_path if os.path.exists(exe_path) else None
patchelf = executable.which('patchelf')
if patchelf is None:
with spack.bootstrap.ensure_bootstrap_configuration():
patchelf = spack.bootstrap.ensure_patchelf_in_path_or_raise()
return patchelf.path
def _elf_rpaths_for(path):

10
lib/spack/spack/test/bootstrap.py
View File

@ -140,3 +140,13 @@ def test_custom_store_in_environment(mutable_config, tmpdir):
with spack.bootstrap.ensure_bootstrap_configuration():
pass
assert str(spack.store.root) == '/tmp/store'
def test_nested_use_of_context_manager(mutable_config):
"""Test nested use of the context manager"""
user_config = spack.config.config
with spack.bootstrap.ensure_bootstrap_configuration():
assert spack.config.config != user_config
with spack.bootstrap.ensure_bootstrap_configuration():
assert spack.config.config != user_config
assert spack.config.config == user_config

50
lib/spack/spack/test/relocate.py
View File

@ -2,7 +2,6 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import os.path
import re
import shutil
@ -73,47 +72,6 @@ def source_file(tmpdir, is_relocatable):
return src
@pytest.fixture(params=['which_found', 'installed', 'to_be_installed'])
def expected_patchelf_path(request, mutable_database, monkeypatch):
"""Prepare the stage to tests different cases that can occur
when searching for patchelf.
"""
case = request.param
# Mock the which function
which_fn = {
'which_found': lambda x: collections.namedtuple(
'_', ['path']
)('/usr/bin/patchelf')
}
monkeypatch.setattr(
spack.util.executable, 'which',
which_fn.setdefault(case, lambda x: None)
)
if case == 'which_found':
return '/usr/bin/patchelf'
# TODO: Mock a case for Darwin architecture
spec = spack.spec.Spec('patchelf')
spec.concretize()
patchelf_cls = type(spec.package)
do_install = patchelf_cls.do_install
expected_path = os.path.join(spec.prefix.bin, 'patchelf')
def do_install_mock(self, **kwargs):
do_install(self, fake=True)
with open(expected_path):
pass
monkeypatch.setattr(patchelf_cls, 'do_install', do_install_mock)
if case == 'installed':
spec.package.do_install()
return expected_path
@pytest.fixture()
def mock_patchelf(tmpdir, mock_executable):
def _factory(output):
@ -227,6 +185,7 @@ def _copy_somewhere(orig_binary):
@pytest.mark.requires_executables(
'/usr/bin/gcc', 'patchelf', 'strings', 'file'
)
@skip_unless_linux
def test_file_is_relocatable(source_file, is_relocatable):
compiler = spack.util.executable.Executable('/usr/bin/gcc')
executable = str(source_file).replace('.c', '.x')
@ -240,6 +199,7 @@ def test_file_is_relocatable(source_file, is_relocatable):
@pytest.mark.requires_executables('patchelf', 'strings', 'file')
@skip_unless_linux
def test_patchelf_is_relocatable():
patchelf = os.path.realpath(spack.relocate._patchelf())
assert llnl.util.filesystem.is_exe(patchelf)
@ -263,12 +223,6 @@ def test_file_is_relocatable_errors(tmpdir):
assert 'is not an absolute path' in str(exc_info.value)
@skip_unless_linux
def test_search_patchelf(expected_patchelf_path):
current = spack.relocate._patchelf()
assert current == expected_patchelf_path
@pytest.mark.parametrize('patchelf_behavior,expected', [
('echo ', []),
('echo /opt/foo/lib:/opt/foo/lib64', ['/opt/foo/lib', '/opt/foo/lib64']),