zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bootstrap GnuPG (#24003)

Browse Source 
* GnuPG: allow bootstrapping from buildcache and sources

* Add a test to bootstrap GnuPG from binaries

* Disable bootstrapping in tests

* Add e2e test to bootstrap GnuPG from sources on Ubuntu

* Add e2e test to bootstrap GnuPG on macOS
This commit is contained in:
Massimiliano Culpo
2021-11-03 07:15:24 +01:00
committed by GitHub
parent 1a3747b2b3
commit 78c08fccd5
8 changed files with 549 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

289
lib/spack/spack/bootstrap.py
View File

@@ -6,6 +6,7 @@
import contextlib
import fnmatch
import functools
import json
import os
import os.path
@@ -34,11 +35,14 @@
import spack.repo
import spack.spec
import spack.store
import spack.user_environment as uenv
import spack.user_environment
import spack.util.executable
import spack.util.path
from spack.util.environment import EnvironmentModifications
#: "spack buildcache" command, initialized lazily
_buildcache_cmd = None
#: Map a bootstrapper type to the corresponding class
_bootstrap_methods = {}
@@ -171,6 +175,34 @@ def _fix_ext_suffix(candidate_spec):
os.symlink(abs_path, link_name)
def _executables_in_store(executables, abstract_spec_str):
"""Return True if at least one of the executables can be retrieved from
a spec in store, False otherwise.
The different executables must provide the same functionality and are
"alternate" to each other, i.e. the function will exit True on the first
executable found.
Args:
executables: list of executables to be searched
abstract_spec_str: abstract spec that may provide the executable
"""
executables_str = ', '.join(executables)
msg = "[BOOTSTRAP EXECUTABLES {0}] Try installed specs with query '{1}'"
tty.debug(msg.format(executables_str, abstract_spec_str))
installed_specs = spack.store.db.query(abstract_spec_str, installed=True)
if installed_specs:
for concrete_spec in installed_specs:
bin_dir = concrete_spec.prefix.bin
# IF we have a "bin" directory and it contains
# the executables we are looking for
if (os.path.exists(bin_dir) and os.path.isdir(bin_dir) and
spack.util.executable.which_string(*executables, path=bin_dir)):
spack.util.environment.path_put_first('PATH', [bin_dir])
return True
return False
@_bootstrapper(type='buildcache')
class _BuildcacheBootstrapper(object):
"""Install the software needed during bootstrapping from a buildcache."""
@@ -178,93 +210,140 @@ def __init__(self, conf):
self.name = conf['name']
self.url = conf['info']['url']
def try_import(self, module, abstract_spec_str):
if _try_import_from_store(module, abstract_spec_str):
return True
@staticmethod
def _spec_and_platform(abstract_spec_str):
"""Return the spec object and platform we need to use when
querying the buildcache.
tty.info("Bootstrapping {0} from pre-built binaries".format(module))
Args:
abstract_spec_str: abstract spec string we are looking for
"""
# This import is local since it is needed only on Cray
import spack.platforms.linux
# Try to install from an unsigned binary cache
abstract_spec = spack.spec.Spec(
abstract_spec_str + ' ^' + spec_for_current_python()
)
abstract_spec = spack.spec.Spec(abstract_spec_str)
# On Cray we want to use Linux binaries if available from mirrors
bincache_platform = spack.platforms.real_host()
if str(bincache_platform) == 'cray':
bincache_platform = spack.platforms.Linux()
with spack.platforms.use_platform(bincache_platform):
abstract_spec = spack.spec.Spec(
abstract_spec_str + ' ^' + spec_for_current_python()
)
abstract_spec = spack.spec.Spec(abstract_spec_str)
return abstract_spec, bincache_platform
# Read information on verified clingo binaries
json_filename = '{0}.json'.format(module)
def _read_metadata(self, package_name):
"""Return metadata about the given package."""
json_filename = '{0}.json'.format(package_name)
json_path = os.path.join(
spack.paths.share_path, 'bootstrap', self.name, json_filename
)
with open(json_path) as f:
data = json.load(f)
return data
buildcache = spack.main.SpackCommand('buildcache')
def _install_by_hash(self, pkg_hash, pkg_sha256, index, bincache_platform):
global _buildcache_cmd
if _buildcache_cmd is None:
_buildcache_cmd = spack.main.SpackCommand('buildcache')
index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
# Reconstruct the compiler that we need to use for bootstrapping
compiler_entry = {
"modules": [],
"operating_system": str(index_spec.os),
"paths": {
"cc": "/dev/null",
"cxx": "/dev/null",
"f77": "/dev/null",
"fc": "/dev/null"
},
"spec": str(index_spec.compiler),
"target": str(index_spec.target.family)
}
with spack.platforms.use_platform(bincache_platform):
with spack.config.override(
'compilers', [{'compiler': compiler_entry}]
):
spec_str = '/' + pkg_hash
install_args = [
'install',
'--sha256', pkg_sha256,
'--only-root',
'-a', '-u', '-o', '-f', spec_str
]
_buildcache_cmd(*install_args, fail_on_error=False)
def _install_and_test(
self, abstract_spec, bincache_platform, bincache_data, test_fn
):
# Ensure we see only the buildcache being used to bootstrap
mirror_scope = spack.config.InternalConfigScope(
'bootstrap_buildcache', {'mirrors:': {self.name: self.url}}
)
with spack.config.override(mirror_scope):
with spack.config.override(self.mirror_scope):
# This index is currently needed to get the compiler used to build some
# specs that wwe know by dag hash.
# specs that we know by dag hash.
spack.binary_distribution.binary_index.regenerate_spec_cache()
index = spack.binary_distribution.update_cache_and_get_specs()
if not index:
raise RuntimeError("The binary index is empty")
for item in data['verified']:
for item in bincache_data['verified']:
candidate_spec = item['spec']
python_spec = item['python']
# This will be None for things that don't depend on python
python_spec = item.get('python', None)
# Skip specs which are not compatible
if not abstract_spec.satisfies(candidate_spec):
continue
if python_spec not in abstract_spec:
if python_spec is not None and python_spec not in abstract_spec:
continue
for pkg_name, pkg_hash, pkg_sha256 in item['binaries']:
msg = ('[BOOTSTRAP MODULE {0}] Try installing "{1}" from binary '
'cache at "{2}"')
tty.debug(msg.format(module, pkg_name, self.url))
index_spec = next(x for x in index if x.dag_hash() == pkg_hash)
# Reconstruct the compiler that we need to use for bootstrapping
compiler_entry = {
"modules": [],
"operating_system": str(index_spec.os),
"paths": {
"cc": "/dev/null",
"cxx": "/dev/null",
"f77": "/dev/null",
"fc": "/dev/null"
},
"spec": str(index_spec.compiler),
"target": str(index_spec.target.family)
}
with spack.platforms.use_platform(bincache_platform):
with spack.config.override(
'compilers', [{'compiler': compiler_entry}]
):
spec_str = '/' + pkg_hash
install_args = [
'install',
'--sha256', pkg_sha256,
'-a', '-u', '-o', '-f', spec_str
]
buildcache(*install_args, fail_on_error=False)
# TODO: undo installations that didn't complete?
# TODO: undo installations that didn't complete?
self._install_by_hash(
pkg_hash, pkg_sha256, index, bincache_platform
)
if _try_import_from_store(module, abstract_spec_str):
if test_fn():
return True
return False
@property
def mirror_scope(self):
return spack.config.InternalConfigScope(
'bootstrap', {'mirrors:': {self.name: self.url}}
)
def try_import(self, module, abstract_spec_str):
test_fn = functools.partial(_try_import_from_store, module, abstract_spec_str)
if test_fn():
return True
tty.info("Bootstrapping {0} from pre-built binaries".format(module))
abstract_spec, bincache_platform = self._spec_and_platform(
abstract_spec_str + ' ^' + spec_for_current_python()
)
data = self._read_metadata(module)
return self._install_and_test(
abstract_spec, bincache_platform, data, test_fn
)
def try_search_path(self, executables, abstract_spec_str):
test_fn = functools.partial(
_executables_in_store, executables, abstract_spec_str
)
if test_fn():
return True
abstract_spec, bincache_platform = self._spec_and_platform(
abstract_spec_str
)
tty.info("Bootstrapping {0} from pre-built binaries".format(abstract_spec.name))
data = self._read_metadata(abstract_spec.name)
return self._install_and_test(
abstract_spec, bincache_platform, data, test_fn
)
@_bootstrapper(type='install')
class _SourceBootstrapper(object):
@@ -307,6 +386,26 @@ def try_import(module, abstract_spec_str):
return _try_import_from_store(module, abstract_spec_str=abstract_spec_str)
def try_search_path(self, executables, abstract_spec_str):
if _executables_in_store(executables, abstract_spec_str):
return True
# If we compile code from sources detecting a few build tools
# might reduce compilation time by a fair amount
_add_externals_if_missing()
# Add hint to use frontend operating system on Cray
if str(spack.platforms.host()) == 'cray':
abstract_spec_str += ' os=fe'
concrete_spec = spack.spec.Spec(abstract_spec_str)
concrete_spec.concretize()
msg = "[BOOTSTRAP GnuPG] Try installing '{0}' from sources"
tty.debug(msg.format(abstract_spec_str))
concrete_spec.package.do_install()
return _executables_in_store(executables, abstract_spec_str)
def _make_bootstrapper(conf):
"""Return a bootstrap object built according to the
@@ -418,6 +517,44 @@ def ensure_module_importable_or_raise(module, abstract_spec=None):
raise ImportError(msg)
def ensure_executables_in_path_or_raise(executables, abstract_spec):
"""Ensure that some executables are in path or raise.
Args:
executables (list): list of executables to be searched in the PATH,
in order. The function exits on the first one found.
abstract_spec (str): abstract spec that provides the executables
Raises:
RuntimeError: if the executables cannot be ensured to be in PATH
"""
if spack.util.executable.which_string(*executables):
return
executables_str = ', '.join(executables)
source_configs = spack.config.get('bootstrap:sources', [])
for current_config in source_configs:
if not _source_is_trusted(current_config):
msg = ('[BOOTSTRAP EXECUTABLES {0}] Skipping source "{1}" since it is '
'not trusted').format(executables_str, current_config['name'])
tty.debug(msg)
continue
b = _make_bootstrapper(current_config)
try:
if b.try_search_path(executables, abstract_spec):
return
except Exception as e:
msg = '[BOOTSTRAP EXECUTABLES {0}] Unexpected error "{1}"'
tty.debug(msg.format(executables_str, str(e)))
# We couldn't import in any way, so raise an import error
msg = 'cannot bootstrap any of the {0} executables'.format(executables_str)
if abstract_spec:
msg += ' from spec "{0}"'.format(abstract_spec)
raise RuntimeError(msg)
def _python_import(module):
try:
__import__(module)
@@ -455,7 +592,9 @@ def get_executable(exe, spec=None, install=False):
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in ispec.traverse(root=True, order='post'):
envmod.extend(uenv.environment_modifications_for_spec(dep))
envmod.extend(
spack.user_environment.environment_modifications_for_spec(dep)
)
ret.add_default_envmod(envmod)
return ret
else:
@@ -484,7 +623,9 @@ def _raise_error(executable, exe_spec):
ret = spack.util.executable.Executable(exe_path[0])
envmod = EnvironmentModifications()
for dep in spec.traverse(root=True, order='post'):
envmod.extend(uenv.environment_modifications_for_spec(dep))
envmod.extend(
spack.user_environment.environment_modifications_for_spec(dep)
)
ret.add_default_envmod(envmod)
return ret
@@ -523,8 +664,11 @@ def _add_compilers_if_missing():
def _add_externals_if_missing():
search_list = [
# clingo
spack.repo.path.get('cmake'),
spack.repo.path.get('bison')
spack.repo.path.get('bison'),
# GnuPG
spack.repo.path.get('gawk')
]
detected_packages = spack.detection.by_executable(search_list)
spack.detection.update_configuration(detected_packages, scope='bootstrap')
@@ -600,10 +744,12 @@ def _config_path():
)
def clingo_root_spec():
# Construct the root spec that will be used to bootstrap clingo
spec_str = 'clingo-bootstrap@spack+python'
def _root_spec(spec_str):
"""Add a proper compiler and target to a spec used during bootstrapping.
Args:
spec_str (str): spec to be bootstrapped. Must be without compiler and target.
"""
# Add a proper compiler hint to the root spec. We use GCC for
# everything but MacOS.
if str(spack.platforms.host()) == 'darwin':
@@ -611,17 +757,32 @@ def clingo_root_spec():
else:
spec_str += ' %gcc'
# Add the generic target
generic_target = archspec.cpu.host().family
spec_str += ' target={0}'.format(str(generic_target))
tty.debug('[BOOTSTRAP ROOT SPEC] clingo: {0}'.format(spec_str))
target = archspec.cpu.host().family
spec_str += ' target={0}'.format(target)
tty.debug('[BOOTSTRAP ROOT SPEC] {0}'.format(spec_str))
return spec_str
def clingo_root_spec():
"""Return the root spec used to bootstrap clingo"""
return _root_spec('clingo-bootstrap@spack+python')
def ensure_clingo_importable_or_raise():
"""Ensure that the clingo module is available for import."""
ensure_module_importable_or_raise(
module='clingo', abstract_spec=clingo_root_spec()
)
def gnupg_root_spec():
"""Return the root spec used to bootstrap GnuPG"""
return _root_spec('gnupg@2.3:')
def ensure_gpg_in_path_or_raise():
"""Ensure gpg or gpg2 are in the PATH or raise."""
ensure_executables_in_path_or_raise(
executables=['gpg2', 'gpg'], abstract_spec=gnupg_root_spec(),
)

14
lib/spack/spack/cmd/buildcache.py
View File

@@ -104,6 +104,9 @@ def setup_parser(subparser):
" instead of default platform and OS")
# This argument is needed by the bootstrapping logic to verify checksums
install.add_argument('--sha256', help=argparse.SUPPRESS)
install.add_argument(
'--only-root', action='store_true', help=argparse.SUPPRESS
)
arguments.add_common_arguments(install, ['specs'])
install.set_defaults(func=installtarball)
@@ -534,9 +537,14 @@ def install_tarball(spec, args):
if s.external or s.virtual:
tty.warn("Skipping external or virtual package %s" % spec.format())
return
for d in s.dependencies(deptype=('link', 'run')):
tty.msg("Installing buildcache for dependency spec %s" % d)
install_tarball(d, args)
# This argument is used only for bootstrapping specs without signatures,
# since we need to check the sha256 of each tarball
if not args.only_root:
for d in s.dependencies(deptype=('link', 'run')):
tty.msg("Installing buildcache for dependency spec %s" % d)
install_tarball(d, args)
package = spack.repo.get(spec)
if s.concrete and package.installed and not args.force:
tty.warn("Package for spec %s already installed." % spec.format())

6
lib/spack/spack/modules/common.py
View File

@@ -40,7 +40,7 @@
import llnl.util.tty as tty
from llnl.util.lang import dedupe
import spack.build_environment as build_environment
import spack.build_environment
import spack.config
import spack.environment
import spack.error
@@ -732,12 +732,12 @@ def environment_modifications(self):
# Let the extendee/dependency modify their extensions/dependencies
# before asking for package-specific modifications
env.extend(
build_environment.modifications_from_dependencies(
spack.build_environment.modifications_from_dependencies(
spec, context='run'
)
)
# Package specific modifications
build_environment.set_module_variables_for_package(spec.package)
spack.build_environment.set_module_variables_for_package(spec.package)
spec.package.setup_run_environment(env)
# Modifications required from modules.yaml

7
lib/spack/spack/test/cmd/gpg.py
View File

@@ -9,6 +9,7 @@
import llnl.util.filesystem as fs
import spack.bootstrap
import spack.util.executable
import spack.util.gpg
from spack.main import SpackCommand
@@ -17,6 +18,7 @@
#: spack command used by tests below
gpg = SpackCommand('gpg')
bootstrap = SpackCommand('bootstrap')
# test gpg command detection
@@ -46,9 +48,10 @@ def test_find_gpg(cmd_name, version, tmpdir, mock_gnupghome, monkeypatch):
assert spack.util.gpg.GPGCONF is not None
def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch):
def test_no_gpg_in_path(tmpdir, mock_gnupghome, monkeypatch, mutable_config):
monkeypatch.setitem(os.environ, "PATH", str(tmpdir))
with pytest.raises(spack.util.gpg.SpackGPGError):
bootstrap('disable')
with pytest.raises(RuntimeError):
spack.util.gpg.init(force=True)

6
lib/spack/spack/util/gpg.py
View File

@@ -8,6 +8,7 @@
import os
import re
import spack.bootstrap
import spack.error
import spack.paths
import spack.util.executable
@@ -59,7 +60,10 @@ def init(gnupghome=None, force=False):
spack.paths.gpg_path)
# Set the executable objects for "gpg" and "gpgconf"
GPG, GPGCONF = _gpg(), _gpgconf()
with spack.bootstrap.ensure_bootstrap_configuration():
spack.bootstrap.ensure_gpg_in_path_or_raise()
GPG, GPGCONF = _gpg(), _gpgconf()
GPG.add_default_env('GNUPGHOME', GNUPGHOME)
if GPGCONF:
GPGCONF.add_default_env('GNUPGHOME', GNUPGHOME)