zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Isolate bootstrap configuration from user configuration (#26071)

Browse Source 
* Isolate bootstrap configuration from user configuration

* Search for build dependencies automatically if bootstrapping from sources

The bootstrapping logic will search for build dependencies
automatically if bootstrapping anything form sources. Any
external spec, if found, is written in a scope that is specific
to bootstrapping.

* Don't clean the bootstrap store with "spack clean -a"

* Copy bootstrap.yaml and config.yaml in the bootstrap area
This commit is contained in:
Massimiliano Culpo 2021-10-05 09:16:09 +02:00 committed by GitHub
parent 3cf426df99
commit 337b54fab0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 96 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
lib/spack/docs/getting_started.rst
View File

@ -197,17 +197,6 @@ Spack will build the required software on the first request to concretize a spec
[ ... ]
zlib@1.2.11%gcc@10.1.0+optimize+pic+shared arch=linux-ubuntu18.04-broadwell
.. tip::
If you want to speed-up bootstrapping ``clingo`` from sources, you may try to
search for ``cmake`` and ``bison`` on your system:
.. code-block:: console
$ spack external find cmake bison
==> The following specs have been detected on this system and added to /home/spack/.spack/packages.yaml
bison@3.0.4 cmake@3.19.4
"""""""""""""""""""
The Bootstrap Store
"""""""""""""""""""

64
lib/spack/spack/bootstrap.py
View File

@ -26,6 +26,7 @@
import spack.architecture
import spack.binary_distribution
import spack.config
import spack.detection
import spack.environment
import spack.main
import spack.modules
@ -209,7 +210,7 @@ def try_import(self, module, abstract_spec_str):
buildcache = spack.main.SpackCommand('buildcache')
# Ensure we see only the buildcache being used to bootstrap
mirror_scope = spack.config.InternalConfigScope(
'bootstrap', {'mirrors:': {self.name: self.url}}
'bootstrap_buildcache', {'mirrors:': {self.name: self.url}}
)
with spack.config.override(mirror_scope):
# This index is currently needed to get the compiler used to build some
@ -218,7 +219,7 @@ def try_import(self, module, abstract_spec_str):
index = spack.binary_distribution.update_cache_and_get_specs()
if not index:
raise RuntimeError("Could not populate the binary index")
raise RuntimeError("could not populate the binary index")
for item in data['verified']:
candidate_spec = item['spec']
@ -279,6 +280,10 @@ def try_import(module, abstract_spec_str):
tty.info("Bootstrapping {0} from sources".format(module))
# If we compile code from sources detecting a few build tools
# might reduce compilation time by a fair amount
_add_externals_if_missing()
# Try to build and install from sources
with spack_python_interpreter():
# Add hint to use frontend operating system on Cray
@ -492,7 +497,11 @@ def _bootstrap_config_scopes():
config_scopes = [
spack.config.InternalConfigScope('_builtin', spack.config.config_defaults)
]
for name, path in spack.config.configuration_paths:
configuration_paths = (
spack.config.configuration_defaults_path,
('bootstrap', _config_path())
)
for name, path in configuration_paths:
platform = spack.architecture.platform().name
platform_scope = spack.config.ConfigScope(
'/'.join([name, platform]), os.path.join(path, platform)
@ -517,9 +526,19 @@ def _add_compilers_if_missing():
spack.compilers.add_compilers_to_config(new_compilers, init_config=False)
def _add_externals_if_missing():
search_list = [
spack.repo.path.get('cmake'),
spack.repo.path.get('bison')
]
detected_packages = spack.detection.by_executable(search_list)
spack.detection.update_configuration(detected_packages, scope='bootstrap')
@contextlib.contextmanager
def ensure_bootstrap_configuration():
bootstrap_store_path = store_path()
user_configuration = _read_and_sanitize_configuration()
with spack.environment.deactivate_environment():
with spack.architecture.use_platform(spack.architecture.real_platform()):
with spack.repo.use_repositories(spack.paths.packages_path):
@ -531,11 +550,29 @@ def ensure_bootstrap_configuration():
# We may need to compile code from sources, so ensure we have
# compilers for the current platform before switching parts.
_add_compilers_if_missing()
spack.config.set('bootstrap', user_configuration['bootstrap'])
spack.config.set('config', user_configuration['config'])
with spack.modules.disable_modules():
with spack_python_interpreter():
yield
def _read_and_sanitize_configuration():
"""Read the user configuration that needs to be reused for bootstrapping
and remove the entries that should not be copied over.
"""
# Read the "config" section but pop the install tree (the entry will not be
# considered due to the use_store context manager, so it will be confusing
# to have it in the configuration).
config_yaml = spack.config.get('config')
config_yaml.pop('install_tree', None)
user_configuration = {
'bootstrap': spack.config.get('bootstrap'),
'config': config_yaml
}
return user_configuration
def store_path():
"""Path to the store used for bootstrapped software"""
enabled = spack.config.get('bootstrap:enable', True)
@ -544,13 +581,28 @@ def store_path():
'Use "spack bootstrap enable" to enable it')
raise RuntimeError(msg)
bootstrap_root_path = spack.config.get(
return _store_path()
def _root_path():
"""Root of all the bootstrap related folders"""
return spack.config.get(
'bootstrap:root', spack.paths.user_bootstrap_path
)
bootstrap_store_path = spack.util.path.canonicalize_path(
def _store_path():
bootstrap_root_path = _root_path()
return spack.util.path.canonicalize_path(
os.path.join(bootstrap_root_path, 'store')
)
return bootstrap_store_path
def _config_path():
bootstrap_root_path = _root_path()
return spack.util.path.canonicalize_path(
os.path.join(bootstrap_root_path, 'config')
)
def clingo_root_spec():

22
lib/spack/spack/cmd/clean.py
View File

@ -7,6 +7,7 @@
import os
import shutil
import llnl.util.filesystem
import llnl.util.tty as tty
import spack.bootstrap
@ -14,9 +15,9 @@
import spack.cmd.common.arguments as arguments
import spack.cmd.test
import spack.config
import spack.main
import spack.repo
import spack.stage
import spack.util.path
from spack.paths import lib_path, var_path
description = "remove temporary build files and/or downloaded archives"
@ -27,7 +28,7 @@
class AllClean(argparse.Action):
"""Activates flags -s -d -f -m and -p simultaneously"""
def __call__(self, parser, namespace, values, option_string=None):
parser.parse_args(['-sdfmpb'], namespace=namespace)
parser.parse_args(['-sdfmp'], namespace=namespace)
def setup_parser(subparser):
@ -48,9 +49,11 @@ def setup_parser(subparser):
help="remove .pyc, .pyo files and __pycache__ folders")
subparser.add_argument(
'-b', '--bootstrap', action='store_true',
help="remove software needed to bootstrap Spack")
help="remove software and configuration needed to bootstrap Spack")
subparser.add_argument(
'-a', '--all', action=AllClean, help="equivalent to -sdfmpb", nargs=0
'-a', '--all', action=AllClean,
help="equivalent to -sdfmp (does not include --bootstrap)",
nargs=0
)
arguments.add_common_arguments(subparser, ['specs'])
@ -102,8 +105,9 @@ def clean(parser, args):
shutil.rmtree(dname)
if args.bootstrap:
msg = 'Removing software in "{0}"'
tty.msg(msg.format(spack.bootstrap.store_path()))
with spack.bootstrap.ensure_bootstrap_configuration():
uninstall = spack.main.SpackCommand('uninstall')
uninstall('-a', '-y')
bootstrap_prefix = spack.util.path.canonicalize_path(
spack.config.get('bootstrap:root')
)
msg = 'Removing bootstrapped software and configuration in "{0}"'
tty.msg(msg.format(bootstrap_prefix))
llnl.util.filesystem.remove_directory_contents(bootstrap_prefix)

7
lib/spack/spack/config.py
View File

@ -84,11 +84,16 @@
all_schemas.update(dict((key, spack.schema.env.schema)
for key in spack.schema.env.keys))
#: Path to the default configuration
configuration_defaults_path = (
'defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')
)
#: Builtin paths to configuration files in Spack
configuration_paths = (
# Default configuration scope is the lowest-level scope. These are
# versioned with Spack and can be overridden by systems, sites or users
('defaults', os.path.join(spack.paths.etc_path, 'spack', 'defaults')),
configuration_defaults_path,
# System configuration is per machine.
# No system-level configs should be checked into spack by default

19
lib/spack/spack/test/bootstrap.py
View File

@ -99,3 +99,22 @@ def test_bootstrap_search_for_compilers_with_environment_active(
with spack.bootstrap.ensure_bootstrap_configuration():
assert spack.compilers.all_compiler_specs(init_config=False)
assert not spack.compilers.all_compiler_specs(init_config=False)
@pytest.mark.regression('26189')
def test_config_yaml_is_preserved_during_bootstrap(mutable_config):
# Mock the command line scope
expected_dir = '/tmp/test'
internal_scope = spack.config.InternalConfigScope(
name='command_line', data={
'config': {
'test_stage': expected_dir
}
}
)
spack.config.config.push_scope(internal_scope)
assert spack.config.get('config:test_stage') == expected_dir
with spack.bootstrap.ensure_bootstrap_configuration():
assert spack.config.get('config:test_stage') == expected_dir
assert spack.config.get('config:test_stage') == expected_dir