zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spack --bootstrap option for accessing bootstrap store (#25601)

Browse Source 
We can see what is in the bootstrap store with `spack find -b`, and you can clean it with `spack
clean -b`, but we can't do much else with it, and if there are bootstrap issues they can be hard to
debug.

We already have `spack --mock`, which allows you to swap in the mock packages from the command
line. This PR introduces `spack -b` / `spack --bootstrap`, which runs all of spack with
`ensure_bootstrap_configuration()` set. This means that you can run `spack -b find`, `spack -b
install`, `spack -b spec`, etc. to see what *would* happen with bootstrap configuration, to remove
specific bootstrap packages, etc. This will hopefully make developers' lives easier as they deal
with bootstrap packages.

This PR also uses a `nullcontext` context manager. `nullcontext` has been implemented in several
other places in Spack, and this PR consolidates them to `llnl.util.lang`, with a note that we can
delete the function if we ever reqyire a new enough Python.

- [x] introduce `spack --bootstrap` option
- [x] consolidated all `nullcontext` usages to `llnl.util.lang`
This commit is contained in:
Todd Gamblin 2022-02-22 11:35:34 -08:00 committed by GitHub
parent 800933bbdf
commit 36b0730fac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 65 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/spack/llnl/util/lang.py
View File

@ -5,6 +5,7 @@
from __future__ import division from __future__ import division
import contextlib
import functools import functools
import inspect import inspect
import os import os
@ -921,3 +922,11 @@ def elide_list(line_list, max_num=10):
return line_list[:max_num - 1] + ['...'] + line_list[-1:] return line_list[:max_num - 1] + ['...'] + line_list[-1:]
else: else:
return line_list return line_list
@contextlib.contextmanager
def nullcontext(*args, **kwargs):
"""Empty context manager.
TODO: replace with contextlib.nullcontext() if we ever require python 3.7.
"""
yield

12
lib/spack/spack/cmd/spec.py
View File

@ -5,9 +5,9 @@
from __future__ import print_function from __future__ import print_function
import contextlib
import sys import sys
import llnl.util.lang as lang
import llnl.util.tty as tty import llnl.util.tty as tty
import spack import spack
@ -59,14 +59,6 @@ def setup_parser(subparser):
spack.cmd.common.arguments.add_concretizer_args(subparser) spack.cmd.common.arguments.add_concretizer_args(subparser)
@contextlib.contextmanager
def nullcontext():
"""Empty context manager.
TODO: replace with contextlib.nullcontext() if we ever require python 3.7.
"""
yield
def spec(parser, args): def spec(parser, args):
name_fmt = '{namespace}.{name}' if args.namespaces else '{name}' name_fmt = '{namespace}.{name}' if args.namespaces else '{name}'
fmt = '{@version}{%compiler}{compiler_flags}{variants}{arch=architecture}' fmt = '{@version}{%compiler}{compiler_flags}{variants}{arch=architecture}'
@ -81,7 +73,7 @@ def spec(parser, args):
# use a read transaction if we are getting install status for every # use a read transaction if we are getting install status for every
# spec in the DAG. This avoids repeatedly querying the DB. # spec in the DAG. This avoids repeatedly querying the DB.
tree_context = nullcontext tree_context = lang.nullcontext
if args.install_status: if args.install_status:
tree_context = spack.store.db.read_transaction tree_context = spack.store.db.read_transaction

11
lib/spack/spack/database.py
View File

@ -38,6 +38,7 @@
pass pass
import llnl.util.filesystem as fs import llnl.util.filesystem as fs
import llnl.util.lang as lang
import llnl.util.tty as tty import llnl.util.tty as tty
import spack.hash_types as ht import spack.hash_types as ht
@ -52,12 +53,6 @@
from spack.util.crypto import bit_length from spack.util.crypto import bit_length
from spack.version import Version from spack.version import Version
@contextlib.contextmanager
def nullcontext(*args, **kwargs):
yield
# TODO: Provide an API automatically retyring a build after detecting and # TODO: Provide an API automatically retyring a build after detecting and
# TODO: clearing a failure. # TODO: clearing a failure.
@ -404,8 +399,8 @@ def __init__(self, root, db_dir=None, upstream_dbs=None,
self._write_transaction_impl = lk.WriteTransaction self._write_transaction_impl = lk.WriteTransaction
self._read_transaction_impl = lk.ReadTransaction self._read_transaction_impl = lk.ReadTransaction
else: else:
self._write_transaction_impl = nullcontext self._write_transaction_impl = lang.nullcontext
self._read_transaction_impl = nullcontext self._read_transaction_impl = lang.nullcontext
self._record_fields = record_fields self._record_fields = record_fields

20
lib/spack/spack/main.py
View File

@ -447,6 +447,9 @@ def make_argument_parser(**kwargs):
parser.add_argument( parser.add_argument(
'-m', '--mock', action='store_true', '-m', '--mock', action='store_true',
help="use mock packages instead of real ones") help="use mock packages instead of real ones")
parser.add_argument(
'-b', '--bootstrap', action='store_true',
help="use bootstrap configuration (bootstrap store, config, externals)")
parser.add_argument( parser.add_argument(
'-p', '--profile', action='store_true', dest='spack_profile', '-p', '--profile', action='store_true', dest='spack_profile',
help="profile execution using cProfile") help="profile execution using cProfile")
@ -856,9 +859,22 @@ def _main(argv=None):
cmd_name = args.command[0] cmd_name = args.command[0]
cmd_name = aliases.get(cmd_name, cmd_name) cmd_name = aliases.get(cmd_name, cmd_name)
command = parser.add_command(cmd_name) # set up a bootstrap context, if asked.
# bootstrap context needs to include parsing the command, b/c things
# like `ConstraintAction` and `ConfigSetAction` happen at parse time.
bootstrap_context = llnl.util.lang.nullcontext()
if args.bootstrap:
import spack.bootstrap as bootstrap # avoid circular imports
bootstrap_context = bootstrap.ensure_bootstrap_configuration()
# Re-parse with the proper sub-parser added. with bootstrap_context:
finish_parse_and_run(parser, cmd_name, env_format_error)
def finish_parse_and_run(parser, cmd_name, env_format_error):
"""Finish parsing after we know the command to run."""
# add the found command to the parser and re-run then re-parse
command = parser.add_command(cmd_name)
args, unknown = parser.parse_known_args() args, unknown = parser.parse_known_args()
# Now that we know what command this is and what its args are, determine # Now that we know what command this is and what its args are, determine

70
lib/spack/spack/test/llnl/util/tty/log.py
View File

@ -18,10 +18,9 @@
import pytest import pytest
import llnl.util.tty.log import llnl.util.tty.log as log
from llnl.util.lang import uniq import llnl.util.lang as lang
from llnl.util.tty.log import log_output import llnl.util.tty.pty as pty
from llnl.util.tty.pty import PseudoShell
from spack.util.executable import which from spack.util.executable import which
@ -33,14 +32,9 @@
pass pass
@contextlib.contextmanager
def nullcontext():
yield
def test_log_python_output_with_echo(capfd, tmpdir): def test_log_python_output_with_echo(capfd, tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt', echo=True): with log.log_output('foo.txt', echo=True):
print('logged') print('logged')
# foo.txt has output # foo.txt has output
@ -53,7 +47,7 @@ def test_log_python_output_with_echo(capfd, tmpdir):
def test_log_python_output_without_echo(capfd, tmpdir): def test_log_python_output_without_echo(capfd, tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt'): with log.log_output('foo.txt'):
print('logged') print('logged')
# foo.txt has output # foo.txt has output
@ -66,7 +60,7 @@ def test_log_python_output_without_echo(capfd, tmpdir):
def test_log_python_output_with_invalid_utf8(capfd, tmpdir): def test_log_python_output_with_invalid_utf8(capfd, tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt'): with log.log_output('foo.txt'):
sys.stdout.buffer.write(b'\xc3\x28\n') sys.stdout.buffer.write(b'\xc3\x28\n')
# python2 and 3 treat invalid UTF-8 differently # python2 and 3 treat invalid UTF-8 differently
@ -85,7 +79,7 @@ def test_log_python_output_with_invalid_utf8(capfd, tmpdir):
def test_log_python_output_and_echo_output(capfd, tmpdir): def test_log_python_output_and_echo_output(capfd, tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
# echo two lines # echo two lines
with log_output('foo.txt') as logger: with log.log_output('foo.txt') as logger:
with logger.force_echo(): with logger.force_echo():
print('force echo') print('force echo')
print('logged') print('logged')
@ -104,7 +98,7 @@ def _log_filter_fn(string):
def test_log_output_with_filter(capfd, tmpdir): def test_log_output_with_filter(capfd, tmpdir):
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt', filter_fn=_log_filter_fn): with log.log_output('foo.txt', filter_fn=_log_filter_fn):
print('foo blah') print('foo blah')
print('blah foo') print('blah foo')
print('foo foo') print('foo foo')
@ -118,7 +112,7 @@ def test_log_output_with_filter(capfd, tmpdir):
# now try with echo # now try with echo
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt', echo=True, filter_fn=_log_filter_fn): with log.log_output('foo.txt', echo=True, filter_fn=_log_filter_fn):
print('foo blah') print('foo blah')
print('blah foo') print('blah foo')
print('foo foo') print('foo foo')
@ -140,7 +134,7 @@ def test_log_subproc_and_echo_output_no_capfd(capfd, tmpdir):
# here, and echoing in test_log_subproc_and_echo_output_capfd below. # here, and echoing in test_log_subproc_and_echo_output_capfd below.
with capfd.disabled(): with capfd.disabled():
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt') as logger: with log.log_output('foo.txt') as logger:
with logger.force_echo(): with logger.force_echo():
echo('echo') echo('echo')
print('logged') print('logged')
@ -157,7 +151,7 @@ def test_log_subproc_and_echo_output_capfd(capfd, tmpdir):
# interferes with the logged data. See # interferes with the logged data. See
# test_log_subproc_and_echo_output_no_capfd for tests on the logfile. # test_log_subproc_and_echo_output_no_capfd for tests on the logfile.
with tmpdir.as_cwd(): with tmpdir.as_cwd():
with log_output('foo.txt') as logger: with log.log_output('foo.txt') as logger:
with logger.force_echo(): with logger.force_echo():
echo('echo') echo('echo')
print('logged') print('logged')
@ -177,7 +171,7 @@ def handler(signum, frame):
signal.signal(signal.SIGUSR1, handler) signal.signal(signal.SIGUSR1, handler)
log_path = kwargs["log_path"] log_path = kwargs["log_path"]
with log_output(log_path): with log.log_output(log_path):
while running[0]: while running[0]:
print("line") print("line")
time.sleep(1e-3) time.sleep(1e-3)
@ -306,25 +300,25 @@ def mock_shell_fg_bg_no_termios(proc, ctl, **kwargs):
@contextlib.contextmanager @contextlib.contextmanager
def no_termios(): def no_termios():
saved = llnl.util.tty.log.termios saved = log.termios
llnl.util.tty.log.termios = None log.termios = None
try: try:
yield yield
finally: finally:
llnl.util.tty.log.termios = saved log.termios = saved
@pytest.mark.skipif(not which("ps"), reason="requires ps utility") @pytest.mark.skipif(not which("ps"), reason="requires ps utility")
@pytest.mark.skipif(not termios, reason="requires termios support") @pytest.mark.skipif(not termios, reason="requires termios support")
@pytest.mark.parametrize('test_fn,termios_on_or_off', [ @pytest.mark.parametrize('test_fn,termios_on_or_off', [
# tests with termios # tests with termios
(mock_shell_fg, nullcontext), (mock_shell_fg, lang.nullcontext),
(mock_shell_bg, nullcontext), (mock_shell_bg, lang.nullcontext),
(mock_shell_bg_fg, nullcontext), (mock_shell_bg_fg, lang.nullcontext),
(mock_shell_fg_bg, nullcontext), (mock_shell_fg_bg, lang.nullcontext),
(mock_shell_tstp_cont, nullcontext), (mock_shell_tstp_cont, lang.nullcontext),
(mock_shell_tstp_tstp_cont, nullcontext), (mock_shell_tstp_tstp_cont, lang.nullcontext),
(mock_shell_tstp_tstp_cont_cont, nullcontext), (mock_shell_tstp_tstp_cont_cont, lang.nullcontext),
# tests without termios # tests without termios
(mock_shell_fg_no_termios, no_termios), (mock_shell_fg_no_termios, no_termios),
(mock_shell_bg, no_termios), (mock_shell_bg, no_termios),
@ -342,7 +336,7 @@ def test_foreground_background(test_fn, termios_on_or_off, tmpdir):
process stop and start. process stop and start.
""" """
shell = PseudoShell(test_fn, simple_logger) shell = pty.PseudoShell(test_fn, simple_logger)
log_path = str(tmpdir.join("log.txt")) log_path = str(tmpdir.join("log.txt"))
# run the shell test # run the shell test
@ -375,7 +369,7 @@ def handler(signum, frame):
v_lock = kwargs["v_lock"] v_lock = kwargs["v_lock"]
sys.stderr.write(os.getcwd() + "\n") sys.stderr.write(os.getcwd() + "\n")
with log_output(log_path) as logger: with log.log_output(log_path) as logger:
with logger.force_echo(): with logger.force_echo():
print("forced output") print("forced output")
@ -446,7 +440,7 @@ def mock_shell_v_v_no_termios(proc, ctl, **kwargs):
@pytest.mark.skipif(not which("ps"), reason="requires ps utility") @pytest.mark.skipif(not which("ps"), reason="requires ps utility")
@pytest.mark.skipif(not termios, reason="requires termios support") @pytest.mark.skipif(not termios, reason="requires termios support")
@pytest.mark.parametrize('test_fn,termios_on_or_off', [ @pytest.mark.parametrize('test_fn,termios_on_or_off', [
(mock_shell_v_v, nullcontext), (mock_shell_v_v, lang.nullcontext),
(mock_shell_v_v_no_termios, no_termios), (mock_shell_v_v_no_termios, no_termios),
]) ])
def test_foreground_background_output( def test_foreground_background_output(
@ -457,7 +451,7 @@ def test_foreground_background_output(
return return
shell = PseudoShell(test_fn, synchronized_logger) shell = pty.PseudoShell(test_fn, synchronized_logger)
log_path = str(tmpdir.join("log.txt")) log_path = str(tmpdir.join("log.txt"))
# Locks for synchronizing with minion # Locks for synchronizing with minion
@ -485,8 +479,8 @@ def test_foreground_background_output(
# also get lines of log file # also get lines of log file
assert os.path.exists(log_path) assert os.path.exists(log_path)
with open(log_path) as log: with open(log_path) as logfile:
log = log.read().strip().split("\n") log_data = logfile.read().strip().split("\n")
# Controller and minion process coordinate with locks such that the # Controller and minion process coordinate with locks such that the
# minion writes "off" when echo is off, and "on" when echo is on. The # minion writes "off" when echo is off, and "on" when echo is on. The
@ -494,12 +488,12 @@ def test_foreground_background_output(
# lines if the controller is slow. The important thing to observe # lines if the controller is slow. The important thing to observe
# here is that we started seeing 'on' in the end. # here is that we started seeing 'on' in the end.
assert ( assert (
['forced output', 'on'] == uniq(output) or ['forced output', 'on'] == lang.uniq(output) or
['forced output', 'off', 'on'] == uniq(output) ['forced output', 'off', 'on'] == lang.uniq(output)
) )
# log should be off for a while, then on, then off # log should be off for a while, then on, then off
assert ( assert (
['forced output', 'off', 'on', 'off'] == uniq(log) and ['forced output', 'off', 'on', 'off'] == lang.uniq(log_data) and
log.count("off") > 2 # ensure some "off" lines were omitted log_data.count("off") > 2 # ensure some "off" lines were omitted
) )

2
share/spack/spack-completion.bash
View File

@ -335,7 +335,7 @@ _spacktivate() {
_spack() { _spack() {
if $list_options if $list_options
then then
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --show-cores --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars" SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --show-cores --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
else else
SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view" SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
fi fi