zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add spack --bootstrap option for accessing bootstrap store (#25601)

Browse Source 
We can see what is in the bootstrap store with `spack find -b`, and you can clean it with `spack
clean -b`, but we can't do much else with it, and if there are bootstrap issues they can be hard to
debug.

We already have `spack --mock`, which allows you to swap in the mock packages from the command
line. This PR introduces `spack -b` / `spack --bootstrap`, which runs all of spack with
`ensure_bootstrap_configuration()` set. This means that you can run `spack -b find`, `spack -b
install`, `spack -b spec`, etc. to see what *would* happen with bootstrap configuration, to remove
specific bootstrap packages, etc. This will hopefully make developers' lives easier as they deal
with bootstrap packages.

This PR also uses a `nullcontext` context manager. `nullcontext` has been implemented in several
other places in Spack, and this PR consolidates them to `llnl.util.lang`, with a note that we can
delete the function if we ever reqyire a new enough Python.

- [x] introduce `spack --bootstrap` option
- [x] consolidated all `nullcontext` usages to `llnl.util.lang`
This commit is contained in:
Todd Gamblin 2022-02-22 11:35:34 -08:00 committed by GitHub
parent 800933bbdf
commit 36b0730fac
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 65 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
lib/spack/llnl/util/lang.py
View File

@ -5,6 +5,7 @@
from __future__ import division
import contextlib
import functools
import inspect
import os
@ -921,3 +922,11 @@ def elide_list(line_list, max_num=10):
return line_list[:max_num - 1] + ['...'] + line_list[-1:]
else:
return line_list
@contextlib.contextmanager
def nullcontext(*args, **kwargs):
"""Empty context manager.
TODO: replace with contextlib.nullcontext() if we ever require python 3.7.
"""
yield

12
lib/spack/spack/cmd/spec.py
View File

@ -5,9 +5,9 @@
from __future__ import print_function
import contextlib
import sys
import llnl.util.lang as lang
import llnl.util.tty as tty
import spack
@ -59,14 +59,6 @@ def setup_parser(subparser):
spack.cmd.common.arguments.add_concretizer_args(subparser)
@contextlib.contextmanager
def nullcontext():
"""Empty context manager.
TODO: replace with contextlib.nullcontext() if we ever require python 3.7.
"""
yield
def spec(parser, args):
name_fmt = '{namespace}.{name}' if args.namespaces else '{name}'
fmt = '{@version}{%compiler}{compiler_flags}{variants}{arch=architecture}'
@ -81,7 +73,7 @@ def spec(parser, args):
# use a read transaction if we are getting install status for every
# spec in the DAG. This avoids repeatedly querying the DB.
tree_context = nullcontext
tree_context = lang.nullcontext
if args.install_status:
tree_context = spack.store.db.read_transaction

11
lib/spack/spack/database.py
View File

@ -38,6 +38,7 @@
pass
import llnl.util.filesystem as fs
import llnl.util.lang as lang
import llnl.util.tty as tty
import spack.hash_types as ht
@ -52,12 +53,6 @@
from spack.util.crypto import bit_length
from spack.version import Version
@contextlib.contextmanager
def nullcontext(*args, **kwargs):
yield
# TODO: Provide an API automatically retyring a build after detecting and
# TODO: clearing a failure.
@ -404,8 +399,8 @@ def __init__(self, root, db_dir=None, upstream_dbs=None,
self._write_transaction_impl = lk.WriteTransaction
self._read_transaction_impl = lk.ReadTransaction
else:
self._write_transaction_impl = nullcontext
self._read_transaction_impl = nullcontext
self._write_transaction_impl = lang.nullcontext
self._read_transaction_impl = lang.nullcontext
self._record_fields = record_fields

20
lib/spack/spack/main.py
View File

@ -447,6 +447,9 @@ def make_argument_parser(**kwargs):
parser.add_argument(
'-m', '--mock', action='store_true',
help="use mock packages instead of real ones")
parser.add_argument(
'-b', '--bootstrap', action='store_true',
help="use bootstrap configuration (bootstrap store, config, externals)")
parser.add_argument(
'-p', '--profile', action='store_true', dest='spack_profile',
help="profile execution using cProfile")
@ -856,9 +859,22 @@ def _main(argv=None):
cmd_name = args.command[0]
cmd_name = aliases.get(cmd_name, cmd_name)
command = parser.add_command(cmd_name)
# set up a bootstrap context, if asked.
# bootstrap context needs to include parsing the command, b/c things
# like `ConstraintAction` and `ConfigSetAction` happen at parse time.
bootstrap_context = llnl.util.lang.nullcontext()
if args.bootstrap:
import spack.bootstrap as bootstrap # avoid circular imports
bootstrap_context = bootstrap.ensure_bootstrap_configuration()
# Re-parse with the proper sub-parser added.
with bootstrap_context:
finish_parse_and_run(parser, cmd_name, env_format_error)
def finish_parse_and_run(parser, cmd_name, env_format_error):
"""Finish parsing after we know the command to run."""
# add the found command to the parser and re-run then re-parse
command = parser.add_command(cmd_name)
args, unknown = parser.parse_known_args()
# Now that we know what command this is and what its args are, determine

70
lib/spack/spack/test/llnl/util/tty/log.py
View File

@ -18,10 +18,9 @@
import pytest
import llnl.util.tty.log
from llnl.util.lang import uniq
from llnl.util.tty.log import log_output
from llnl.util.tty.pty import PseudoShell
import llnl.util.tty.log as log
import llnl.util.lang as lang
import llnl.util.tty.pty as pty
from spack.util.executable import which
@ -33,14 +32,9 @@
pass
@contextlib.contextmanager
def nullcontext():
yield
def test_log_python_output_with_echo(capfd, tmpdir):
with tmpdir.as_cwd():
with log_output('foo.txt', echo=True):
with log.log_output('foo.txt', echo=True):
print('logged')
# foo.txt has output
@ -53,7 +47,7 @@ def test_log_python_output_with_echo(capfd, tmpdir):
def test_log_python_output_without_echo(capfd, tmpdir):
with tmpdir.as_cwd():
with log_output('foo.txt'):
with log.log_output('foo.txt'):
print('logged')
# foo.txt has output
@ -66,7 +60,7 @@ def test_log_python_output_without_echo(capfd, tmpdir):
def test_log_python_output_with_invalid_utf8(capfd, tmpdir):
with tmpdir.as_cwd():
with log_output('foo.txt'):
with log.log_output('foo.txt'):
sys.stdout.buffer.write(b'\xc3\x28\n')
# python2 and 3 treat invalid UTF-8 differently
@ -85,7 +79,7 @@ def test_log_python_output_with_invalid_utf8(capfd, tmpdir):
def test_log_python_output_and_echo_output(capfd, tmpdir):
with tmpdir.as_cwd():
# echo two lines
with log_output('foo.txt') as logger:
with log.log_output('foo.txt') as logger:
with logger.force_echo():
print('force echo')
print('logged')
@ -104,7 +98,7 @@ def _log_filter_fn(string):
def test_log_output_with_filter(capfd, tmpdir):
with tmpdir.as_cwd():
with log_output('foo.txt', filter_fn=_log_filter_fn):
with log.log_output('foo.txt', filter_fn=_log_filter_fn):
print('foo blah')
print('blah foo')
print('foo foo')
@ -118,7 +112,7 @@ def test_log_output_with_filter(capfd, tmpdir):
# now try with echo
with tmpdir.as_cwd():
with log_output('foo.txt', echo=True, filter_fn=_log_filter_fn):
with log.log_output('foo.txt', echo=True, filter_fn=_log_filter_fn):
print('foo blah')
print('blah foo')
print('foo foo')
@ -140,7 +134,7 @@ def test_log_subproc_and_echo_output_no_capfd(capfd, tmpdir):
# here, and echoing in test_log_subproc_and_echo_output_capfd below.
with capfd.disabled():
with tmpdir.as_cwd():
with log_output('foo.txt') as logger:
with log.log_output('foo.txt') as logger:
with logger.force_echo():
echo('echo')
print('logged')
@ -157,7 +151,7 @@ def test_log_subproc_and_echo_output_capfd(capfd, tmpdir):
# interferes with the logged data. See
# test_log_subproc_and_echo_output_no_capfd for tests on the logfile.
with tmpdir.as_cwd():
with log_output('foo.txt') as logger:
with log.log_output('foo.txt') as logger:
with logger.force_echo():
echo('echo')
print('logged')
@ -177,7 +171,7 @@ def handler(signum, frame):
signal.signal(signal.SIGUSR1, handler)
log_path = kwargs["log_path"]
with log_output(log_path):
with log.log_output(log_path):
while running[0]:
print("line")
time.sleep(1e-3)
@ -306,25 +300,25 @@ def mock_shell_fg_bg_no_termios(proc, ctl, **kwargs):
@contextlib.contextmanager
def no_termios():
saved = llnl.util.tty.log.termios
llnl.util.tty.log.termios = None
saved = log.termios
log.termios = None
try:
yield
finally:
llnl.util.tty.log.termios = saved
log.termios = saved
@pytest.mark.skipif(not which("ps"), reason="requires ps utility")
@pytest.mark.skipif(not termios, reason="requires termios support")
@pytest.mark.parametrize('test_fn,termios_on_or_off', [
# tests with termios
(mock_shell_fg, nullcontext),
(mock_shell_bg, nullcontext),
(mock_shell_bg_fg, nullcontext),
(mock_shell_fg_bg, nullcontext),
(mock_shell_tstp_cont, nullcontext),
(mock_shell_tstp_tstp_cont, nullcontext),
(mock_shell_tstp_tstp_cont_cont, nullcontext),
(mock_shell_fg, lang.nullcontext),
(mock_shell_bg, lang.nullcontext),
(mock_shell_bg_fg, lang.nullcontext),
(mock_shell_fg_bg, lang.nullcontext),
(mock_shell_tstp_cont, lang.nullcontext),
(mock_shell_tstp_tstp_cont, lang.nullcontext),
(mock_shell_tstp_tstp_cont_cont, lang.nullcontext),
# tests without termios
(mock_shell_fg_no_termios, no_termios),
(mock_shell_bg, no_termios),
@ -342,7 +336,7 @@ def test_foreground_background(test_fn, termios_on_or_off, tmpdir):
process stop and start.
"""
shell = PseudoShell(test_fn, simple_logger)
shell = pty.PseudoShell(test_fn, simple_logger)
log_path = str(tmpdir.join("log.txt"))
# run the shell test
@ -375,7 +369,7 @@ def handler(signum, frame):
v_lock = kwargs["v_lock"]
sys.stderr.write(os.getcwd() + "\n")
with log_output(log_path) as logger:
with log.log_output(log_path) as logger:
with logger.force_echo():
print("forced output")
@ -446,7 +440,7 @@ def mock_shell_v_v_no_termios(proc, ctl, **kwargs):
@pytest.mark.skipif(not which("ps"), reason="requires ps utility")
@pytest.mark.skipif(not termios, reason="requires termios support")
@pytest.mark.parametrize('test_fn,termios_on_or_off', [
(mock_shell_v_v, nullcontext),
(mock_shell_v_v, lang.nullcontext),
(mock_shell_v_v_no_termios, no_termios),
])
def test_foreground_background_output(
@ -457,7 +451,7 @@ def test_foreground_background_output(
return
shell = PseudoShell(test_fn, synchronized_logger)
shell = pty.PseudoShell(test_fn, synchronized_logger)
log_path = str(tmpdir.join("log.txt"))
# Locks for synchronizing with minion
@ -485,8 +479,8 @@ def test_foreground_background_output(
# also get lines of log file
assert os.path.exists(log_path)
with open(log_path) as log:
log = log.read().strip().split("\n")
with open(log_path) as logfile:
log_data = logfile.read().strip().split("\n")
# Controller and minion process coordinate with locks such that the
# minion writes "off" when echo is off, and "on" when echo is on. The
@ -494,12 +488,12 @@ def test_foreground_background_output(
# lines if the controller is slow. The important thing to observe
# here is that we started seeing 'on' in the end.
assert (
['forced output', 'on'] == uniq(output) or
['forced output', 'off', 'on'] == uniq(output)
['forced output', 'on'] == lang.uniq(output) or
['forced output', 'off', 'on'] == lang.uniq(output)
)
# log should be off for a while, then on, then off
assert (
['forced output', 'off', 'on', 'off'] == uniq(log) and
log.count("off") > 2 # ensure some "off" lines were omitted
['forced output', 'off', 'on', 'off'] == lang.uniq(log_data) and
log_data.count("off") > 2 # ensure some "off" lines were omitted
)

2
share/spack/spack-completion.bash
View File

@ -335,7 +335,7 @@ _spacktivate() {
_spack() {
if $list_options
then
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --show-cores --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
SPACK_COMPREPLY="-h --help -H --all-help --color -c --config -C --config-scope -d --debug --show-cores --timestamp --pdb -e --env -D --env-dir -E --no-env --use-env-repo -k --insecure -l --enable-locks -L --disable-locks -m --mock -b --bootstrap -p --profile --sorted-profile --lines -v --verbose --stacktrace -V --version --print-shell-vars"
else
SPACK_COMPREPLY="activate add analyze arch audit blame bootstrap build-env buildcache cd checksum ci clean clone commands compiler compilers concretize config containerize create deactivate debug dependencies dependents deprecate dev-build develop diff docs edit env extensions external fetch find gc gpg graph help info install license list load location log-parse maintainers mark mirror module monitor patch pkg providers pydoc python reindex remove rm repo resource restage solve spec stage style tags test test-env tutorial undevelop uninstall unit-test unload url verify versions view"
fi