2022-01-12 11:21:41 -08:00
|
|
|
# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
|
2018-10-07 13:52:23 -07:00
|
|
|
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
2014-01-12 18:19:18 +01:00
|
|
|
#
|
2018-10-07 13:52:23 -07:00
|
|
|
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
|
|
|
|
2019-06-07 18:57:26 +02:00
|
|
|
from __future__ import unicode_literals
|
|
|
|
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
import contextlib
|
2017-05-01 22:08:47 +02:00
|
|
|
import os
|
2014-11-23 17:53:21 -06:00
|
|
|
import struct
|
2017-05-01 22:08:47 +02:00
|
|
|
import sys
|
|
|
|
import textwrap
|
2016-11-15 15:17:03 -05:00
|
|
|
import traceback
|
2019-06-07 18:57:26 +02:00
|
|
|
from datetime import datetime
|
2021-04-01 15:34:56 -04:00
|
|
|
from sys import platform as _platform
|
2021-07-08 15:12:30 -07:00
|
|
|
|
|
|
|
import six
|
2017-03-07 14:25:48 -08:00
|
|
|
from six import StringIO
|
2017-04-23 10:32:08 -05:00
|
|
|
from six.moves import input
|
2021-10-21 16:30:16 -04:00
|
|
|
|
|
|
|
if _platform != "win32":
|
|
|
|
import fcntl
|
|
|
|
import termios
|
2014-04-14 12:53:23 -07:00
|
|
|
|
2021-07-08 15:12:30 -07:00
|
|
|
from llnl.util.tty.color import cescape, clen, cprint, cwrite
|
2013-02-13 17:50:44 -08:00
|
|
|
|
2020-07-23 00:49:57 -07:00
|
|
|
# Globals
|
|
|
|
_debug = 0
|
2014-06-19 08:47:06 -07:00
|
|
|
_verbose = False
|
2016-11-15 15:17:03 -05:00
|
|
|
_stacktrace = False
|
2019-02-13 10:14:35 -08:00
|
|
|
_timestamp = False
|
|
|
|
_msg_enabled = True
|
|
|
|
_warn_enabled = True
|
|
|
|
_error_enabled = True
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
_output_filter = lambda s: s
|
2017-05-01 22:08:47 +02:00
|
|
|
indent = " "
|
2013-02-13 17:50:44 -08:00
|
|
|
|
2016-08-09 13:23:53 -07:00
|
|
|
|
2020-07-23 00:49:57 -07:00
|
|
|
def debug_level():
|
|
|
|
return _debug
|
|
|
|
|
|
|
|
|
2015-05-29 17:18:33 -07:00
|
|
|
def is_verbose():
|
|
|
|
return _verbose
|
|
|
|
|
|
|
|
|
2020-07-23 00:49:57 -07:00
|
|
|
def is_debug(level=1):
|
|
|
|
return _debug >= level
|
2015-05-29 17:18:33 -07:00
|
|
|
|
|
|
|
|
2016-11-15 15:17:03 -05:00
|
|
|
def is_stacktrace():
|
|
|
|
return _stacktrace
|
|
|
|
|
|
|
|
|
2020-07-23 00:49:57 -07:00
|
|
|
def set_debug(level=0):
|
2014-06-19 08:47:06 -07:00
|
|
|
global _debug
|
2022-07-30 15:19:18 -07:00
|
|
|
assert level >= 0, "Debug level must be a positive value"
|
2020-07-23 00:49:57 -07:00
|
|
|
_debug = level
|
2014-06-19 08:47:06 -07:00
|
|
|
|
|
|
|
|
|
|
|
def set_verbose(flag):
|
|
|
|
global _verbose
|
|
|
|
_verbose = flag
|
2017-02-17 15:45:02 -06:00
|
|
|
|
2016-11-15 15:17:03 -05:00
|
|
|
|
2019-02-13 10:14:35 -08:00
|
|
|
def set_timestamp(flag):
|
|
|
|
global _timestamp
|
|
|
|
_timestamp = flag
|
|
|
|
|
|
|
|
|
|
|
|
def set_msg_enabled(flag):
|
|
|
|
global _msg_enabled
|
|
|
|
_msg_enabled = flag
|
|
|
|
|
|
|
|
|
|
|
|
def set_warn_enabled(flag):
|
|
|
|
global _warn_enabled
|
|
|
|
_warn_enabled = flag
|
|
|
|
|
|
|
|
|
|
|
|
def set_error_enabled(flag):
|
|
|
|
global _error_enabled
|
|
|
|
_error_enabled = flag
|
|
|
|
|
|
|
|
|
|
|
|
def msg_enabled():
|
|
|
|
return _msg_enabled
|
|
|
|
|
|
|
|
|
|
|
|
def warn_enabled():
|
|
|
|
return _warn_enabled
|
|
|
|
|
|
|
|
|
|
|
|
def error_enabled():
|
|
|
|
return _error_enabled
|
|
|
|
|
|
|
|
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
@contextlib.contextmanager
|
|
|
|
def output_filter(filter_fn):
|
|
|
|
"""Context manager that applies a filter to all output."""
|
|
|
|
global _output_filter
|
|
|
|
saved_filter = _output_filter
|
|
|
|
try:
|
|
|
|
_output_filter = filter_fn
|
|
|
|
yield
|
|
|
|
finally:
|
|
|
|
_output_filter = saved_filter
|
|
|
|
|
|
|
|
|
2019-02-13 10:14:35 -08:00
|
|
|
class SuppressOutput:
|
|
|
|
"""Class for disabling output in a scope using 'with' keyword"""
|
|
|
|
|
2022-07-30 15:19:18 -07:00
|
|
|
def __init__(self, msg_enabled=True, warn_enabled=True, error_enabled=True):
|
2019-02-13 10:14:35 -08:00
|
|
|
|
|
|
|
self._msg_enabled_initial = _msg_enabled
|
|
|
|
self._warn_enabled_initial = _warn_enabled
|
|
|
|
self._error_enabled_initial = _error_enabled
|
|
|
|
|
|
|
|
self._msg_enabled = msg_enabled
|
|
|
|
self._warn_enabled = warn_enabled
|
|
|
|
self._error_enabled = error_enabled
|
|
|
|
|
|
|
|
def __enter__(self):
|
|
|
|
set_msg_enabled(self._msg_enabled)
|
|
|
|
set_warn_enabled(self._warn_enabled)
|
|
|
|
set_error_enabled(self._error_enabled)
|
|
|
|
|
|
|
|
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
|
|
set_msg_enabled(self._msg_enabled_initial)
|
|
|
|
set_warn_enabled(self._warn_enabled_initial)
|
|
|
|
set_error_enabled(self._error_enabled_initial)
|
|
|
|
|
|
|
|
|
2016-11-15 15:17:03 -05:00
|
|
|
def set_stacktrace(flag):
|
|
|
|
global _stacktrace
|
|
|
|
_stacktrace = flag
|
|
|
|
|
|
|
|
|
|
|
|
def process_stacktrace(countback):
|
2016-11-15 18:10:37 -05:00
|
|
|
"""Gives file and line frame 'countback' frames from the bottom"""
|
2016-11-15 15:17:03 -05:00
|
|
|
st = traceback.extract_stack()
|
2016-11-17 12:47:15 -05:00
|
|
|
# Not all entries may be spack files, we have to remove those that aren't.
|
2016-11-16 12:14:25 -05:00
|
|
|
file_list = []
|
|
|
|
for frame in st:
|
2016-11-17 12:47:15 -05:00
|
|
|
# Check that the file is a spack file
|
2021-10-22 14:00:02 -04:00
|
|
|
if frame[0].find(os.path.sep + "spack") >= 0:
|
2016-11-17 12:47:15 -05:00
|
|
|
file_list.append(frame[0])
|
|
|
|
# We use commonprefix to find what the spack 'root' directory is.
|
2016-11-16 12:14:25 -05:00
|
|
|
root_dir = os.path.commonprefix(file_list)
|
|
|
|
root_len = len(root_dir)
|
2016-11-15 18:10:37 -05:00
|
|
|
st_idx = len(st) - countback - 1
|
2016-11-15 15:17:03 -05:00
|
|
|
st_text = "%s:%i " % (st[st_idx][0][root_len:], st[st_idx][1])
|
|
|
|
return st_text
|
2014-06-19 08:47:06 -07:00
|
|
|
|
|
|
|
|
2020-07-23 00:49:57 -07:00
|
|
|
def show_pid():
|
|
|
|
return is_debug(2)
|
|
|
|
|
|
|
|
|
2019-02-13 10:14:35 -08:00
|
|
|
def get_timestamp(force=False):
|
|
|
|
"""Get a string timestamp"""
|
|
|
|
if _debug or _timestamp or force:
|
Distributed builds (#13100)
Fixes #9394
Closes #13217.
## Background
Spack provides the ability to enable/disable parallel builds through two options: package `parallel` and configuration `build_jobs`. This PR changes the algorithm to allow multiple, simultaneous processes to coordinate the installation of the same spec (and specs with overlapping dependencies.).
The `parallel` (boolean) property sets the default for its package though the value can be overridden in the `install` method.
Spack's current parallel builds are limited to build tools supporting `jobs` arguments (e.g., `Makefiles`). The number of jobs actually used is calculated as`min(config:build_jobs, # cores, 16)`, which can be overridden in the package or on the command line (i.e., `spack install -j <# jobs>`).
This PR adds support for distributed (single- and multi-node) parallel builds. The goals of this work include improving the efficiency of installing packages with many dependencies and reducing the repetition associated with concurrent installations of (dependency) packages.
## Approach
### File System Locks
Coordination between concurrent installs of overlapping packages to a Spack instance is accomplished through bottom-up dependency DAG processing and file system locks. The runs can be a combination of interactive and batch processes affecting the same file system. Exclusive prefix locks are required to install a package while shared prefix locks are required to check if the package is installed.
Failures are communicated through a separate exclusive prefix failure lock, for concurrent processes, combined with a persistent store, for separate, related build processes. The resulting file contains the failing spec to facilitate manual debugging.
### Priority Queue
Management of dependency builds changed from reliance on recursion to use of a priority queue where the priority of a spec is based on the number of its remaining uninstalled dependencies.
Using a queue required a change to dependency build exception handling with the most visible issue being that the `install` method *must* install something in the prefix. Consequently, packages can no longer get away with an install method consisting of `pass`, for example.
## Caveats
- This still only parallelizes a single-rooted build. Multi-rooted installs (e.g., for environments) are TBD in a future PR.
Tasks:
- [x] Adjust package lock timeout to correspond to value used in the demo
- [x] Adjust database lock timeout to reduce contention on startup of concurrent
`spack install <spec>` calls
- [x] Replace (test) package's `install: pass` methods with file creation since post-install
`sanity_check_prefix` will otherwise error out with `Install failed .. Nothing was installed!`
- [x] Resolve remaining existing test failures
- [x] Respond to alalazo's initial feedback
- [x] Remove `bin/demo-locks.py`
- [x] Add new tests to address new coverage issues
- [x] Replace built-in package's `def install(..): pass` to "install" something
(i.e., only `apple-libunwind`)
- [x] Increase code coverage
2020-02-19 00:04:22 -08:00
|
|
|
# Note inclusion of the PID is useful for parallel builds.
|
2022-07-30 15:19:18 -07:00
|
|
|
pid = ", {0}".format(os.getpid()) if show_pid() else ""
|
|
|
|
return "[{0}{1}] ".format(datetime.now().strftime("%Y-%m-%d-%H:%M:%S.%f"), pid)
|
2019-02-13 10:14:35 -08:00
|
|
|
else:
|
2022-07-30 15:19:18 -07:00
|
|
|
return ""
|
2019-02-13 10:14:35 -08:00
|
|
|
|
|
|
|
|
2017-02-17 15:45:02 -06:00
|
|
|
def msg(message, *args, **kwargs):
|
2019-02-13 10:14:35 -08:00
|
|
|
if not msg_enabled():
|
|
|
|
return
|
|
|
|
|
2019-06-05 17:23:40 -07:00
|
|
|
if isinstance(message, Exception):
|
|
|
|
message = "%s: %s" % (message.__class__.__name__, str(message))
|
|
|
|
|
2022-07-30 15:19:18 -07:00
|
|
|
newline = kwargs.get("newline", True)
|
2016-11-15 15:17:03 -05:00
|
|
|
st_text = ""
|
|
|
|
if _stacktrace:
|
|
|
|
st_text = process_stacktrace(2)
|
2017-02-17 15:45:02 -06:00
|
|
|
if newline:
|
2022-07-30 15:19:18 -07:00
|
|
|
cprint("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
|
2017-02-17 15:45:02 -06:00
|
|
|
else:
|
2022-07-30 15:19:18 -07:00
|
|
|
cwrite("@*b{%s==>} %s%s" % (st_text, get_timestamp(), cescape(_output_filter(message))))
|
2013-06-29 15:59:43 -07:00
|
|
|
for arg in args:
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
print(indent + _output_filter(six.text_type(arg)))
|
2013-02-13 17:50:44 -08:00
|
|
|
|
|
|
|
|
2013-06-29 15:59:43 -07:00
|
|
|
def info(message, *args, **kwargs):
|
2019-06-05 17:23:40 -07:00
|
|
|
if isinstance(message, Exception):
|
|
|
|
message = "%s: %s" % (message.__class__.__name__, str(message))
|
|
|
|
|
2022-07-30 15:19:18 -07:00
|
|
|
format = kwargs.get("format", "*b")
|
|
|
|
stream = kwargs.get("stream", sys.stdout)
|
|
|
|
wrap = kwargs.get("wrap", False)
|
|
|
|
break_long_words = kwargs.get("break_long_words", False)
|
|
|
|
st_countback = kwargs.get("countback", 3)
|
2014-07-08 01:56:32 -07:00
|
|
|
|
2016-11-15 15:17:03 -05:00
|
|
|
st_text = ""
|
|
|
|
if _stacktrace:
|
|
|
|
st_text = process_stacktrace(st_countback)
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
cprint(
|
2022-07-30 15:19:18 -07:00
|
|
|
"@%s{%s==>} %s%s"
|
|
|
|
% (format, st_text, get_timestamp(), cescape(_output_filter(six.text_type(message)))),
|
|
|
|
stream=stream,
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
)
|
2013-06-29 15:59:43 -07:00
|
|
|
for arg in args:
|
2016-01-14 10:26:31 -08:00
|
|
|
if wrap:
|
|
|
|
lines = textwrap.wrap(
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
_output_filter(six.text_type(arg)),
|
|
|
|
initial_indent=indent,
|
|
|
|
subsequent_indent=indent,
|
2022-07-30 15:19:18 -07:00
|
|
|
break_long_words=break_long_words,
|
installation: filter padding from all `tty` output
This is both a bugfix and a generalization of #25168. In #25168, we attempted to filter padding
*just* from the debug output of `spack.util.executable.Executable` objects. It turns out we got it
wrong -- filtering the command line string instead of the arg list resulted in output like this:
```
==> [2021-08-05-21:34:19.918576] ["'", '/', 'b', 'i', 'n', '/', 't', 'a', 'r', "'", ' ', "'", '-', 'o', 'x', 'f', "'", ' ', "'", '/', 't', 'm', 'p', '/', 'r', 'o', 'o', 't', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '/', 's', 'p', 'a', 'c', 'k', '-', 's', 't', 'a', 'g', 'e', '-', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '-', 'w', 'p', 'h', 'p', 't', 'l', 'h', 'w', 'u', 's', 'e', 'i', 'a', '4', 'k', 'p', 'g', 'y', 'd', 'q', 'l', 'l', 'i', '2', '4', 'q', 'b', '5', '5', 'q', 'u', '4', '/', 'p', 'a', 't', 'c', 'h', 'e', 'l', 'f', '-', '0', '.', '1', '3', '.', 't', 'a', 'r', '.', 'b', 'z', '2', "'"]
```
Additionally, plenty of builds output padded paths in other plcaes -- e.g., not just command
arguments, but in other `tty` messages via `llnl.util.filesystem` and other places. `Executable`
isn't really the right place for this.
This PR reverts the changes to `Executable` and moves the filtering into `llnl.util.tty`. There is
now a context manager there that you can use to install a filter for all output.
`spack.installer.build_process()` now uses this context manager to make `tty` do path filtering
when padding is enabled.
- [x] revert filtering in `Executable`
- [x] add ability for `tty` to filter output
- [x] install output filter in `build_process()`
- [x] tests
2021-08-06 17:31:27 -07:00
|
|
|
)
|
2016-01-14 10:26:31 -08:00
|
|
|
for line in lines:
|
2022-07-30 15:19:18 -07:00
|
|
|
stream.write(line + "\n")
|
2016-01-14 10:26:31 -08:00
|
|
|
else:
|
2022-07-30 15:19:18 -07:00
|
|
|
stream.write(indent + _output_filter(six.text_type(arg)) + "\n")
|
2013-02-13 17:50:44 -08:00
|
|
|
|
|
|
|
|
2016-01-14 10:26:31 -08:00
|
|
|
def verbose(message, *args, **kwargs):
|
2014-06-19 08:47:06 -07:00
|
|
|
if _verbose:
|
2022-07-30 15:19:18 -07:00
|
|
|
kwargs.setdefault("format", "c")
|
2016-01-14 10:26:31 -08:00
|
|
|
info(message, *args, **kwargs)
|
2013-02-13 17:50:44 -08:00
|
|
|
|
|
|
|
|
2016-01-14 10:26:31 -08:00
|
|
|
def debug(message, *args, **kwargs):
|
2022-07-30 15:19:18 -07:00
|
|
|
level = kwargs.get("level", 1)
|
2020-07-23 00:49:57 -07:00
|
|
|
if is_debug(level):
|
2022-07-30 15:19:18 -07:00
|
|
|
kwargs.setdefault("format", "g")
|
|
|
|
kwargs.setdefault("stream", sys.stderr)
|
2016-01-14 10:26:31 -08:00
|
|
|
info(message, *args, **kwargs)
|
2013-02-18 23:46:04 -08:00
|
|
|
|
2013-02-13 17:50:44 -08:00
|
|
|
|
2016-01-14 10:26:31 -08:00
|
|
|
def error(message, *args, **kwargs):
|
2019-02-13 10:14:35 -08:00
|
|
|
if not error_enabled():
|
|
|
|
return
|
|
|
|
|
2022-07-30 15:19:18 -07:00
|
|
|
kwargs.setdefault("format", "*r")
|
|
|
|
kwargs.setdefault("stream", sys.stderr)
|
2019-06-07 18:57:26 +02:00
|
|
|
info("Error: " + six.text_type(message), *args, **kwargs)
|
2013-02-13 17:50:44 -08:00
|
|
|
|
|
|
|
|
2016-01-14 10:26:31 -08:00
|
|
|
def warn(message, *args, **kwargs):
|
2019-02-13 10:14:35 -08:00
|
|
|
if not warn_enabled():
|
|
|
|
return
|
|
|
|
|
2022-07-30 15:19:18 -07:00
|
|
|
kwargs.setdefault("format", "*Y")
|
|
|
|
kwargs.setdefault("stream", sys.stderr)
|
2019-06-07 18:57:26 +02:00
|
|
|
info("Warning: " + six.text_type(message), *args, **kwargs)
|
2013-02-13 17:50:44 -08:00
|
|
|
|
2013-06-29 15:59:43 -07:00
|
|
|
|
2016-01-14 10:26:31 -08:00
|
|
|
def die(message, *args, **kwargs):
|
2022-07-30 15:19:18 -07:00
|
|
|
kwargs.setdefault("countback", 4)
|
2016-01-14 10:26:31 -08:00
|
|
|
error(message, *args, **kwargs)
|
2013-02-13 17:50:44 -08:00
|
|
|
sys.exit(1)
|
|
|
|
|
2013-06-29 15:59:43 -07:00
|
|
|
|
2013-12-24 00:57:56 -08:00
|
|
|
def get_number(prompt, **kwargs):
|
2022-07-30 15:19:18 -07:00
|
|
|
default = kwargs.get("default", None)
|
|
|
|
abort = kwargs.get("abort", None)
|
2013-12-24 00:57:56 -08:00
|
|
|
|
|
|
|
if default is not None and abort is not None:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " (default is %s, %s to abort) " % (default, abort)
|
2013-12-24 00:57:56 -08:00
|
|
|
elif default is not None:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " (default is %s) " % default
|
2013-12-24 00:57:56 -08:00
|
|
|
elif abort is not None:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " (%s to abort) " % abort
|
2013-12-24 00:57:56 -08:00
|
|
|
|
|
|
|
number = None
|
|
|
|
while number is None:
|
2017-02-17 15:45:02 -06:00
|
|
|
msg(prompt, newline=False)
|
2017-04-23 10:32:08 -05:00
|
|
|
ans = input()
|
2019-06-07 18:57:26 +02:00
|
|
|
if ans == six.text_type(abort):
|
2013-12-24 00:57:56 -08:00
|
|
|
return None
|
|
|
|
|
|
|
|
if ans:
|
|
|
|
try:
|
|
|
|
number = int(ans)
|
|
|
|
if number < 1:
|
|
|
|
msg("Please enter a valid number.")
|
|
|
|
number = None
|
|
|
|
except ValueError:
|
|
|
|
msg("Please enter a valid number.")
|
|
|
|
elif default is not None:
|
|
|
|
number = default
|
|
|
|
return number
|
2014-04-14 12:53:23 -07:00
|
|
|
|
|
|
|
|
2014-11-08 23:20:01 -08:00
|
|
|
def get_yes_or_no(prompt, **kwargs):
|
2022-07-30 15:19:18 -07:00
|
|
|
default_value = kwargs.get("default", None)
|
2014-11-08 23:20:01 -08:00
|
|
|
|
|
|
|
if default_value is None:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " [y/n] "
|
2014-11-08 23:20:01 -08:00
|
|
|
elif default_value is True:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " [Y/n] "
|
2014-11-08 23:20:01 -08:00
|
|
|
elif default_value is False:
|
2022-07-30 15:19:18 -07:00
|
|
|
prompt += " [y/N] "
|
2014-11-08 23:20:01 -08:00
|
|
|
else:
|
2022-07-30 15:19:18 -07:00
|
|
|
raise ValueError("default for get_yes_no() must be True, False, or None.")
|
2014-11-08 23:20:01 -08:00
|
|
|
|
|
|
|
result = None
|
|
|
|
while result is None:
|
2017-02-17 15:45:02 -06:00
|
|
|
msg(prompt, newline=False)
|
2017-04-23 10:32:08 -05:00
|
|
|
ans = input().lower()
|
2014-11-08 23:20:01 -08:00
|
|
|
if not ans:
|
|
|
|
result = default_value
|
|
|
|
if result is None:
|
2017-03-07 14:25:48 -08:00
|
|
|
print("Please enter yes or no.")
|
2014-11-08 23:20:01 -08:00
|
|
|
else:
|
2022-07-30 15:19:18 -07:00
|
|
|
if ans == "y" or ans == "yes":
|
2014-11-08 23:20:01 -08:00
|
|
|
result = True
|
2022-07-30 15:19:18 -07:00
|
|
|
elif ans == "n" or ans == "no":
|
2014-11-08 23:20:01 -08:00
|
|
|
result = False
|
|
|
|
return result
|
|
|
|
|
|
|
|
|
2014-04-14 12:53:23 -07:00
|
|
|
def hline(label=None, **kwargs):
|
2014-12-02 21:56:22 -08:00
|
|
|
"""Draw a labeled horizontal line.
|
2017-04-26 00:24:02 -05:00
|
|
|
|
|
|
|
Keyword Arguments:
|
|
|
|
char (str): Char to draw the line with. Default '-'
|
|
|
|
max_width (int): Maximum width of the line. Default is 64 chars.
|
2014-04-14 12:53:23 -07:00
|
|
|
"""
|
2022-07-30 15:19:18 -07:00
|
|
|
char = kwargs.pop("char", "-")
|
|
|
|
max_width = kwargs.pop("max_width", 64)
|
2014-12-02 21:56:22 -08:00
|
|
|
if kwargs:
|
2016-08-09 13:23:53 -07:00
|
|
|
raise TypeError(
|
2022-07-30 15:19:18 -07:00
|
|
|
"'%s' is an invalid keyword argument for this function." % next(kwargs.iterkeys())
|
|
|
|
)
|
2014-04-14 12:53:23 -07:00
|
|
|
|
2014-11-23 17:53:21 -06:00
|
|
|
rows, cols = terminal_size()
|
2014-04-14 12:53:23 -07:00
|
|
|
if not cols:
|
|
|
|
cols = max_width
|
|
|
|
else:
|
|
|
|
cols -= 2
|
|
|
|
cols = min(max_width, cols)
|
|
|
|
|
2019-06-07 18:57:26 +02:00
|
|
|
label = six.text_type(label)
|
2014-12-02 21:56:22 -08:00
|
|
|
prefix = char * 2 + " "
|
|
|
|
suffix = " " + (cols - len(prefix) - clen(label)) * char
|
2014-04-14 12:53:23 -07:00
|
|
|
|
|
|
|
out = StringIO()
|
2014-12-02 21:56:22 -08:00
|
|
|
out.write(prefix)
|
|
|
|
out.write(label)
|
2014-04-14 12:53:23 -07:00
|
|
|
out.write(suffix)
|
|
|
|
|
2017-03-07 14:25:48 -08:00
|
|
|
print(out.getvalue())
|
2014-04-14 12:53:23 -07:00
|
|
|
|
|
|
|
|
|
|
|
def terminal_size():
|
2014-11-23 17:53:21 -06:00
|
|
|
"""Gets the dimensions of the console: (rows, cols)."""
|
2021-10-21 16:30:16 -04:00
|
|
|
if _platform != "win32":
|
2022-07-30 15:19:18 -07:00
|
|
|
|
2021-10-21 16:30:16 -04:00
|
|
|
def ioctl_gwinsz(fd):
|
|
|
|
try:
|
2022-07-30 15:19:18 -07:00
|
|
|
rc = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
|
2021-10-21 16:30:16 -04:00
|
|
|
except BaseException:
|
|
|
|
return
|
|
|
|
return rc
|
2022-07-30 15:19:18 -07:00
|
|
|
|
2021-10-21 16:30:16 -04:00
|
|
|
rc = ioctl_gwinsz(0) or ioctl_gwinsz(1) or ioctl_gwinsz(2)
|
|
|
|
if not rc:
|
|
|
|
try:
|
|
|
|
fd = os.open(os.ctermid(), os.O_RDONLY)
|
|
|
|
rc = ioctl_gwinsz(fd)
|
|
|
|
os.close(fd)
|
|
|
|
except BaseException:
|
|
|
|
pass
|
|
|
|
if not rc:
|
2022-07-30 15:19:18 -07:00
|
|
|
rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
|
2021-10-21 16:30:16 -04:00
|
|
|
|
|
|
|
return int(rc[0]), int(rc[1])
|
|
|
|
else:
|
2022-01-24 14:35:44 -05:00
|
|
|
if sys.version_info[0] < 3:
|
2022-07-30 15:19:18 -07:00
|
|
|
raise RuntimeError(
|
|
|
|
"Terminal size not obtainable on Windows with a\
|
|
|
|
Python version older than 3"
|
|
|
|
)
|
|
|
|
rc = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", 80))
|
2021-10-21 16:30:16 -04:00
|
|
|
return int(rc[0]), int(rc[1])
|