diff --git a/lib/spack/spack/build_environment.py b/lib/spack/spack/build_environment.py index bd1898ccb42..a8857aecea9 100644 --- a/lib/spack/spack/build_environment.py +++ b/lib/spack/spack/build_environment.py @@ -36,9 +36,11 @@ import multiprocessing import os import re +import signal import sys import traceback import types +import warnings from collections import defaultdict from enum import Flag, auto from itertools import chain @@ -1216,15 +1218,45 @@ def _setup_pkg_and_run( input_pipe.close() -def start_build_process(pkg, function, kwargs): +class BuildProcess: + def __init__(self, *, target, args) -> None: + self.p = multiprocessing.Process(target=target, args=args) + + def start(self) -> None: + self.p.start() + + def is_alive(self) -> bool: + return self.p.is_alive() + + def join(self, *, timeout: Optional[int] = None): + self.p.join(timeout=timeout) + + def terminate(self): + # Opportunity for graceful termination + self.p.terminate() + self.p.join(timeout=1) + + # If the process didn't gracefully terminate, forcefully kill + if self.p.is_alive(): + # TODO (python 3.6 removal): use self.p.kill() instead, consider removing this class + assert isinstance(self.p.pid, int), f"unexpected value for PID: {self.p.pid}" + os.kill(self.p.pid, signal.SIGKILL) + self.p.join() + + @property + def exitcode(self): + return self.p.exitcode + + +def start_build_process(pkg, function, kwargs, *, timeout: Optional[int] = None): """Create a child process to do part of a spack build. Args: pkg (spack.package_base.PackageBase): package whose environment we should set up the child process for. - function (typing.Callable): argless function to run in the child - process. + function (typing.Callable): argless function to run in the child process. + timeout: maximum time allowed to finish the execution of function Usage:: @@ -1252,14 +1284,14 @@ def child_fun(): # Forward sys.stdin when appropriate, to allow toggling verbosity if sys.platform != "win32" and sys.stdin.isatty() and hasattr(sys.stdin, "fileno"): input_fd = Connection(os.dup(sys.stdin.fileno())) - mflags = os.environ.get("MAKEFLAGS", False) - if mflags: + mflags = os.environ.get("MAKEFLAGS") + if mflags is not None: m = re.search(r"--jobserver-[^=]*=(\d),(\d)", mflags) if m: jobserver_fd1 = Connection(int(m.group(1))) jobserver_fd2 = Connection(int(m.group(2))) - p = multiprocessing.Process( + p = BuildProcess( target=_setup_pkg_and_run, args=( serialized_pkg, @@ -1293,14 +1325,17 @@ def exitcode_msg(p): typ = "exit" if p.exitcode >= 0 else "signal" return f"{typ} {abs(p.exitcode)}" + p.join(timeout=timeout) + if p.is_alive(): + warnings.warn(f"Terminating process, since the timeout of {timeout}s was exceeded") + p.terminate() + p.join() + try: child_result = read_pipe.recv() except EOFError: - p.join() raise InstallError(f"The process has stopped unexpectedly ({exitcode_msg(p)})") - p.join() - # If returns a StopPhase, raise it if isinstance(child_result, spack.error.StopPhase): # do not print diff --git a/lib/spack/spack/ci/__init__.py b/lib/spack/spack/ci/__init__.py index 244b3a527f5..a4cec793862 100644 --- a/lib/spack/spack/ci/__init__.py +++ b/lib/spack/spack/ci/__init__.py @@ -14,7 +14,7 @@ import tempfile import zipfile from collections import namedtuple -from typing import Callable, Dict, List, Set, Union +from typing import Callable, Dict, List, Optional, Set, Union from urllib.request import Request import llnl.path @@ -1294,35 +1294,34 @@ def display_broken_spec_messages(base_url, hashes): tty.msg(msg) -def run_standalone_tests(**kwargs): +def run_standalone_tests( + *, + cdash: Optional[CDashHandler] = None, + fail_fast: bool = False, + log_file: Optional[str] = None, + job_spec: Optional[spack.spec.Spec] = None, + repro_dir: Optional[str] = None, + timeout: Optional[int] = None, +): """Run stand-alone tests on the current spec. - Arguments: - kwargs (dict): dictionary of arguments used to run the tests - - List of recognized keys: - - * "cdash" (CDashHandler): (optional) cdash handler instance - * "fail_fast" (bool): (optional) terminate tests after the first failure - * "log_file" (str): (optional) test log file name if NOT CDash reporting - * "job_spec" (Spec): spec that was built - * "repro_dir" (str): reproduction directory + Args: + cdash: cdash handler instance + fail_fast: terminate tests after the first failure + log_file: test log file name if NOT CDash reporting + job_spec: spec that was built + repro_dir: reproduction directory + timeout: maximum time (in seconds) that tests are allowed to run """ - cdash = kwargs.get("cdash") - fail_fast = kwargs.get("fail_fast") - log_file = kwargs.get("log_file") - if cdash and log_file: tty.msg(f"The test log file {log_file} option is ignored with CDash reporting") log_file = None # Error out but do NOT terminate if there are missing required arguments. - job_spec = kwargs.get("job_spec") if not job_spec: tty.error("Job spec is required to run stand-alone tests") return - repro_dir = kwargs.get("repro_dir") if not repro_dir: tty.error("Reproduction directory is required for stand-alone tests") return @@ -1331,6 +1330,9 @@ def run_standalone_tests(**kwargs): if fail_fast: test_args.append("--fail-fast") + if timeout is not None: + test_args.extend(["--timeout", str(timeout)]) + if cdash: test_args.extend(cdash.args()) else: diff --git a/lib/spack/spack/cmd/ci.py b/lib/spack/spack/cmd/ci.py index 99b4c742026..e18439a97ea 100644 --- a/lib/spack/spack/cmd/ci.py +++ b/lib/spack/spack/cmd/ci.py @@ -160,6 +160,12 @@ def setup_parser(subparser): default=False, help="stop stand-alone tests after the first failure", ) + rebuild.add_argument( + "--timeout", + type=int, + default=None, + help="maximum time (in seconds) that tests are allowed to run", + ) rebuild.set_defaults(func=ci_rebuild) spack.cmd.common.arguments.add_common_arguments(rebuild, ["jobs"]) @@ -521,6 +527,7 @@ def ci_rebuild(args): fail_fast=args.fail_fast, log_file=log_file, repro_dir=repro_dir, + timeout=args.timeout, ) except Exception as err: diff --git a/lib/spack/spack/cmd/test.py b/lib/spack/spack/cmd/test.py index a1425bb92cb..feb3aa313ac 100644 --- a/lib/spack/spack/cmd/test.py +++ b/lib/spack/spack/cmd/test.py @@ -65,6 +65,12 @@ def setup_parser(subparser): run_parser.add_argument( "--help-cdash", action="store_true", help="show usage instructions for CDash reporting" ) + run_parser.add_argument( + "--timeout", + type=int, + default=None, + help="maximum time (in seconds) that tests are allowed to run", + ) cd_group = run_parser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ["clean", "dirty"]) @@ -176,7 +182,7 @@ def test_run(args): for spec in specs: matching = spack.store.STORE.db.query_local(spec, hashes=hashes, explicit=explicit) if spec and not matching: - tty.warn("No {0}installed packages match spec {1}".format(explicit_str, spec)) + tty.warn(f"No {explicit_str}installed packages match spec {spec}") # TODO: Need to write out a log message and/or CDASH Testing # output that package not installed IF continue to process @@ -192,7 +198,7 @@ def test_run(args): # test_stage_dir test_suite = spack.install_test.TestSuite(specs_to_test, args.alias) test_suite.ensure_stage() - tty.msg("Spack test %s" % test_suite.name) + tty.msg(f"Spack test {test_suite.name}") # Set up reporter setattr(args, "package", [s.format() for s in test_suite.specs]) @@ -204,6 +210,7 @@ def test_run(args): dirty=args.dirty, fail_first=args.fail_first, externals=args.externals, + timeout=args.timeout, ) diff --git a/lib/spack/spack/install_test.py b/lib/spack/spack/install_test.py index de69f75e929..d0b3b5c4b11 100644 --- a/lib/spack/spack/install_test.py +++ b/lib/spack/spack/install_test.py @@ -12,7 +12,7 @@ import shutil import sys from collections import Counter, OrderedDict -from typing import Callable, List, Optional, Tuple, Type, TypeVar, Union +from typing import Callable, Iterable, List, Optional, Tuple, Type, TypeVar, Union import llnl.util.filesystem as fs import llnl.util.tty as tty @@ -391,7 +391,7 @@ def phase_tests(self, builder, phase_name: str, method_names: List[str]): if self.test_failures: raise TestFailure(self.test_failures) - def stand_alone_tests(self, kwargs): + def stand_alone_tests(self, kwargs, timeout: Optional[int] = None) -> None: """Run the package's stand-alone tests. Args: @@ -399,7 +399,9 @@ def stand_alone_tests(self, kwargs): """ import spack.build_environment # avoid circular dependency - spack.build_environment.start_build_process(self.pkg, test_process, kwargs) + spack.build_environment.start_build_process( + self.pkg, test_process, kwargs, timeout=timeout + ) def parts(self) -> int: """The total number of (checked) test parts.""" @@ -847,7 +849,7 @@ def write_test_summary(counts: "Counter"): class TestSuite: """The class that manages specs for ``spack test run`` execution.""" - def __init__(self, specs, alias=None): + def __init__(self, specs: Iterable[Spec], alias: Optional[str] = None) -> None: # copy so that different test suites have different package objects # even if they contain the same spec self.specs = [spec.copy() for spec in specs] @@ -855,42 +857,43 @@ def __init__(self, specs, alias=None): self.current_base_spec = None # spec currently running do_test self.alias = alias - self._hash = None - self._stage = None + self._hash: Optional[str] = None + self._stage: Optional[Prefix] = None self.counts: "Counter" = Counter() @property - def name(self): + def name(self) -> str: """The name (alias or, if none, hash) of the test suite.""" return self.alias if self.alias else self.content_hash @property - def content_hash(self): + def content_hash(self) -> str: """The hash used to uniquely identify the test suite.""" if not self._hash: json_text = sjson.dump(self.to_dict()) + assert json_text is not None, f"{__name__} unexpected value for 'json_text'" sha = hashlib.sha1(json_text.encode("utf-8")) b32_hash = base64.b32encode(sha.digest()).lower() b32_hash = b32_hash.decode("utf-8") self._hash = b32_hash return self._hash - def __call__(self, *args, **kwargs): + def __call__( + self, + *, + remove_directory: bool = True, + dirty: bool = False, + fail_first: bool = False, + externals: bool = False, + timeout: Optional[int] = None, + ): self.write_reproducibility_data() - - remove_directory = kwargs.get("remove_directory", True) - dirty = kwargs.get("dirty", False) - fail_first = kwargs.get("fail_first", False) - externals = kwargs.get("externals", False) - for spec in self.specs: try: if spec.package.test_suite: raise TestSuiteSpecError( - "Package {} cannot be run in two test suites at once".format( - spec.package.name - ) + f"Package {spec.package.name} cannot be run in two test suites at once" ) # Set up the test suite to know which test is running @@ -905,7 +908,7 @@ def __call__(self, *args, **kwargs): fs.mkdirp(test_dir) # run the package tests - spec.package.do_test(dirty=dirty, externals=externals) + spec.package.do_test(dirty=dirty, externals=externals, timeout=timeout) # Clean up on success if remove_directory: @@ -956,15 +959,12 @@ def __call__(self, *args, **kwargs): if failures: raise TestSuiteFailure(failures) - def test_status(self, spec: spack.spec.Spec, externals: bool) -> Optional[TestStatus]: - """Determine the overall test results status for the spec. + def test_status(self, spec: spack.spec.Spec, externals: bool) -> TestStatus: + """Returns the overall test results status for the spec. Args: spec: instance of the spec under test externals: ``True`` if externals are to be tested, else ``False`` - - Returns: - the spec's test status if available or ``None`` """ tests_status_file = self.tested_file_for_spec(spec) if not os.path.exists(tests_status_file): @@ -981,109 +981,84 @@ def test_status(self, spec: spack.spec.Spec, externals: bool) -> Optional[TestSt value = (f.read()).strip("\n") return TestStatus(int(value)) if value else TestStatus.NO_TESTS - def ensure_stage(self): + def ensure_stage(self) -> None: """Ensure the test suite stage directory exists.""" if not os.path.exists(self.stage): fs.mkdirp(self.stage) @property - def stage(self): - """The root test suite stage directory. - - Returns: - str: the spec's test stage directory path - """ + def stage(self) -> Prefix: + """The root test suite stage directory""" if not self._stage: self._stage = Prefix(fs.join_path(get_test_stage_dir(), self.content_hash)) return self._stage @stage.setter - def stage(self, value): + def stage(self, value: Union[Prefix, str]) -> None: """Set the value of a non-default stage directory.""" self._stage = value if isinstance(value, Prefix) else Prefix(value) @property - def results_file(self): + def results_file(self) -> Prefix: """The path to the results summary file.""" return self.stage.join(results_filename) @classmethod - def test_pkg_id(cls, spec): + def test_pkg_id(cls, spec: Spec) -> str: """The standard install test package identifier. Args: spec: instance of the spec under test - - Returns: - str: the install test package identifier """ return spec.format_path("{name}-{version}-{hash:7}") @classmethod - def test_log_name(cls, spec): + def test_log_name(cls, spec: Spec) -> str: """The standard log filename for a spec. Args: - spec (spack.spec.Spec): instance of the spec under test - - Returns: - str: the spec's log filename + spec: instance of the spec under test """ - return "%s-test-out.txt" % cls.test_pkg_id(spec) + return f"{cls.test_pkg_id(spec)}-test-out.txt" - def log_file_for_spec(self, spec): + def log_file_for_spec(self, spec: Spec) -> Prefix: """The test log file path for the provided spec. Args: - spec (spack.spec.Spec): instance of the spec under test - - Returns: - str: the path to the spec's log file + spec: instance of the spec under test """ return self.stage.join(self.test_log_name(spec)) - def test_dir_for_spec(self, spec): + def test_dir_for_spec(self, spec: Spec) -> Prefix: """The path to the test stage directory for the provided spec. Args: - spec (spack.spec.Spec): instance of the spec under test - - Returns: - str: the spec's test stage directory path + spec: instance of the spec under test """ return Prefix(self.stage.join(self.test_pkg_id(spec))) @classmethod - def tested_file_name(cls, spec): + def tested_file_name(cls, spec: Spec) -> str: """The standard test status filename for the spec. Args: - spec (spack.spec.Spec): instance of the spec under test - - Returns: - str: the spec's test status filename + spec: instance of the spec under test """ return "%s-tested.txt" % cls.test_pkg_id(spec) - def tested_file_for_spec(self, spec): + def tested_file_for_spec(self, spec: Spec) -> str: """The test status file path for the spec. Args: - spec (spack.spec.Spec): instance of the spec under test - - Returns: - str: the spec's test status file path + spec: instance of the spec under test """ return fs.join_path(self.stage, self.tested_file_name(spec)) @property - def current_test_cache_dir(self): + def current_test_cache_dir(self) -> str: """Path to the test stage directory where the current spec's cached build-time files were automatically copied. - Returns: - str: path to the current spec's staged, cached build-time files. - Raises: TestSuiteSpecError: If there is no spec being tested """ @@ -1095,13 +1070,10 @@ def current_test_cache_dir(self): return self.test_dir_for_spec(base_spec).cache.join(test_spec.name) @property - def current_test_data_dir(self): + def current_test_data_dir(self) -> str: """Path to the test stage directory where the current spec's custom package (data) files were automatically copied. - Returns: - str: path to the current spec's staged, custom package (data) files - Raises: TestSuiteSpecError: If there is no spec being tested """ @@ -1112,17 +1084,17 @@ def current_test_data_dir(self): base_spec = self.current_base_spec return self.test_dir_for_spec(base_spec).data.join(test_spec.name) - def write_test_result(self, spec, result): + def write_test_result(self, spec: Spec, result: TestStatus) -> None: """Write the spec's test result to the test suite results file. Args: - spec (spack.spec.Spec): instance of the spec under test - result (str): result from the spec's test execution (e.g, PASSED) + spec: instance of the spec under test + result: result from the spec's test execution (e.g, PASSED) """ msg = f"{self.test_pkg_id(spec)} {result}" _add_msg_to_file(self.results_file, msg) - def write_reproducibility_data(self): + def write_reproducibility_data(self) -> None: for spec in self.specs: repo_cache_path = self.stage.repo.join(spec.name) spack.repo.PATH.dump_provenance(spec, repo_cache_path) @@ -1167,12 +1139,12 @@ def from_dict(d): return TestSuite(specs, alias) @staticmethod - def from_file(filename): + def from_file(filename: str) -> "TestSuite": """Instantiate a TestSuite using the specs and optional alias provided in the given file. Args: - filename (str): The path to the JSON file containing the test + filename: The path to the JSON file containing the test suite specs and optional alias. Raises: diff --git a/lib/spack/spack/package_base.py b/lib/spack/spack/package_base.py index 4fa398e582b..d5b803b903f 100644 --- a/lib/spack/spack/package_base.py +++ b/lib/spack/spack/package_base.py @@ -1821,7 +1821,7 @@ def _resource_stage(self, resource): resource_stage_folder = "-".join(pieces) return resource_stage_folder - def do_test(self, dirty=False, externals=False): + def do_test(self, *, dirty=False, externals=False, timeout: Optional[int] = None): if self.test_requires_compiler and not any( lang in self.spec for lang in ("c", "cxx", "fortran") ): @@ -1839,7 +1839,7 @@ def do_test(self, dirty=False, externals=False): "verbose": tty.is_verbose(), } - self.tester.stand_alone_tests(kwargs) + self.tester.stand_alone_tests(kwargs, timeout=timeout) def unit_test_check(self): """Hook for unit tests to assert things about package internals. diff --git a/lib/spack/spack/test/build_environment.py b/lib/spack/spack/test/build_environment.py index db76cd0cabd..9da886cb0bb 100644 --- a/lib/spack/spack/test/build_environment.py +++ b/lib/spack/spack/test/build_environment.py @@ -1,9 +1,12 @@ # Copyright Spack Project Developers. See COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections +import multiprocessing import os import posixpath import sys +from typing import Dict, Optional, Tuple import pytest @@ -828,3 +831,88 @@ def test_extra_rpaths_is_set( assert os.environ["SPACK_COMPILER_EXTRA_RPATHS"] == expected_rpaths else: assert "SPACK_COMPILER_EXTRA_RPATHS" not in os.environ + + +class _TestProcess: + calls: Dict[str, int] = collections.defaultdict(int) + terminated = False + runtime = 0 + + def __init__(self, *, target, args): + self.alive = None + self.exitcode = 0 + self._reset() + + def start(self): + self.calls["start"] += 1 + self.alive = True + + def is_alive(self): + self.calls["is_alive"] += 1 + return self.alive + + def join(self, timeout: Optional[int] = None): + self.calls["join"] += 1 + if timeout is not None and timeout > self.runtime: + self.alive = False + + def terminate(self): + self.calls["terminate"] += 1 + self._set_terminated() + self.alive = False + + @classmethod + def _set_terminated(cls): + cls.terminated = True + + @classmethod + def _reset(cls): + cls.calls.clear() + cls.terminated = False + + +class _TestPipe: + def close(self): + pass + + def recv(self): + if _TestProcess.terminated is True: + return 1 + return 0 + + +def _pipe_fn(*, duplex: bool = False) -> Tuple[_TestPipe, _TestPipe]: + return _TestPipe(), _TestPipe() + + +@pytest.fixture() +def mock_build_process(monkeypatch): + monkeypatch.setattr(spack.build_environment, "BuildProcess", _TestProcess) + monkeypatch.setattr(multiprocessing, "Pipe", _pipe_fn) + + def _factory(*, runtime: int): + _TestProcess.runtime = runtime + + return _factory + + +@pytest.mark.parametrize( + "runtime,timeout,expected_result,expected_calls", + [ + # execution time < timeout + (2, 5, 0, {"start": 1, "join": 1, "is_alive": 1}), + # execution time > timeout + (5, 2, 1, {"start": 1, "join": 2, "is_alive": 1, "terminate": 1}), + ], +) +def test_build_process_timeout( + mock_build_process, runtime, timeout, expected_result, expected_calls +): + """Tests that we make the correct function calls in different timeout scenarios.""" + mock_build_process(runtime=runtime) + result = spack.build_environment.start_build_process( + pkg=None, function=None, kwargs={}, timeout=timeout + ) + + assert result == expected_result + assert _TestProcess.calls == expected_calls diff --git a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml index 3e81461e48d..e67ffdcda89 100644 --- a/share/spack/gitlab/cloud_pipelines/configs/ci.yaml +++ b/share/spack/gitlab/cloud_pipelines/configs/ci.yaml @@ -22,7 +22,7 @@ ci: script:: - - if [ -n "$SPACK_EXTRA_MIRROR" ]; then spack mirror add local "${SPACK_EXTRA_MIRROR}/${SPACK_CI_STACK_NAME}"; fi - spack config blame mirrors - - - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) + - - spack --color=always --backtrace ci rebuild -j ${SPACK_BUILD_JOBS} --tests --timeout 300 > >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_out.txt) 2> >(tee ${SPACK_ARTIFACTS_ROOT}/user_data/pipeline_err.txt >&2) after_script: - - cat /proc/loadavg || true - cat /proc/meminfo | grep 'MemTotal\|MemFree' || true diff --git a/share/spack/spack-completion.bash b/share/spack/spack-completion.bash index f2a3c7be1dc..65d276b9743 100644 --- a/share/spack/spack-completion.bash +++ b/share/spack/spack-completion.bash @@ -700,7 +700,7 @@ _spack_ci_rebuild_index() { } _spack_ci_rebuild() { - SPACK_COMPREPLY="-h --help -t --tests --fail-fast -j --jobs" + SPACK_COMPREPLY="-h --help -t --tests --fail-fast --timeout -j --jobs" } _spack_ci_reproduce_build() { @@ -1903,7 +1903,7 @@ _spack_test() { _spack_test_run() { if $list_options then - SPACK_COMPREPLY="-h --help --alias --fail-fast --fail-first --externals -x --explicit --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --clean --dirty" + SPACK_COMPREPLY="-h --help --alias --fail-fast --fail-first --externals -x --explicit --keep-stage --log-format --log-file --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp --help-cdash --timeout --clean --dirty" else _installed_packages fi diff --git a/share/spack/spack-completion.fish b/share/spack/spack-completion.fish index b4d4f7932b7..0a6c986757f 100644 --- a/share/spack/spack-completion.fish +++ b/share/spack/spack-completion.fish @@ -989,13 +989,15 @@ complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help complete -c spack -n '__fish_spack_using_command ci rebuild-index' -s h -l help -d 'show this help message and exit' # spack ci rebuild -set -g __fish_spack_optspecs_spack_ci_rebuild h/help t/tests fail-fast j/jobs= +set -g __fish_spack_optspecs_spack_ci_rebuild h/help t/tests fail-fast timeout= j/jobs= complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command ci rebuild' -s h -l help -d 'show this help message and exit' complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -f -a tests complete -c spack -n '__fish_spack_using_command ci rebuild' -s t -l tests -d 'run stand-alone tests after the build' complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -f -a fail_fast complete -c spack -n '__fish_spack_using_command ci rebuild' -l fail-fast -d 'stop stand-alone tests after the first failure' +complete -c spack -n '__fish_spack_using_command ci rebuild' -l timeout -r -f -a timeout +complete -c spack -n '__fish_spack_using_command ci rebuild' -l timeout -r -d 'maximum time (in seconds) that tests are allowed to run' complete -c spack -n '__fish_spack_using_command ci rebuild' -s j -l jobs -r -f -a jobs complete -c spack -n '__fish_spack_using_command ci rebuild' -s j -l jobs -r -d 'explicitly set number of parallel jobs' @@ -2950,7 +2952,7 @@ complete -c spack -n '__fish_spack_using_command test' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command test' -s h -l help -d 'show this help message and exit' # spack test run -set -g __fish_spack_optspecs_spack_test_run h/help alias= fail-fast fail-first externals x/explicit keep-stage log-format= log-file= cdash-upload-url= cdash-build= cdash-site= cdash-track= cdash-buildstamp= help-cdash clean dirty +set -g __fish_spack_optspecs_spack_test_run h/help alias= fail-fast fail-first externals x/explicit keep-stage log-format= log-file= cdash-upload-url= cdash-build= cdash-site= cdash-track= cdash-buildstamp= help-cdash timeout= clean dirty complete -c spack -n '__fish_spack_using_command_pos_remainder 0 test run' -f -a '(__fish_spack_installed_specs)' complete -c spack -n '__fish_spack_using_command test run' -s h -l help -f -a help complete -c spack -n '__fish_spack_using_command test run' -s h -l help -d 'show this help message and exit' @@ -2977,6 +2979,8 @@ complete -c spack -n '__fish_spack_using_command test run' -l cdash-track -r -f complete -c spack -n '__fish_spack_using_command test run' -l cdash-buildstamp -r -f -a cdash_buildstamp complete -c spack -n '__fish_spack_using_command test run' -l help-cdash -f -a help_cdash complete -c spack -n '__fish_spack_using_command test run' -l help-cdash -d 'show usage instructions for CDash reporting' +complete -c spack -n '__fish_spack_using_command test run' -l timeout -r -f -a timeout +complete -c spack -n '__fish_spack_using_command test run' -l timeout -r -d 'maximum time (in seconds) that tests are allowed to run' complete -c spack -n '__fish_spack_using_command test run' -l clean -f -a dirty complete -c spack -n '__fish_spack_using_command test run' -l clean -d 'unset harmful variables in the build environment (default)' complete -c spack -n '__fish_spack_using_command test run' -l dirty -f -a dirty