cmd/python: use runpy to allow multiprocessing in scripts (#41789)

Running a `spack-python` script like this:

```python

import spack
import multiprocessing

def echo(args):
    print(args)

if __name__ == "__main__":
    pool = multiprocessing.Pool(2)
    pool.map(echo, range(10))
```

will fail in `develop` with an error like this:

```console
_pickle.PicklingError: Can't pickle <function echo at 0x104865820>: attribute lookup echo on __main__ failed
```

Python expects to be able to look up the method `echo` in `sys.path["__main__"]` in
subprocesses spawned by `multiprocessing`, but because we use `InteractiveConsole` to
run `spack python`, the executed file isn't considered to be the `__main__` module, and
lookups in subprocesses fail. We tried to fake this by setting `__name__` to `__main__`
in the `spack python` command, but that doesn't fix the fact that no `__main__` module
exists.

Another annoyance with `InteractiveConsole` is that `__file__` is not defined in the
main script scope, so you can't use it in your scripts.

We can use the [runpy.run_path()](https://docs.python.org/3/library/runpy.html#runpy.run_path) function,
which has been around since Python 3.2, to fix this.

- [x] Use `runpy` module to launch non-interactive `spack python` invocations
- [x] Only use `InteractiveConsole` for interactive `spack python`

lib/spack/spack/cmd/python.py

@@ -116,6 +116,11 @@ def ipython_interpreter(args):
 
 def python_interpreter(args):
     """A python interpreter is the default interpreter"""
+
+    if args.python_args and not args.python_command:
+        sys.argv = args.python_args
+        runpy.run_path(args.python_args[0], run_name="__main__")
+    else:
         # Fake a main python shell by setting __name__ to __main__.
         console = code.InteractiveConsole({"__name__": "__main__", "spack": spack})
         if "PYTHONSTARTUP" in os.environ:
@@ -123,15 +128,9 @@ def python_interpreter(args):
             if os.path.isfile(startup_file):
                 with open(startup_file) as startup:
                     console.runsource(startup.read(), startup_file, "exec")
-
         if args.python_command:
             propagate_exceptions_from(console)
             console.runsource(args.python_command)
-    elif args.python_args:
-        propagate_exceptions_from(console)
-        sys.argv = args.python_args
-        with open(args.python_args[0]) as file:
-            console.runsource(file.read(), args.python_args[0], "exec")
         else:
             # Provides readline support, allowing user to use arrow keys
             console.push("import readline")

lib/spack/spack/cmd/unit_test.py

@@ -34,6 +34,13 @@ def setup_parser(subparser):
         default=False,
         help="show full pytest help, with advanced options",
     )
+    subparser.add_argument(
+        "-n",
+        "--numprocesses",
+        type=int,
+        default=1,
+        help="run tests in parallel up to this wide, default 1 for sequential",
+    )
 
     # extra spack arguments to list tests
     list_group = subparser.add_argument_group("listing tests")
@@ -229,6 +236,16 @@ def unit_test(parser, args, unknown_args):
     if args.extension:
         pytest_root = spack.extensions.load_extension(args.extension)
 
+    if args.numprocesses is not None and args.numprocesses > 1:
+        pytest_args.extend(
+            [
+                "--dist",
+                "loadfile",
+                "--tx",
+                f"{args.numprocesses}*popen//python=spack-tmpconfig spack python",
+            ]
+        )
+
     # pytest.ini lives in the root of the spack repository.
     with llnl.util.filesystem.working_dir(pytest_root):
         if args.list:

share/spack/spack-completion.bash

@@ -1956,7 +1956,7 @@ _spack_uninstall() {
 _spack_unit_test() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help -H --pytest-help -l --list -L --list-long -N --list-names --extension -s -k --showlocals"
+        SPACK_COMPREPLY="-h --help -H --pytest-help -n --numprocesses -l --list -L --list-long -N --list-names --extension -s -k --showlocals"
     else
         _unit_tests
     fi

share/spack/spack-completion.fish

@@ -2958,12 +2958,14 @@ complete -c spack -n '__fish_spack_using_command uninstall' -l origin -r -f -a o
 complete -c spack -n '__fish_spack_using_command uninstall' -l origin -r -d 'only remove DB records with the specified origin'
 
 # spack unit-test
-set -g __fish_spack_optspecs_spack_unit_test h/help H/pytest-help l/list L/list-long N/list-names extension= s/ k/= showlocals
+set -g __fish_spack_optspecs_spack_unit_test h/help H/pytest-help n/numprocesses= l/list L/list-long N/list-names extension= s/ k/= showlocals
 complete -c spack -n '__fish_spack_using_command_pos_remainder 0 unit-test' -f -a '(__fish_spack_unit_tests)'
 complete -c spack -n '__fish_spack_using_command unit-test' -s h -l help -f -a help
 complete -c spack -n '__fish_spack_using_command unit-test' -s h -l help -d 'show this help message and exit'
 complete -c spack -n '__fish_spack_using_command unit-test' -s H -l pytest-help -f -a pytest_help
 complete -c spack -n '__fish_spack_using_command unit-test' -s H -l pytest-help -d 'show full pytest help, with advanced options'
+complete -c spack -n '__fish_spack_using_command unit-test' -s n -l numprocesses -r -f -a numprocesses
+complete -c spack -n '__fish_spack_using_command unit-test' -s n -l numprocesses -r -d 'run tests in parallel up to this wide, default 1 for sequential'
 complete -c spack -n '__fish_spack_using_command unit-test' -s l -l list -f -a list
 complete -c spack -n '__fish_spack_using_command unit-test' -s l -l list -d 'list test filenames'
 complete -c spack -n '__fish_spack_using_command unit-test' -s L -l list-long -f -a list