Allow recent pytest versions to be used with Spack (#25371)

Currently Spack vendors `pytest` at a version which is three major 
versions behind the latest (3.2.5 vs. 6.2.4). We do that since v3.2.5 
is the latest version supporting Python 2.6. Remaining so much 
behind the currently supported versions though might introduce 
some incompatibilities and is surely a technical debt.

This PR modifies Spack to:
- Use the vendored `pytest@3.2.5` only as a fallback solution, 
  if the Python interpreter used for Spack doesn't provide a newer one
- Be able to parse `pytest --collect-only` in all the different output 
  formats from v3.2.5 to v6.2.4 and use it consistently for `spack unit-test --list-*`
- Updating the unit tests in Github Actions to use a more recent `pytest` version

.github/workflows/unit_tests.yaml

@@ -114,7 +114,7 @@ jobs:
               patchelf cmake bison libbison-dev kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools codecov coverage[toml]
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]
           # ensure style checks are not skipped in unit tests for python >= 3.6
           # note that true/false (i.e., 1/0) are opposite in conditions in python and bash
           if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then
@@ -173,7 +173,7 @@ jobs:
           sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools codecov coverage[toml]
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -274,7 +274,7 @@ jobs:
               patchelf kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools codecov coverage[toml] clingo
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml] clingo
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -317,7 +317,7 @@ jobs:
     - name: Install Python packages
       run: |
           pip install --upgrade pip six setuptools
-          pip install --upgrade codecov coverage[toml]
+          pip install --upgrade pytest codecov coverage[toml]
     - name: Setup Homebrew packages
       run: |
         brew install dash fish gcc gnupg2 kcov

lib/spack/docs/conf.py

@@ -30,6 +30,7 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 sys.path.insert(0, os.path.abspath('_spack_root/lib/spack/external'))
+sys.path.insert(0, os.path.abspath('_spack_root/lib/spack/external/pytest-fallback'))
 
 if sys.version_info[0] < 3:
     sys.path.insert(

lib/spack/spack/cmd/unit_test.py

@@ -7,14 +7,19 @@
 
 import argparse
 import collections
+import os.path
 import re
 import sys
 
-import pytest
+try:
+    import pytest
+except ImportError:
+    pytest = None   # type: ignore
+
 from six import StringIO
 
+import llnl.util.filesystem
 import llnl.util.tty.color as color
-from llnl.util.filesystem import working_dir
 from llnl.util.tty.colify import colify
 
 import spack.paths
@@ -67,38 +72,6 @@ def setup_parser(subparser):
 
 def do_list(args, extra_args):
     """Print a lists of tests than what pytest offers."""
-    # Run test collection and get the tree out.
-    old_output = sys.stdout
-    try:
-        sys.stdout = output = StringIO()
-        pytest.main(['--collect-only'] + extra_args)
-    finally:
-        sys.stdout = old_output
-
-    lines = output.getvalue().split('\n')
-    tests = collections.defaultdict(lambda: set())
-    prefix = []
-
-    # collect tests into sections
-    for line in lines:
-        match = re.match(r"(\s*)<([^ ]*) '([^']*)'", line)
-        if not match:
-            continue
-        indent, nodetype, name = match.groups()
-
-        # strip parametrized tests
-        if "[" in name:
-            name = name[:name.index("[")]
-
-        depth = len(indent) // 2
-
-        if nodetype.endswith("Function"):
-            key = tuple(prefix)
-            tests[key].add(name)
-        else:
-            prefix = prefix[:depth]
-            prefix.append(name)
-
     def colorize(c, prefix):
         if isinstance(prefix, tuple):
             return "::".join(
@@ -107,8 +80,65 @@ def colorize(c, prefix):
             )
         return color.colorize("@%s{%s}" % (c, prefix))
 
+    # To list the files we just need to inspect the filesystem,
+    # which doesn't need to wait for pytest collection and doesn't
+    # require parsing pytest output
+    files = llnl.util.filesystem.find(
+        root=spack.paths.test_path, files='*.py', recursive=True
+    )
+    files = [
+        os.path.relpath(f, start=spack.paths.spack_root)
+        for f in files if not f.endswith(('conftest.py', '__init__.py'))
+    ]
+
+    old_output = sys.stdout
+    try:
+        sys.stdout = output = StringIO()
+        pytest.main(['--collect-only'] + extra_args)
+    finally:
+        sys.stdout = old_output
+
+    lines = output.getvalue().split('\n')
+    tests = collections.defaultdict(set)
+
+    # collect tests into sections
+    node_regexp = re.compile(r"(\s*)<([^ ]*) ['\"]?([^']*)['\"]?>")
+    key_parts, name_parts = [], []
+    for line in lines:
+        match = node_regexp.match(line)
+        if not match:
+            continue
+        indent, nodetype, name = match.groups()
+
+        # strip parametrized tests
+        if "[" in name:
+            name = name[:name.index("[")]
+
+        len_indent = len(indent)
+        if os.path.isabs(name):
+            name = os.path.relpath(name, start=spack.paths.spack_root)
+
+        item = (len_indent, name, nodetype)
+
+        # Reduce the parts to the scopes that are of interest
+        name_parts = [x for x in name_parts if x[0] < len_indent]
+        key_parts = [x for x in key_parts if x[0] < len_indent]
+
+        # From version 3.X to version 6.X the output format
+        # changed a lot in pytest, and probably will change
+        # in the future - so this manipulation might be fragile
+        if nodetype.lower() == 'function':
+            name_parts.append(item)
+            key_end = os.path.join(*[x[1] for x in key_parts])
+            key = next(f for f in files if f.endswith(key_end))
+            tests[key].add(tuple(x[1] for x in name_parts))
+        elif nodetype.lower() == 'class':
+            name_parts.append(item)
+        elif nodetype.lower() in ('package', 'module'):
+            key_parts.append(item)
+
     if args.list == "list":
-        files = set(prefix[0] for prefix in tests)
+        files = set(tests.keys())
         color_files = [colorize("B", file) for file in sorted(files)]
         colify(color_files)
 
@@ -144,6 +174,14 @@ def add_back_pytest_args(args, unknown_args):
 
 
 def unit_test(parser, args, unknown_args):
+    global pytest
+    if pytest is None:
+        vendored_pytest_dir = os.path.join(
+            spack.paths.external_path, 'pytest-fallback'
+        )
+        sys.path.append(vendored_pytest_dir)
+        import pytest
+
     if args.pytest_help:
         # make the pytest.main help output more accurate
         sys.argv[0] = 'spack unit-test'
@@ -161,7 +199,7 @@ def unit_test(parser, args, unknown_args):
         pytest_root = spack.extensions.path_for_extension(target, *extensions)
 
     # pytest.ini lives in the root of the spack repository.
-    with working_dir(pytest_root):
+    with llnl.util.filesystem.working_dir(pytest_root):
         if args.list:
             do_list(args, pytest_args)
             return

lib/spack/spack/test/cmd/unit_test.py

@@ -22,7 +22,10 @@ def test_list_with_pytest_arg():
 
 
 def test_list_with_keywords():
-    output = spack_test('--list', '-k', 'cmd/unit_test.py')
+    # Here we removed querying with a "/" to separate directories
+    # since the behavior is inconsistent across different pytest
+    # versions, see https://stackoverflow.com/a/48814787/771663
+    output = spack_test('--list', '-k', 'unit_test.py')
     assert output.strip() == cmd_test_py
 
 

lib/spack/spack/test/conftest.py

@@ -430,8 +430,14 @@ def _skip_if_missing_executables(request):
     """Permits to mark tests with 'require_executables' and skip the
     tests if the executables passed as arguments are not found.
     """
-    if request.node.get_marker('requires_executables'):
+    if hasattr(request.node, 'get_marker'):
-        required_execs = request.node.get_marker('requires_executables').args
+        # TODO: Remove the deprecated API as soon as we drop support for Python 2.6
+        marker = request.node.get_marker('requires_executables')
+    else:
+        marker = request.node.get_closest_marker('requires_executables')
+
+    if marker:
+        required_execs = marker.args
         missing_execs = [
             x for x in required_execs if spack.util.executable.which(x) is None
         ]
@@ -1453,7 +1459,7 @@ def invalid_spec(request):
     return request.param
 
 
-@pytest.fixture("module")
+@pytest.fixture(scope='module')
 def mock_test_repo(tmpdir_factory):
     """Create an empty repository."""
     repo_namespace = 'mock_test_repo'