use module and package flags to get more correct mypy behavior (#21225)

The first of my two upstream patches to mypy landed in the 0.800 tag that was released this morning, which lets us use module and package parameters with a .mypy.ini file that has a files key. This uses those parameters to check all of spack in style, but leaves the packages out for now since they are still very, very broken. If no package has been modified, the packages are not checked, but if one has they are. Includes some fixes for the log tests since they were not type checking.

Should also fix all failures related to "duplicate module named package" errors.

Hopefully the next drop of mypy will include my other patch so we can just specify the modules and packages in the config file to begin with, but for now we'll have to live with a bare mypy doing a check of the libs but not the packages.

* use module and package flags to check packages properly
* stop checking package files, use package flag for libs

The packages are not type checkable yet, need to finish out another PR
before they can be.  The previous commit also didn't check the libraries
properly, this one does.

lib/spack/spack/cmd/style.py

@@ -10,6 +10,12 @@
 import sys
 import argparse
 
+from llnl.util.filesystem import working_dir
+import llnl.util.tty as tty
+
+import spack.paths
+from spack.util.executable import which
+
 if sys.version_info < (3, 0):
     from itertools import izip_longest  # novm
 
@@ -17,12 +23,6 @@
 else:
     from itertools import zip_longest  # novm
 
-from llnl.util.filesystem import working_dir
-import llnl.util.tty as tty
-
-import spack.paths
-from spack.util.executable import which
-
 
 description = (
     "runs source code style checks on Spack. Requires flake8, mypy, black for "
@@ -162,10 +162,7 @@ def setup_parser(subparser):
 
 
 def rewrite_and_print_output(
-    output,
-    args,
-    re_obj=re.compile(r"^(.+):([0-9]+):"),
-    replacement=r"{0}:{1}:",
+    output, args, re_obj=re.compile(r"^(.+):([0-9]+):"), replacement=r"{0}:{1}:"
 ):
     """rewrite ouput with <file>:<line>: format to respect path args"""
     if args.root_relative or re_obj is None:
@@ -176,8 +173,7 @@ def rewrite_and_print_output(
         def cwd_relative(path):
             return replacement.format(
                 os.path.relpath(
-                    os.path.join(spack.paths.prefix, path.group(1)),
-                    os.getcwd(),
+                    os.path.join(spack.paths.prefix, path.group(1)), os.getcwd()
                 ),
                 *list(path.groups()[1:])
             )
@@ -185,12 +181,7 @@ def cwd_relative(path):
         for line in output.split("\n"):
             if not line:
                 continue
-            print(
-                re_obj.sub(
-                    cwd_relative,
-                    line,
-                )
-            )
+            print(re_obj.sub(cwd_relative, line))
 
 
 def print_style_header(file_list, args):
@@ -251,16 +242,13 @@ def run_mypy(file_list, args):
         return 1
 
     print_tool_header("mypy")
+    mpy_args = ["--package", "spack", "--package", "llnl"]
+    # not yet, need other updates to enable this
+    # if any([is_package(f) for f in file_list]):
+    #     mpy_args.extend(["--package", "packages"])
 
-    returncode = 0
-    output = ""
-    # run in chunks of 100 at a time to avoid line length limit
-    # filename parameter in config *does not work* for this reliably
-    for chunk in grouper(file_list, 100):
-        chunk = filter(lambda e: e is not None, chunk)
-
-        output = mypy_cmd(*chunk, fail_on_error=False, output=str)
-        returncode |= mypy_cmd.returncode
+    output = mypy_cmd(*mpy_args, fail_on_error=False, output=str)
+    returncode = mypy_cmd.returncode
 
     rewrite_and_print_output(output, args)
 

lib/spack/spack/test/llnl/util/tty/log.py

@@ -11,10 +11,8 @@
 import sys
 import time
 
-try:
-    import termios
-except ImportError:
-    termios = None
+from typing import Optional  # novm
+from types import ModuleType  # novm
 
 import pytest
 
@@ -25,6 +23,13 @@
 
 from spack.util.executable import which
 
+termios = None  # type: Optional[ModuleType]
+try:
+    import termios as term_mod
+    termios = term_mod
+except ImportError:
+    pass
+
 
 @contextlib.contextmanager
 def nullcontext():