spack style: automatically bootstrap dependencies

This uses our bootstrapping logic to automatically install dependencies for
`spack style`. Users should no longer have to pre-install all of the tools
(`isort`, `mypy`, `black`, `flake8`). The command will do it for them.

- [x] add logic to bootstrap specs with specific version requirements in `spack style`
- [x] remove style tools from CI requirements (to ensure we test bootstrapping)
- [x] rework dependencies for `mypy` and `py-typed-ast`
      - `py-typed-ast` needs to be a link dependency
      - it needs to be at 1.4.1 or higher to work with python 3.9

Signed-off-by: vsoch <vsoch@users.noreply.github.com>

.github/workflows/unit_tests.yaml

@@ -39,7 +39,7 @@ jobs:
         python-version: 3.9
     - name: Install Python packages
       run: |
-        pip install --upgrade pip six setuptools flake8 isort>=4.3.5 mypy>=0.900 black types-six
+        pip install --upgrade pip six setuptools types-six
     - name: Setup git configuration
       run: |
         # Need this for the git tests to succeed.
@@ -370,7 +370,6 @@ jobs:
       run: |
           pip install --upgrade pip six setuptools
           pip install --upgrade codecov coverage[toml]
-          pip install --upgrade flake8 isort>=4.3.5 mypy>=0.900
     - name: Setup Homebrew packages
       run: |
         brew install dash fish gcc gnupg2 kcov

lib/spack/docs/developer_guide.rst

@@ -818,6 +818,29 @@ Developer commands
 ``spack doc``
 ^^^^^^^^^^^^^
 
+.. _cmd-spack-style:
+
+^^^^^^^^^^^^^^^
+``spack style``
+^^^^^^^^^^^^^^^
+
+spack style exists to help the developer user to check imports and style with
+mypy, flake8, isort, and (soon) black. To run all style checks, simply do:
+
+.. code-block:: console
+
+    $ spack style
+
+To run automatic fixes for isort you can do:
+
+.. code-block:: console
+
+    $ spack style --fix
+
+You do not need any of these Python packages installed on your system for
+the checks to work! Spack will bootstrap install them from packages for
+your use.
+
 ^^^^^^^^^^^^^^^^^^^
 ``spack unit-test``
 ^^^^^^^^^^^^^^^^^^^

lib/spack/spack/cmd/style.py

@@ -14,6 +14,7 @@
 import llnl.util.tty.color as color
 from llnl.util.filesystem import working_dir
 
+import spack.bootstrap
 import spack.paths
 from spack.util.executable import which
 
@@ -44,9 +45,15 @@ def grouper(iterable, n, fillvalue=None):
 #: List of directories to exclude from checks.
 exclude_directories = [spack.paths.external_path]
 
-#: order in which tools should be run. flake8 is last so that it can
+#: Order in which tools should be run. flake8 is last so that it can
 #: double-check the results of other tools (if, e.g., --fix was provided)
-tool_order = ["isort", "mypy", "black", "flake8"]
+#: The list maps an executable name to a spack spec needed to install it.
+tool_order = [
+    ("isort", "py-isort@4.3.5:"),
+    ("mypy", "py-mypy@0.900:"),
+    ("black", "py-black"),
+    ("flake8", "py-flake8"),
+]
 
 #: tools we run in spack style
 tools = {}
@@ -210,7 +217,7 @@ def translate(match):
 
 
 def print_style_header(file_list, args):
-    tools = [tool for tool in tool_order if getattr(args, tool)]
+    tools = [tool for tool, _ in tool_order if getattr(args, tool)]
     tty.msg("Running style checks on spack:", "selected: " + ", ".join(tools))
 
     # translate modified paths to cwd_relative if needed
@@ -338,15 +345,20 @@ def prefix_relative(path):
 
         # run tools in order defined in tool_order
         returncode = 0
-        for tool_name in tool_order:
+        for tool_name, tool_spec in tool_order:
             if getattr(args, tool_name):
                 run_function, required = tools[tool_name]
                 print_tool_header(tool_name)
 
-                cmd = which(tool_name, required=required)
+                # Bootstrap tools so we don't need to require install
-                if not cmd:
+                with spack.bootstrap.ensure_bootstrap_configuration():
-                    color.cprint("  @y{%s not in PATH, skipped}" % tool_name)
+                    spec = spack.spec.Spec(tool_spec)
-                    continue
+                    cmd = spack.bootstrap.get_executable(
+                        tool_name, spec=spec, install=True
+                    )
+                    if not cmd:
+                        color.cprint("  @y{%s not in PATH, skipped}" % tool_name)
+                        continue
 
                 returncode |= run_function(cmd, file_list, args)
 

share/spack/qa/run-style-tests

@@ -15,7 +15,6 @@
 #     run-flake8-tests
 #
 . "$(dirname $0)/setup.sh"
-check_dependencies flake8 mypy
 
 # verify that the code style is correct
 spack style --root-relative

var/spack/repos/builtin/packages/py-mypy/package.py

@@ -13,15 +13,15 @@ class PyMypy(PythonPackage):
     pypi = "mypy/mypy-0.740.tar.gz"
 
     version('0.910', sha256='704098302473cb31a218f1775a873b376b30b4c18229421e9e9dc8916fd16150')
+    version('0.900', sha256='65c78570329c54fb40f956f7645e2359af5da9d8c54baa44f461cdc7f4984108')
     version('0.800', sha256='e0202e37756ed09daf4b0ba64ad2c245d357659e014c3f51d8cd0681ba66940a')
     version('0.740', sha256='48c8bc99380575deb39f5d3400ebb6a8a1cb5cc669bbba4d3bb30f904e0a0e7d')
 
     variant('python2', default=False, description='Enable checking of python 2 type annotations')
 
-    depends_on('python@3.5:', type=('build', 'run'))
+    depends_on("python@3.5:", type=("build", "run"))
     depends_on('py-setuptools', type=('build', 'run'))
-    depends_on('py-typed-ast@1.4.0:1.4.999', when='@0.900:+python2', type=('build', 'run'))
+    depends_on('py-typed-ast@1.4.0:1.4.999', type=('build', 'run'))
-    depends_on('py-typed-ast@1.4.0:1.4.999', when='@:899', type=('build', 'run'))
     depends_on('py-typing-extensions@3.7.4:', type=('build', 'run'))
     depends_on('py-mypy-extensions@0.4.3:0.4.999', type=('build', 'run'))
     depends_on('py-toml', when='@0.900:', type=('build', 'run'))

var/spack/repos/builtin/packages/py-typed-ast/package.py

@@ -12,7 +12,11 @@ class PyTypedAst(PythonPackage):
     homepage = "https://github.com/python/typed_ast"
     pypi = "typed-ast/typed_ast-1.4.0.tar.gz"
 
+    version('1.4.3', sha256='fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65')
+    version('1.4.2', sha256='9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a')
+    version('1.4.1', sha256='8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b')
     version('1.4.0', sha256='66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34')
 
-    depends_on('python@3.3:', type=('build', 'run'))
+    depends_on('python@3.3:', type=('build', 'link', 'run'))
+    depends_on('python@:3.8', when="@:1.4.0")  # build errors with 3.9 until 1.4.1
     depends_on('py-setuptools', type='build')