Reworked _find_matches_in_path to return lazy commands

These commands, when invoked, give back a tuple (CompilerKey,path) or
None if the compiler was not found. CompilerKey is a namedtuple
containing all the information needed to identify a compiler.

find_compiler has been changed accordingly.

lib/spack/spack/architecture.py

@@ -56,12 +56,12 @@
 attributes front_os and back_os. The operating system as described earlier,
 will be responsible for compiler detection.
 """
+import collections
 import os
 import inspect
 import itertools
 import platform as py_platform
 
-import llnl.util.multiproc as mp
 import llnl.util.tty as tty
 from llnl.util.lang import memoized, list_modules, key_ordering
 
@@ -246,7 +246,8 @@ def find_compilers(self, *path_hints):
         # NOTE: we import spack.compilers here to avoid init order cycles
         import spack.compilers
         types = spack.compilers.all_compiler_types()
-        compiler_lists = mp.parmap(
+        # TODO: was parmap before
+        compiler_lists = map(
             lambda cmp_cls: self.find_compiler(cmp_cls, *paths),
             types)
 
@@ -266,24 +267,41 @@ def find_compiler(self, cmp_cls, *search_paths):
            prefixes, suffixes, and versions.  e.g., gcc-mp-4.7 would
            be grouped with g++-mp-4.7 and gfortran-mp-4.7.
         """
-        dicts = mp.parmap(
+        # The commands returned here are already sorted by language
-            lambda t: cmp_cls._find_matches_in_path(*t),
+        commands = cmp_cls.search_compiler_commands(*search_paths)
-            [('cc',) + tuple(search_paths), ('cxx',) + tuple(search_paths),
-             ('f77',) + tuple(search_paths), ('fc',) + tuple(search_paths)])
 
-        all_keys = set(key for d in dicts for key in d)
+        def invoke(f):
+            return f()
+
+        compilers = map(invoke, commands)
+
+        # Remove search with no results
+        compilers = filter(None, compilers)
 
         # Skip compilers with unknown version
-        def has_known_version(x):
+        def has_known_version(compiler_entry):
             """Returns True if the key has not an unknown version."""
-            version, _, _ = x
+            compiler_key, _ = compiler_entry
-            return version != 'unknown'
+            return compiler_key.version != 'unknown'
 
-        valid_keys = filter(has_known_version, all_keys)
+        compilers = filter(has_known_version, compilers)
+
+        compilers_by_language = collections.defaultdict(dict)
+        language_key = lambda x: x[0].language
+        for language, group in itertools.groupby(compilers, language_key):
+            # The 'successful' list is ordered like the input paths.
+            # Reverse it here so that the dict creation (last insert wins)
+            # does not spoil the intended precedence.
+            compilers_by_language[language] = dict(reversed(list(group)))
+
+        dicts = [compilers_by_language[language]
+                 for language in ('cc', 'cxx', 'f77', 'fc')]
+
+        valid_keys = set(key for d in dicts for key in d)
 
         compilers = {}
         for k in valid_keys:
-            ver, _, _ = k
+            ver = k.version
 
             paths = tuple(pn.get(k, None) for pn in dicts)
             spec = spack.spec.CompilerSpec(cmp_cls.name, ver)

lib/spack/spack/compiler.py

@@ -3,18 +3,18 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import collections
 import os
 import re
 import itertools
 
 import llnl.util.lang
 import llnl.util.tty as tty
-import llnl.util.multiproc as mp
 
 import spack.error
 import spack.spec
 import spack.architecture
-from spack.util.executable import Executable, ProcessError
+import spack.util.executable
 
 __all__ = ['Compiler']
 
@@ -249,19 +249,11 @@ def fc_version(cls, fc):
         return cls.default_version(fc)
 
     @classmethod
-    def _find_matches_in_path(cls, compiler_language, *search_paths):
+    def search_compiler_commands(cls, *search_paths):
-        """Finds compilers for a given language in the paths supplied.
+        """Returns a list of commands that, when invoked, search for compilers
-
+        in the paths supplied.
-        Looks for all combinations of ``compiler_names`` with the
-        ``prefixes`` and ``suffixes`` defined for this compiler
-        class.  If any compilers match the compiler_names,
-        prefixes, or suffixes, uses ``detect_version`` to figure
-        out what version the compiler is.
 
         Args:
-            compiler_language (str): language of the compiler (either
-                'cc', 'cxx', 'f77' or 'fc')
-
             *search_paths (list of paths): paths where to look for a
                 compiler
 
@@ -276,11 +268,16 @@ def is_accessible_dir(x):
         # Select accessible directories
         search_directories = filter(is_accessible_dir, search_paths)
 
-        # Get compiler names and the callback to detect their versions
+        search_args = []
-        compiler_names = getattr(cls, '{0}_names'.format(compiler_language))
+        for language in ('cc', 'cxx', 'f77', 'fc'):
-        detect_version = getattr(cls, '{0}_version'.format(compiler_language))
 
-        # Compile all the regular expressions used for files beforehand
+            # Get compiler names and the callback to detect their versions
+            compiler_names = getattr(cls, '{0}_names'.format(language))
+            detect_version = getattr(cls, '{0}_version'.format(language))
+
+            # Compile all the regular expressions used for files beforehand.
+            # This searches for any combination of <prefix><name><suffix>
+            # defined for the compiler
             prefixes = [''] + cls.prefixes
             suffixes = [''] + cls.suffixes
             regexp_fmt = r'^({0}){1}({2})$'
@@ -291,7 +288,6 @@ def is_accessible_dir(x):
             ]
 
             # Select only the files matching a regexp
-        checks = []
             for d in search_directories:
                 # Only select actual files, use the full path
                 files = filter(
@@ -302,17 +298,12 @@ def is_accessible_dir(x):
                     for regexp in search_regexps:
                         match = regexp.match(file)
                         if match:
-                        key = (full_path,) + match.groups() + (detect_version,)
+                            key = (detect_version, full_path, cls, language) \
-                        checks.append(key)
+                                + tuple(match.groups())
+                            search_args.append(key)
 
-        successful = [k for k in mp.parmap(_get_versioned_tuple, checks)
+        commands = [detect_version_command(*args) for args in search_args]
-                      if k is not None]
+        return commands
-
-        # The 'successful' list is ordered like the input paths.
-        # Reverse it here so that the dict creation (last insert wins)
-        # does not spoil the intented precedence.
-        successful.reverse()
-        return dict(((v, p, s), path) for v, p, s, path in successful)
 
     def setup_custom_environment(self, pkg, env):
         """Set any environment variables necessary to use the compiler."""
@@ -330,26 +321,49 @@ def __str__(self):
                 str(self.operating_system)))))
 
 
-def _get_versioned_tuple(compiler_check_tuple):
+CompilerKey = collections.namedtuple('CompilerKey', [
-    full_path, prefix, suffix, detect_version = compiler_check_tuple
+    'os', 'cmp_cls', 'language', 'version', 'prefix', 'suffix'
+])
+
+
+def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix):
+    """Returns a command that, when invoked, searches for a compiler and
+    detects its version.
+
+    Args:
+        callback (callable): function that given the full path to search
+            returns a tuple of (CompilerKey, full path) or None
+        path (path): absolute path to search
+        cmp_cls (Compiler): compiler class for this specific compiler
+        lang (str): language of the compiler
+        prefix (str): prefix of the compiler name
+        suffix (str): suffix of the compiler name
+
+    Returns:
+        Callable to be invoked.
+    """
+    def _detect_version():
         try:
-        version = detect_version(full_path)
+            version = callback(path)
             if (not version) or (not str(version).strip()):
                 tty.debug(
-                "Couldn't get version for compiler %s" % full_path)
+                    "Couldn't get version for compiler %s" % path)
                 return None
-        return (version, prefix, suffix, full_path)
+            return CompilerKey(
-    except ProcessError as e:
+                None, cmp_cls, lang, version, prefix, suffix
+            ), path
+        except spack.util.executable.ProcessError as e:
             tty.debug(
-            "Couldn't get version for compiler %s" % full_path, e)
+                "Couldn't get version for compiler %s" % path, e)
             return None
         except Exception as e:
             # Catching "Exception" here is fine because it just
             # means something went wrong running a candidate executable.
             tty.debug("Error while executing candidate compiler %s"
-                  % full_path,
+                      % path,
                       "%s: %s" % (e.__class__.__name__, e))
             return None
+    return _detect_version
 
 
 class CompilerAccessError(spack.error.SpackError):

lib/spack/spack/operating_systems/cnl.py

@@ -6,7 +6,6 @@
 import re
 
 import llnl.util.tty as tty
-import llnl.util.multiproc as mp
 
 from spack.architecture import OperatingSystem
 from spack.util.module_cmd import module
@@ -40,7 +39,8 @@ def find_compilers(self, *paths):
         import spack.compilers
 
         types = spack.compilers.all_compiler_types()
-        compiler_lists = mp.parmap(
+        # TODO: was parmap before
+        compiler_lists = map(
             lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
 
         # ensure all the version calls we made are cached in the parent

lib/spack/spack/test/compilers.py

@@ -23,7 +23,7 @@
 import spack.compilers.xl_r
 import spack.compilers.fj
 
-from spack.compiler import _get_versioned_tuple, Compiler
+from spack.compiler import detect_version_command, Compiler, CompilerKey
 
 
 def test_get_compiler_duplicates(config):
@@ -46,16 +46,20 @@ def test_all_compilers(config):
 
 
 def test_version_detection_is_empty():
-    no_version = lambda x: None
+    command = detect_version_command(
-    compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', no_version)
+        callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None,
-    assert not _get_versioned_tuple(compiler_check_tuple)
+        lang='cc', prefix='', suffix=r'\d\d'
+    )
+    assert command() is None
 
 
 def test_version_detection_is_successful():
-    version = lambda x: '4.9'
+    command = detect_version_command(
-    compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', version)
+        callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None,
-    assert _get_versioned_tuple(compiler_check_tuple) == (
+        lang='cc', prefix='', suffix=r'\d\d'
-        '4.9', '', r'\d\d', '/usr/bin/gcc')
+    )
+    correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc'
+    assert command() == correct
 
 
 def test_compiler_flags_from_config_are_grouped():