Reworked _find_matches_in_path to return lazy commands

These commands, when invoked, give back a tuple (CompilerKey,path) or
None if the compiler was not found. CompilerKey is a namedtuple
containing all the information needed to identify a compiler.

find_compiler has been changed accordingly.

lib/spack/spack/architecture.py

@@ -56,12 +56,12 @@
 attributes front_os and back_os. The operating system as described earlier,
 will be responsible for compiler detection.
 """
+import collections
 import os
 import inspect
 import itertools
 import platform as py_platform
 
-import llnl.util.multiproc as mp
 import llnl.util.tty as tty
 from llnl.util.lang import memoized, list_modules, key_ordering
 
@@ -246,7 +246,8 @@ def find_compilers(self, *path_hints):
         # NOTE: we import spack.compilers here to avoid init order cycles
         import spack.compilers
         types = spack.compilers.all_compiler_types()
-        compiler_lists = mp.parmap(
+        # TODO: was parmap before
+        compiler_lists = map(
             lambda cmp_cls: self.find_compiler(cmp_cls, *paths),
             types)
 
@@ -266,24 +267,41 @@ def find_compiler(self, cmp_cls, *search_paths):
            prefixes, suffixes, and versions.  e.g., gcc-mp-4.7 would
            be grouped with g++-mp-4.7 and gfortran-mp-4.7.
         """
-        dicts = mp.parmap(
+        # The commands returned here are already sorted by language
-            lambda t: cmp_cls._find_matches_in_path(*t),
+        commands = cmp_cls.search_compiler_commands(*search_paths)
-            [('cc',) + tuple(search_paths), ('cxx',) + tuple(search_paths),
-             ('f77',) + tuple(search_paths), ('fc',) + tuple(search_paths)])
 
-        all_keys = set(key for d in dicts for key in d)
+        def invoke(f):
+            return f()
+
+        compilers = map(invoke, commands)
+
+        # Remove search with no results
+        compilers = filter(None, compilers)
 
         # Skip compilers with unknown version
-        def has_known_version(x):
+        def has_known_version(compiler_entry):
             """Returns True if the key has not an unknown version."""
-            version, _, _ = x
+            compiler_key, _ = compiler_entry
-            return version != 'unknown'
+            return compiler_key.version != 'unknown'
 
-        valid_keys = filter(has_known_version, all_keys)
+        compilers = filter(has_known_version, compilers)
+
+        compilers_by_language = collections.defaultdict(dict)
+        language_key = lambda x: x[0].language
+        for language, group in itertools.groupby(compilers, language_key):
+            # The 'successful' list is ordered like the input paths.
+            # Reverse it here so that the dict creation (last insert wins)
+            # does not spoil the intended precedence.
+            compilers_by_language[language] = dict(reversed(list(group)))
+
+        dicts = [compilers_by_language[language]
+                 for language in ('cc', 'cxx', 'f77', 'fc')]
+
+        valid_keys = set(key for d in dicts for key in d)
 
         compilers = {}
         for k in valid_keys:
-            ver, _, _ = k
+            ver = k.version
 
             paths = tuple(pn.get(k, None) for pn in dicts)
             spec = spack.spec.CompilerSpec(cmp_cls.name, ver)

lib/spack/spack/compiler.py

@@ -3,18 +3,18 @@
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
+import collections
 import os
 import re
 import itertools
 
 import llnl.util.lang
 import llnl.util.tty as tty
-import llnl.util.multiproc as mp
 
 import spack.error
 import spack.spec
 import spack.architecture
-from spack.util.executable import Executable, ProcessError
+import spack.util.executable
 
 __all__ = ['Compiler']
 
@@ -249,19 +249,11 @@ def fc_version(cls, fc):
         return cls.default_version(fc)
 
     @classmethod
-    def _find_matches_in_path(cls, compiler_language, *search_paths):
+    def search_compiler_commands(cls, *search_paths):
-        """Finds compilers for a given language in the paths supplied.
+        """Returns a list of commands that, when invoked, search for compilers
-
+        in the paths supplied.
-        Looks for all combinations of ``compiler_names`` with the
-        ``prefixes`` and ``suffixes`` defined for this compiler
-        class.  If any compilers match the compiler_names,
-        prefixes, or suffixes, uses ``detect_version`` to figure
-        out what version the compiler is.
 
         Args:
-            compiler_language (str): language of the compiler (either
-                'cc', 'cxx', 'f77' or 'fc')
-
             *search_paths (list of paths): paths where to look for a
                 compiler
 
@@ -276,43 +268,42 @@ def is_accessible_dir(x):
         # Select accessible directories
         search_directories = filter(is_accessible_dir, search_paths)
 
-        # Get compiler names and the callback to detect their versions
+        search_args = []
-        compiler_names = getattr(cls, '{0}_names'.format(compiler_language))
+        for language in ('cc', 'cxx', 'f77', 'fc'):
-        detect_version = getattr(cls, '{0}_version'.format(compiler_language))
 
-        # Compile all the regular expressions used for files beforehand
+            # Get compiler names and the callback to detect their versions
-        prefixes = [''] + cls.prefixes
+            compiler_names = getattr(cls, '{0}_names'.format(language))
-        suffixes = [''] + cls.suffixes
+            detect_version = getattr(cls, '{0}_version'.format(language))
-        regexp_fmt = r'^({0}){1}({2})$'
-        search_regexps = [
-            re.compile(regexp_fmt.format(prefix, re.escape(name), suffix))
-            for prefix, name, suffix in
-            itertools.product(prefixes, compiler_names, suffixes)
-        ]
 
-        # Select only the files matching a regexp
+            # Compile all the regular expressions used for files beforehand.
-        checks = []
+            # This searches for any combination of <prefix><name><suffix>
-        for d in search_directories:
+            # defined for the compiler
-            # Only select actual files, use the full path
+            prefixes = [''] + cls.prefixes
-            files = filter(
+            suffixes = [''] + cls.suffixes
-                os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)]
+            regexp_fmt = r'^({0}){1}({2})$'
-            )
+            search_regexps = [
-            for full_path in files:
+                re.compile(regexp_fmt.format(prefix, re.escape(name), suffix))
-                file = os.path.basename(full_path)
+                for prefix, name, suffix in
-                for regexp in search_regexps:
+                itertools.product(prefixes, compiler_names, suffixes)
-                    match = regexp.match(file)
+            ]
-                    if match:
-                        key = (full_path,) + match.groups() + (detect_version,)
-                        checks.append(key)
 
-        successful = [k for k in mp.parmap(_get_versioned_tuple, checks)
+            # Select only the files matching a regexp
-                      if k is not None]
+            for d in search_directories:
+                # Only select actual files, use the full path
+                files = filter(
+                    os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)]
+                )
+                for full_path in files:
+                    file = os.path.basename(full_path)
+                    for regexp in search_regexps:
+                        match = regexp.match(file)
+                        if match:
+                            key = (detect_version, full_path, cls, language) \
+                                + tuple(match.groups())
+                            search_args.append(key)
 
-        # The 'successful' list is ordered like the input paths.
+        commands = [detect_version_command(*args) for args in search_args]
-        # Reverse it here so that the dict creation (last insert wins)
+        return commands
-        # does not spoil the intented precedence.
-        successful.reverse()
-        return dict(((v, p, s), path) for v, p, s, path in successful)
 
     def setup_custom_environment(self, pkg, env):
         """Set any environment variables necessary to use the compiler."""
@@ -330,26 +321,49 @@ def __str__(self):
                 str(self.operating_system)))))
 
 
-def _get_versioned_tuple(compiler_check_tuple):
+CompilerKey = collections.namedtuple('CompilerKey', [
-    full_path, prefix, suffix, detect_version = compiler_check_tuple
+    'os', 'cmp_cls', 'language', 'version', 'prefix', 'suffix'
-    try:
+])
-        version = detect_version(full_path)
+
-        if (not version) or (not str(version).strip()):
+
+def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix):
+    """Returns a command that, when invoked, searches for a compiler and
+    detects its version.
+
+    Args:
+        callback (callable): function that given the full path to search
+            returns a tuple of (CompilerKey, full path) or None
+        path (path): absolute path to search
+        cmp_cls (Compiler): compiler class for this specific compiler
+        lang (str): language of the compiler
+        prefix (str): prefix of the compiler name
+        suffix (str): suffix of the compiler name
+
+    Returns:
+        Callable to be invoked.
+    """
+    def _detect_version():
+        try:
+            version = callback(path)
+            if (not version) or (not str(version).strip()):
+                tty.debug(
+                    "Couldn't get version for compiler %s" % path)
+                return None
+            return CompilerKey(
+                None, cmp_cls, lang, version, prefix, suffix
+            ), path
+        except spack.util.executable.ProcessError as e:
             tty.debug(
-                "Couldn't get version for compiler %s" % full_path)
+                "Couldn't get version for compiler %s" % path, e)
             return None
-        return (version, prefix, suffix, full_path)
+        except Exception as e:
-    except ProcessError as e:
+            # Catching "Exception" here is fine because it just
-        tty.debug(
+            # means something went wrong running a candidate executable.
-            "Couldn't get version for compiler %s" % full_path, e)
+            tty.debug("Error while executing candidate compiler %s"
-        return None
+                      % path,
-    except Exception as e:
+                      "%s: %s" % (e.__class__.__name__, e))
-        # Catching "Exception" here is fine because it just
+            return None
-        # means something went wrong running a candidate executable.
+    return _detect_version
-        tty.debug("Error while executing candidate compiler %s"
-                  % full_path,
-                  "%s: %s" % (e.__class__.__name__, e))
-        return None
 
 
 class CompilerAccessError(spack.error.SpackError):

lib/spack/spack/operating_systems/cnl.py

@@ -6,7 +6,6 @@
 import re
 
 import llnl.util.tty as tty
-import llnl.util.multiproc as mp
 
 from spack.architecture import OperatingSystem
 from spack.util.module_cmd import module
@@ -40,7 +39,8 @@ def find_compilers(self, *paths):
         import spack.compilers
 
         types = spack.compilers.all_compiler_types()
-        compiler_lists = mp.parmap(
+        # TODO: was parmap before
+        compiler_lists = map(
             lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
 
         # ensure all the version calls we made are cached in the parent

lib/spack/spack/test/compilers.py

@@ -23,7 +23,7 @@
 import spack.compilers.xl_r
 import spack.compilers.fj
 
-from spack.compiler import _get_versioned_tuple, Compiler
+from spack.compiler import detect_version_command, Compiler, CompilerKey
 
 
 def test_get_compiler_duplicates(config):
@@ -46,16 +46,20 @@ def test_all_compilers(config):
 
 
 def test_version_detection_is_empty():
-    no_version = lambda x: None
+    command = detect_version_command(
-    compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', no_version)
+        callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None,
-    assert not _get_versioned_tuple(compiler_check_tuple)
+        lang='cc', prefix='', suffix=r'\d\d'
+    )
+    assert command() is None
 
 
 def test_version_detection_is_successful():
-    version = lambda x: '4.9'
+    command = detect_version_command(
-    compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', version)
+        callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None,
-    assert _get_versioned_tuple(compiler_check_tuple) == (
+        lang='cc', prefix='', suffix=r'\d\d'
-        '4.9', '', r'\d\d', '/usr/bin/gcc')
+    )
+    correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc'
+    assert command() == correct
 
 
 def test_compiler_flags_from_config_are_grouped():