Finding compilers executes all queries first and then process them

The function calls to find compilers have been reorganized to:
1. Collect all the queries that are needed to detect compiler's version
2. Execute them on demand
3. Process the results and register compilers

lib/spack/llnl/util/multiproc.py

@@ -8,25 +8,46 @@
 than multiprocessing.Pool.apply() can.  For example, apply() will fail
 to pickle functions if they're passed indirectly as parameters.
 """
-from multiprocessing import Process, Pipe, Semaphore, Value
+import functools
+from multiprocessing import Semaphore, Value
 
-__all__ = ['spawn', 'parmap', 'Barrier']
+__all__ = ['Barrier']
 
 
-def spawn(f):
+def deferred(func):
-    def fun(pipe, x):
+    """Package a function call into something that can be invoked
-        pipe.send(f(x))
+    at a later moment.
-        pipe.close()
+
-    return fun
+    Args:
+        func (callable): callable that must be deferred
+
+    Returns:
+        Deferred version of the same function
+    """
+    @functools.wraps(func)
+    def _impl(*args, **kwargs):
+        def _deferred_call():
+            return func(*args, **kwargs)
+        return _deferred_call
+    return _impl
 
 
-def parmap(f, elements):
+def invoke(f):
-    pipe = [Pipe() for x in elements]
+    return f()
-    proc = [Process(target=spawn(f), args=(c, x))
+
-            for x, (p, c) in zip(elements, pipe)]
+
-    [p.start() for p in proc]
+def execute(command_list, executor=map):
-    [p.join() for p in proc]
+    """Execute a list of packaged commands and return their result.
-    return [p.recv() for (p, c) in pipe]
+
+    Args:
+        command_list: list of commands to be executed
+        executor: object that execute each command. Must have the
+            same semantic as ``map``.
+
+    Returns:
+        List of results
+    """
+    return executor(invoke, command_list)
 
 
 class Barrier:

lib/spack/spack/architecture.py

@@ -56,15 +56,15 @@
 attributes front_os and back_os. The operating system as described earlier,
 will be responsible for compiler detection.
 """
-import collections
 import os
 import inspect
 import itertools
-import platform as py_platform
 
+import llnl.util.multiproc
 import llnl.util.tty as tty
 from llnl.util.lang import memoized, list_modules, key_ordering
 
+import spack.compiler
 import spack.paths
 import spack.error as serr
 from spack.util.naming import mod_to_class
@@ -230,7 +230,30 @@ def __repr__(self):
         return self.__str__()
 
     def _cmp_key(self):
-        return (self.name, self.version)
+        return self.name, self.version
+
+    def search_compiler_commands(self, *path_hints):
+        """Returns a list of commands that, when invoked, search for
+        compilers tied to this OS.
+
+        Args:
+            *path_hints (list of paths): path where to look for compilers
+
+        Returns:
+             List of callable functions.
+        """
+        # Turn the path hints into paths that are to be searched
+        paths = executable_search_paths(path_hints or get_path('PATH'))
+
+        # NOTE: we import spack.compilers here to avoid init order cycles
+        import spack.compilers
+
+        commands = []
+        for compiler_cls in spack.compilers.all_compiler_types():
+            commands.extend(
+                compiler_cls.search_compiler_commands(self, *paths)
+            )
+        return commands
 
     def find_compilers(self, *path_hints):
         """
@@ -238,89 +261,10 @@ def find_compilers(self, *path_hints):
         This invokes the find() method for each Compiler class,
         and appends the compilers detected to a list.
         """
-        paths = executable_search_paths(path_hints or get_path('PATH'))
+        commands = self.search_compiler_commands(*path_hints)
-
+        compilers = llnl.util.multiproc.execute(commands)
-        # Once the paths are cleaned up, do a search for each type of
+        compilers = spack.compiler.discard_invalid(compilers)
-        # compiler.  We can spawn a bunch of parallel searches to reduce
+        return spack.compiler.make_compiler_list(compilers)
-        # the overhead of spelunking all these directories.
-        # NOTE: we import spack.compilers here to avoid init order cycles
-        import spack.compilers
-        types = spack.compilers.all_compiler_types()
-        # TODO: was parmap before
-        compiler_lists = map(
-            lambda cmp_cls: self.find_compiler(cmp_cls, *paths),
-            types)
-
-        # ensure all the version calls we made are cached in the parent
-        # process, as well.  This speeds up Spack a lot.
-        clist = [comp for cl in compiler_lists for comp in cl]
-        return clist
-
-    def find_compiler(self, cmp_cls, *search_paths):
-        """Try to find the given type of compiler in the user's
-           environment. For each set of compilers found, this returns
-           compiler objects with the cc, cxx, f77, fc paths and the
-           version filled in.
-
-           This will search for compilers with the names in cc_names,
-           cxx_names, etc. and it will group them if they have common
-           prefixes, suffixes, and versions.  e.g., gcc-mp-4.7 would
-           be grouped with g++-mp-4.7 and gfortran-mp-4.7.
-        """
-        # The commands returned here are already sorted by language
-        commands = cmp_cls.search_compiler_commands(*search_paths)
-
-        def invoke(f):
-            return f()
-
-        compilers = map(invoke, commands)
-
-        # Remove search with no results
-        compilers = filter(None, compilers)
-
-        # Skip compilers with unknown version
-        def has_known_version(compiler_entry):
-            """Returns True if the key has not an unknown version."""
-            compiler_key, _ = compiler_entry
-            return compiler_key.version != 'unknown'
-
-        compilers = filter(has_known_version, compilers)
-
-        compilers_by_language = collections.defaultdict(dict)
-        language_key = lambda x: x[0].language
-        for language, group in itertools.groupby(compilers, language_key):
-            # The 'successful' list is ordered like the input paths.
-            # Reverse it here so that the dict creation (last insert wins)
-            # does not spoil the intended precedence.
-            compilers_by_language[language] = dict(reversed(list(group)))
-
-        dicts = [compilers_by_language[language]
-                 for language in ('cc', 'cxx', 'f77', 'fc')]
-
-        valid_keys = set(key for d in dicts for key in d)
-
-        compilers = {}
-        for k in valid_keys:
-            ver = k.version
-
-            paths = tuple(pn.get(k, None) for pn in dicts)
-            spec = spack.spec.CompilerSpec(cmp_cls.name, ver)
-
-            if ver in compilers:
-                prev = compilers[ver]
-
-                # prefer the one with more compilers.
-                prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
-                newcount = len([p for p in paths if p is not None])
-                prevcount = len([p for p in prev_paths if p is not None])
-
-                # Don't add if it's not an improvement over prev compiler.
-                if newcount <= prevcount:
-                    continue
-
-            compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths)
-
-        return list(compilers.values())
 
     def to_dict(self):
         return {

lib/spack/spack/compiler.py

@@ -8,7 +8,10 @@
 import re
 import itertools
 
+import platform as py_platform
+
 import llnl.util.lang
+import llnl.util.multiproc
 import llnl.util.tty as tty
 
 import spack.error
@@ -249,11 +252,12 @@ def fc_version(cls, fc):
         return cls.default_version(fc)
 
     @classmethod
-    def search_compiler_commands(cls, *search_paths):
+    def search_compiler_commands(cls, operating_system, *search_paths):
         """Returns a list of commands that, when invoked, search for compilers
         in the paths supplied.
 
         Args:
+            operating_system (OperatingSystem): the OS requesting the search
             *search_paths (list of paths): paths where to look for a
                 compiler
 
@@ -266,7 +270,7 @@ def is_accessible_dir(x):
             return os.path.isdir(x) and os.access(x, os.R_OK | os.X_OK)
 
         # Select accessible directories
-        search_directories = filter(is_accessible_dir, search_paths)
+        search_directories = list(filter(is_accessible_dir, search_paths))
 
         search_args = []
         for language in ('cc', 'cxx', 'f77', 'fc'):
@@ -298,12 +302,15 @@ def is_accessible_dir(x):
                     for regexp in search_regexps:
                         match = regexp.match(file)
                         if match:
-                            key = (detect_version, full_path, cls, language) \
+                            key = (detect_version, full_path, operating_system,
-                                + tuple(match.groups())
+                                   cls, language) + tuple(match.groups())
                             search_args.append(key)
 
-        commands = [detect_version_command(*args) for args in search_args]
+        # The 'successful' list is ordered like the input paths.
-        return commands
+        # Reverse it here so that the dict creation (last insert wins)
+        # does not spoil the intended precedence.
+        return [detect_version_command(*args)
+                for args in reversed(search_args)]
 
     def setup_custom_environment(self, pkg, env):
         """Set any environment variables necessary to use the compiler."""
@@ -326,44 +333,98 @@ def __str__(self):
 ])
 
 
-def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix):
+@llnl.util.multiproc.deferred
-    """Returns a command that, when invoked, searches for a compiler and
+def detect_version_command(
-    detects its version.
+        callback, path, operating_system, cmp_cls, lang, prefix, suffix
+):
+    """Search for a compiler and eventually detect its version.
 
     Args:
         callback (callable): function that given the full path to search
             returns a tuple of (CompilerKey, full path) or None
         path (path): absolute path to search
+        operating_system (OperatingSystem): the OS for which we are
+            looking for a compiler
         cmp_cls (Compiler): compiler class for this specific compiler
         lang (str): language of the compiler
         prefix (str): prefix of the compiler name
         suffix (str): suffix of the compiler name
 
     Returns:
-        Callable to be invoked.
+        A (CompilerKey, path) tuple if anything is found, else None
     """
-    def _detect_version():
+    try:
-        try:
+        version = callback(path)
-            version = callback(path)
+        if (not version) or (not str(version).strip()):
-            if (not version) or (not str(version).strip()):
-                tty.debug(
-                    "Couldn't get version for compiler %s" % path)
-                return None
-            return CompilerKey(
-                None, cmp_cls, lang, version, prefix, suffix
-            ), path
-        except spack.util.executable.ProcessError as e:
             tty.debug(
-                "Couldn't get version for compiler %s" % path, e)
+                "Couldn't get version for compiler %s" % path)
             return None
-        except Exception as e:
+        return CompilerKey(
-            # Catching "Exception" here is fine because it just
+            operating_system, cmp_cls, lang, version, prefix, suffix
-            # means something went wrong running a candidate executable.
+        ), path
-            tty.debug("Error while executing candidate compiler %s"
+    except spack.util.executable.ProcessError as e:
-                      % path,
+        tty.debug(
-                      "%s: %s" % (e.__class__.__name__, e))
+            "Couldn't get version for compiler %s" % path, e)
-            return None
+        return None
-    return _detect_version
+    except Exception as e:
+        # Catching "Exception" here is fine because it just
+        # means something went wrong running a candidate executable.
+        tty.debug("Error while executing candidate compiler %s"
+                  % path,
+                  "%s: %s" % (e.__class__.__name__, e))
+        return None
+
+
+def discard_invalid(compilers):
+    # Remove search with no results
+    compilers = filter(None, compilers)
+
+    # Skip compilers with unknown version
+    def has_known_version(compiler_entry):
+        """Returns True if the key has not an unknown version."""
+        compiler_key, _ = compiler_entry
+        return compiler_key.version != 'unknown'
+
+    return filter(has_known_version, compilers)
+
+
+def make_compiler_list(compilers):
+    # Group by (os, compiler type, version), (prefix, suffix), language
+    def sort_key_fn(item):
+        key, _ = item
+        return (key.os, str(key.cmp_cls), key.version), \
+               (key.prefix, key.suffix), key.language
+
+    compilers_s = sorted(compilers, key=sort_key_fn)
+    cmp_cls_d = {str(key.cmp_cls): key.cmp_cls for key, _ in compilers_s}
+
+    compilers_d = {}
+    for sort_key, group in itertools.groupby(compilers_s, sort_key_fn):
+        compiler_entry, ps, language = sort_key
+        by_compiler_entry = compilers_d.setdefault(compiler_entry, {})
+        by_ps = by_compiler_entry.setdefault(ps, {})
+        by_ps[language] = list(x[1] for x in group).pop()
+
+    # For each (os, compiler type, version) select the compiler
+    # with most entries and add it to a list
+    compilers = []
+    for compiler_entry, by_compiler_entry in compilers_d.items():
+        # Select the (prefix, suffix) match with most entries
+        max_lang, max_ps = max(
+            (len(by_compiler_entry[ps]), ps) for ps in by_compiler_entry
+        )
+
+        # Add it to the list of compilers
+        operating_system, cmp_cls_key, version = compiler_entry
+        cmp_cls = cmp_cls_d[cmp_cls_key]
+        spec = spack.spec.CompilerSpec(cmp_cls.name, version)
+        paths = [by_compiler_entry[max_ps].get(language, None)
+                 for language in ('cc', 'cxx', 'f77', 'fc')]
+        compilers.append(
+            cmp_cls(spec, operating_system, py_platform.machine(), paths)
+        )
+
+    return compilers
 
 
 class CompilerAccessError(spack.error.SpackError):

lib/spack/spack/compilers/__init__.py

@@ -9,6 +9,7 @@
 import itertools
 import os
 
+import llnl.util.multiproc
 from llnl.util.lang import list_modules
 
 import spack.paths
@@ -189,9 +190,14 @@ def find_compilers(*paths):
     Returns:
         List of compilers found in the supplied paths
     """
-    return list(itertools.chain.from_iterable(
+    search_commands = itertools.chain.from_iterable(
-        o.find_compilers(*paths) for o in all_os_classes()
+        o.search_compiler_commands(*paths) for o in all_os_classes()
-    ))
+    )
+    # TODO: activate multiprocessing
+    # with multiprocessing.Pool(processes=None) as p:
+    compilers = llnl.util.multiproc.execute(search_commands, executor=map)
+    compilers = spack.compiler.discard_invalid(compilers)
+    return spack.compiler.make_compiler_list(compilers)
 
 
 def supported_compilers():

lib/spack/spack/test/compilers.py

@@ -47,16 +47,16 @@ def test_all_compilers(config):
 
 def test_version_detection_is_empty():
     command = detect_version_command(
-        callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None,
+        callback=lambda x: None, path='/usr/bin/gcc', operating_system=None,
-        lang='cc', prefix='', suffix=r'\d\d'
+        cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
     )
     assert command() is None
 
 
 def test_version_detection_is_successful():
     command = detect_version_command(
-        callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None,
+        callback=lambda x: '4.9', path='/usr/bin/gcc', operating_system=None,
-        lang='cc', prefix='', suffix=r'\d\d'
+        cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
     )
     correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc'
     assert command() == correct