Reduced the amount of data sent to workers

Compiler detection has been reorganized to send to workers only
the function call that detects the compiler's version, without further
information.

lib/spack/spack/architecture.py

@@ -239,7 +239,10 @@ def search_compiler_commands(self, *path_hints):
             *path_hints (list of paths): path where to look for compilers
 
         Returns:
-             List of callable functions.
+            (tags, commands): ``tags`` is a list of compiler tags, containing
+                all the information on a compiler, but version. ``commands``
+                is a list of commands that, when executed, will detect the
+                version of the corresponding compiler.
         """
         # Turn the path hints into paths that are to be searched
         paths = executable_search_paths(path_hints or get_path('PATH'))
@@ -247,12 +250,12 @@ def search_compiler_commands(self, *path_hints):
         # NOTE: we import spack.compilers here to avoid init order cycles
         import spack.compilers
 
-        commands = []
+        tags, commands = [], []
         for compiler_cls in spack.compilers.all_compiler_types():
-            commands.extend(
+            t, c = compiler_cls.search_compiler_commands(self, *paths)
-                compiler_cls.search_compiler_commands(self, *paths)
+            tags.extend(t), commands.extend(c)
-            )
+
-        return commands
+        return tags, commands
 
     def to_dict(self):
         return {

lib/spack/spack/compiler.py

@@ -263,8 +263,10 @@ def search_compiler_commands(cls, operating_system, *search_paths):
                 compiler
 
         Returns:
-            Dictionary with compilers grouped by (version, prefix, suffix)
+            (tags, commands): ``tags`` is a list of compiler tags, containing
-            tuples.
+                all the information on a compiler, but version. ``commands``
+                is a list of commands that, when executed, will detect the
+                version of the corresponding compiler.
         """
         def is_accessible_dir(x):
             """Returns True if the argument is an accessible directory."""
@@ -273,12 +275,12 @@ def is_accessible_dir(x):
         # Select accessible directories
         search_directories = list(filter(is_accessible_dir, search_paths))
 
-        search_args = []
+        tags, commands = [], []
         for language in ('cc', 'cxx', 'f77', 'fc'):
 
             # Get compiler names and the callback to detect their versions
             compiler_names = getattr(cls, '{0}_names'.format(language))
-            detect_version = getattr(cls, '{0}_version'.format(language))
+            callback = getattr(cls, '{0}_version'.format(language))
 
             # Compile all the regular expressions used for files beforehand.
             # This searches for any combination of <prefix><name><suffix>
@@ -303,15 +305,17 @@ def is_accessible_dir(x):
                     for regexp in search_regexps:
                         match = regexp.match(file)
                         if match:
-                            key = (detect_version, full_path, operating_system,
+                            tags.append(
-                                   cls, language) + tuple(match.groups())
+                                (_CompilerID(operating_system, cls, None),
-                            search_args.append(key)
+                                 _NameVariation(*match.groups()), language)
+                            )
+                            commands.append(
+                                detect_version_command(callback, full_path)
+                            )
 
-        # The 'successful' list is ordered like the input paths.
         # Reverse it here so that the dict creation (last insert wins)
         # does not spoil the intended precedence.
-        return [detect_version_command(*args)
+        return reversed(tags), reversed(commands)
-                for args in reversed(search_args)]
 
     def setup_custom_environment(self, pkg, env):
         """Set any environment variables necessary to use the compiler."""
@@ -356,62 +360,55 @@ def __hash__(self):
 
 
 @llnl.util.multiproc.deferred
-def detect_version_command(
+def detect_version_command(callback, path):
-        callback, path, operating_system, cmp_cls, lang, prefix, suffix
+    """Detects the version of a compiler at a given path.
-):
-    """Search for a compiler and eventually detect its version.
 
     Args:
-        callback (callable): function that given the full path to search
+        callback (callable): function that detects the version of
-            returns a tuple of (CompilerKey, full path) or None
+            the compiler at ``path``
         path (path): absolute path to search
-        operating_system (OperatingSystem): the OS for which we are
-            looking for a compiler
-        cmp_cls (Compiler): compiler class for this specific compiler
-        lang (str): language of the compiler
-        prefix (str): prefix of the compiler name
-        suffix (str): suffix of the compiler name
 
     Returns:
-        A (CompilerKey, path) tuple if anything is found, else None
+        (value, error): if anything is found ``value`` is a ``(version, path)``
+            tuple and ``error`` is None. If ``error`` is not None, ``value``
+            is meaningless and can be discarded.
     """
     try:
         version = callback(path)
-        if (not version) or (not str(version).strip()):
+        if version and str(version).strip():
-            tty.debug(
+            return (version, path), None
-                "Couldn't get version for compiler %s" % path)
+        error = "Couldn't get version for compiler {0}".format(path)
-            return None
-        return (_CompilerID(operating_system, cmp_cls, version),
-                _NameVariation(prefix, suffix), lang), path
     except spack.util.executable.ProcessError as e:
-        tty.debug(
+        error = "Couldn't get version for compiler {0}\n".format(path) + str(e)
-            "Couldn't get version for compiler %s" % path, e)
-        return None
     except Exception as e:
         # Catching "Exception" here is fine because it just
         # means something went wrong running a candidate executable.
-        tty.debug("Error while executing candidate compiler %s"
+        error = "Error while executing candidate compiler {0}" \
-                  % path,
+                "\n{1}: {2}".format(path, e.__class__.__name__, str(e))
-                  "%s: %s" % (e.__class__.__name__, e))
+    return None, error
-        return None
 
 
-def _discard_invalid(compilers):
+def make_compiler_list(tags, compiler_versions):
-    """Removes invalid compilers from the list"""
+    assert len(tags) == len(compiler_versions), \
-    # Remove search with no results
+        "the two arguments must have the same length"
-    compilers = filter(None, compilers)
 
-    # Skip compilers with unknown version
+    compilers_s = []
-    def has_known_version(compiler_entry):
+    for (compiler_id, name_variation, lang), (return_value, error) \
-        """Returns True if the key has not an unknown version."""
+            in zip(tags, compiler_versions):
-        (compiler_id, _, _), _ = compiler_entry
+        # If we had an error, move to the next element
-        return compiler_id.version != 'unknown'
+        if error:
+            tty.debug(error)
+            continue
 
-    return filter(has_known_version, compilers)
+        # Skip unknown versions
+        version, path = return_value
+        if version == 'unknown':
+            continue
 
+        tag = compiler_id._replace(version=version), name_variation, lang
+        compilers_s.append((tag, path))
 
-def make_compiler_list(compilers):
+    compilers_s.sort()
-    compilers_s = sorted(_discard_invalid(compilers))
 
     compilers_d = {}
     for sort_key, group in itertools.groupby(compilers_s, key=lambda x: x[0]):

lib/spack/spack/compilers/__init__.py

@@ -6,11 +6,9 @@
 """This module contains functions related to finding compilers on the
 system and configuring Spack to use multiple compilers.
 """
-import itertools
 import multiprocessing.pool
 import os
 
-
 import llnl.util.multiproc
 from llnl.util.lang import list_modules
 
@@ -192,14 +190,15 @@ def find_compilers(*paths):
     Returns:
         List of compilers found in the supplied paths
     """
-    search_commands = itertools.chain.from_iterable(
+    tags, commands = [], []
-        o.search_compiler_commands(*paths) for o in all_os_classes()
+    for o in all_os_classes():
-    )
+        t, c = o.search_compiler_commands(*paths)
+        tags.extend(t), commands.extend(c)
 
     with multiprocessing.pool.ThreadPool() as tp:
-        compilers = llnl.util.multiproc.execute(search_commands, map_fn=tp.map)
+        compiler_versions = llnl.util.multiproc.execute(commands, tp.map)
 
-    return spack.compiler.make_compiler_list(compilers)
+    return spack.compiler.make_compiler_list(tags, compiler_versions)
 
 
 def supported_compilers():

lib/spack/spack/test/compilers.py

@@ -24,7 +24,6 @@
 import spack.compilers.fj
 
 from spack.compiler import detect_version_command, Compiler
-from spack.compiler import _CompilerID, _NameVariation
 
 
 def test_get_compiler_duplicates(config):
@@ -47,21 +46,14 @@ def test_all_compilers(config):
 
 
 def test_version_detection_is_empty():
-    command = detect_version_command(
+    command = detect_version_command(lambda x: None, path='/usr/bin/gcc')
-        callback=lambda x: None, path='/usr/bin/gcc', operating_system=None,
+    expected = (None,  "Couldn't get version for compiler /usr/bin/gcc")
-        cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
+    assert command() == expected
-    )
-    assert command() is None
 
 
 def test_version_detection_is_successful():
-    command = detect_version_command(
+    command = detect_version_command(lambda x: '4.9', path='/usr/bin/gcc')
-        callback=lambda x: '4.9', path='/usr/bin/gcc', operating_system=None,
+    assert command() == (('4.9', '/usr/bin/gcc'), None)
-        cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
-    )
-    correct = (_CompilerID(None, None, '4.9'),
-               _NameVariation('', r'\d\d'), 'cc'), '/usr/bin/gcc'
-    assert command() == correct
 
 
 def test_compiler_flags_from_config_are_grouped():