diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c90be9f1d72..832e0287296 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -239,7 +239,10 @@ def search_compiler_commands(self, *path_hints): *path_hints (list of paths): path where to look for compilers Returns: - List of callable functions. + (tags, commands): ``tags`` is a list of compiler tags, containing + all the information on a compiler, but version. ``commands`` + is a list of commands that, when executed, will detect the + version of the corresponding compiler. """ # Turn the path hints into paths that are to be searched paths = executable_search_paths(path_hints or get_path('PATH')) @@ -247,12 +250,12 @@ def search_compiler_commands(self, *path_hints): # NOTE: we import spack.compilers here to avoid init order cycles import spack.compilers - commands = [] + tags, commands = [], [] for compiler_cls in spack.compilers.all_compiler_types(): - commands.extend( - compiler_cls.search_compiler_commands(self, *paths) - ) - return commands + t, c = compiler_cls.search_compiler_commands(self, *paths) + tags.extend(t), commands.extend(c) + + return tags, commands def to_dict(self): return { diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index e2e0d27bfaf..4ad49a96778 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -263,8 +263,10 @@ def search_compiler_commands(cls, operating_system, *search_paths): compiler Returns: - Dictionary with compilers grouped by (version, prefix, suffix) - tuples. + (tags, commands): ``tags`` is a list of compiler tags, containing + all the information on a compiler, but version. ``commands`` + is a list of commands that, when executed, will detect the + version of the corresponding compiler. """ def is_accessible_dir(x): """Returns True if the argument is an accessible directory.""" @@ -273,12 +275,12 @@ def is_accessible_dir(x): # Select accessible directories search_directories = list(filter(is_accessible_dir, search_paths)) - search_args = [] + tags, commands = [], [] for language in ('cc', 'cxx', 'f77', 'fc'): # Get compiler names and the callback to detect their versions compiler_names = getattr(cls, '{0}_names'.format(language)) - detect_version = getattr(cls, '{0}_version'.format(language)) + callback = getattr(cls, '{0}_version'.format(language)) # Compile all the regular expressions used for files beforehand. # This searches for any combination of @@ -303,15 +305,17 @@ def is_accessible_dir(x): for regexp in search_regexps: match = regexp.match(file) if match: - key = (detect_version, full_path, operating_system, - cls, language) + tuple(match.groups()) - search_args.append(key) + tags.append( + (_CompilerID(operating_system, cls, None), + _NameVariation(*match.groups()), language) + ) + commands.append( + detect_version_command(callback, full_path) + ) - # The 'successful' list is ordered like the input paths. # Reverse it here so that the dict creation (last insert wins) # does not spoil the intended precedence. - return [detect_version_command(*args) - for args in reversed(search_args)] + return reversed(tags), reversed(commands) def setup_custom_environment(self, pkg, env): """Set any environment variables necessary to use the compiler.""" @@ -356,62 +360,55 @@ def __hash__(self): @llnl.util.multiproc.deferred -def detect_version_command( - callback, path, operating_system, cmp_cls, lang, prefix, suffix -): - """Search for a compiler and eventually detect its version. +def detect_version_command(callback, path): + """Detects the version of a compiler at a given path. Args: - callback (callable): function that given the full path to search - returns a tuple of (CompilerKey, full path) or None + callback (callable): function that detects the version of + the compiler at ``path`` path (path): absolute path to search - operating_system (OperatingSystem): the OS for which we are - looking for a compiler - cmp_cls (Compiler): compiler class for this specific compiler - lang (str): language of the compiler - prefix (str): prefix of the compiler name - suffix (str): suffix of the compiler name Returns: - A (CompilerKey, path) tuple if anything is found, else None + (value, error): if anything is found ``value`` is a ``(version, path)`` + tuple and ``error`` is None. If ``error`` is not None, ``value`` + is meaningless and can be discarded. """ try: version = callback(path) - if (not version) or (not str(version).strip()): - tty.debug( - "Couldn't get version for compiler %s" % path) - return None - return (_CompilerID(operating_system, cmp_cls, version), - _NameVariation(prefix, suffix), lang), path + if version and str(version).strip(): + return (version, path), None + error = "Couldn't get version for compiler {0}".format(path) except spack.util.executable.ProcessError as e: - tty.debug( - "Couldn't get version for compiler %s" % path, e) - return None + error = "Couldn't get version for compiler {0}\n".format(path) + str(e) except Exception as e: # Catching "Exception" here is fine because it just # means something went wrong running a candidate executable. - tty.debug("Error while executing candidate compiler %s" - % path, - "%s: %s" % (e.__class__.__name__, e)) - return None + error = "Error while executing candidate compiler {0}" \ + "\n{1}: {2}".format(path, e.__class__.__name__, str(e)) + return None, error -def _discard_invalid(compilers): - """Removes invalid compilers from the list""" - # Remove search with no results - compilers = filter(None, compilers) +def make_compiler_list(tags, compiler_versions): + assert len(tags) == len(compiler_versions), \ + "the two arguments must have the same length" - # Skip compilers with unknown version - def has_known_version(compiler_entry): - """Returns True if the key has not an unknown version.""" - (compiler_id, _, _), _ = compiler_entry - return compiler_id.version != 'unknown' + compilers_s = [] + for (compiler_id, name_variation, lang), (return_value, error) \ + in zip(tags, compiler_versions): + # If we had an error, move to the next element + if error: + tty.debug(error) + continue - return filter(has_known_version, compilers) + # Skip unknown versions + version, path = return_value + if version == 'unknown': + continue + tag = compiler_id._replace(version=version), name_variation, lang + compilers_s.append((tag, path)) -def make_compiler_list(compilers): - compilers_s = sorted(_discard_invalid(compilers)) + compilers_s.sort() compilers_d = {} for sort_key, group in itertools.groupby(compilers_s, key=lambda x: x[0]): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 0549d2f6988..2551e6fdf5a 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -6,11 +6,9 @@ """This module contains functions related to finding compilers on the system and configuring Spack to use multiple compilers. """ -import itertools import multiprocessing.pool import os - import llnl.util.multiproc from llnl.util.lang import list_modules @@ -192,14 +190,15 @@ def find_compilers(*paths): Returns: List of compilers found in the supplied paths """ - search_commands = itertools.chain.from_iterable( - o.search_compiler_commands(*paths) for o in all_os_classes() - ) + tags, commands = [], [] + for o in all_os_classes(): + t, c = o.search_compiler_commands(*paths) + tags.extend(t), commands.extend(c) with multiprocessing.pool.ThreadPool() as tp: - compilers = llnl.util.multiproc.execute(search_commands, map_fn=tp.map) + compiler_versions = llnl.util.multiproc.execute(commands, tp.map) - return spack.compiler.make_compiler_list(compilers) + return spack.compiler.make_compiler_list(tags, compiler_versions) def supported_compilers(): diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index 050800354ad..2d3d497f248 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -24,7 +24,6 @@ import spack.compilers.fj from spack.compiler import detect_version_command, Compiler -from spack.compiler import _CompilerID, _NameVariation def test_get_compiler_duplicates(config): @@ -47,21 +46,14 @@ def test_all_compilers(config): def test_version_detection_is_empty(): - command = detect_version_command( - callback=lambda x: None, path='/usr/bin/gcc', operating_system=None, - cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d' - ) - assert command() is None + command = detect_version_command(lambda x: None, path='/usr/bin/gcc') + expected = (None, "Couldn't get version for compiler /usr/bin/gcc") + assert command() == expected def test_version_detection_is_successful(): - command = detect_version_command( - callback=lambda x: '4.9', path='/usr/bin/gcc', operating_system=None, - cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d' - ) - correct = (_CompilerID(None, None, '4.9'), - _NameVariation('', r'\d\d'), 'cc'), '/usr/bin/gcc' - assert command() == correct + command = detect_version_command(lambda x: '4.9', path='/usr/bin/gcc') + assert command() == (('4.9', '/usr/bin/gcc'), None) def test_compiler_flags_from_config_are_grouped():