diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index d1d7e9f5f50..c9594e9b4e6 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -56,12 +56,12 @@ attributes front_os and back_os. The operating system as described earlier, will be responsible for compiler detection. """ +import collections import os import inspect import itertools import platform as py_platform -import llnl.util.multiproc as mp import llnl.util.tty as tty from llnl.util.lang import memoized, list_modules, key_ordering @@ -246,7 +246,8 @@ def find_compilers(self, *path_hints): # NOTE: we import spack.compilers here to avoid init order cycles import spack.compilers types = spack.compilers.all_compiler_types() - compiler_lists = mp.parmap( + # TODO: was parmap before + compiler_lists = map( lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) @@ -266,24 +267,41 @@ def find_compiler(self, cmp_cls, *search_paths): prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would be grouped with g++-mp-4.7 and gfortran-mp-4.7. """ - dicts = mp.parmap( - lambda t: cmp_cls._find_matches_in_path(*t), - [('cc',) + tuple(search_paths), ('cxx',) + tuple(search_paths), - ('f77',) + tuple(search_paths), ('fc',) + tuple(search_paths)]) + # The commands returned here are already sorted by language + commands = cmp_cls.search_compiler_commands(*search_paths) - all_keys = set(key for d in dicts for key in d) + def invoke(f): + return f() + + compilers = map(invoke, commands) + + # Remove search with no results + compilers = filter(None, compilers) # Skip compilers with unknown version - def has_known_version(x): + def has_known_version(compiler_entry): """Returns True if the key has not an unknown version.""" - version, _, _ = x - return version != 'unknown' + compiler_key, _ = compiler_entry + return compiler_key.version != 'unknown' - valid_keys = filter(has_known_version, all_keys) + compilers = filter(has_known_version, compilers) + + compilers_by_language = collections.defaultdict(dict) + language_key = lambda x: x[0].language + for language, group in itertools.groupby(compilers, language_key): + # The 'successful' list is ordered like the input paths. + # Reverse it here so that the dict creation (last insert wins) + # does not spoil the intended precedence. + compilers_by_language[language] = dict(reversed(list(group))) + + dicts = [compilers_by_language[language] + for language in ('cc', 'cxx', 'f77', 'fc')] + + valid_keys = set(key for d in dicts for key in d) compilers = {} for k in valid_keys: - ver, _, _ = k + ver = k.version paths = tuple(pn.get(k, None) for pn in dicts) spec = spack.spec.CompilerSpec(cmp_cls.name, ver) diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 3ad6e5b0db8..19e526a4df8 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -3,18 +3,18 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) +import collections import os import re import itertools import llnl.util.lang import llnl.util.tty as tty -import llnl.util.multiproc as mp import spack.error import spack.spec import spack.architecture -from spack.util.executable import Executable, ProcessError +import spack.util.executable __all__ = ['Compiler'] @@ -249,19 +249,11 @@ def fc_version(cls, fc): return cls.default_version(fc) @classmethod - def _find_matches_in_path(cls, compiler_language, *search_paths): - """Finds compilers for a given language in the paths supplied. - - Looks for all combinations of ``compiler_names`` with the - ``prefixes`` and ``suffixes`` defined for this compiler - class. If any compilers match the compiler_names, - prefixes, or suffixes, uses ``detect_version`` to figure - out what version the compiler is. + def search_compiler_commands(cls, *search_paths): + """Returns a list of commands that, when invoked, search for compilers + in the paths supplied. Args: - compiler_language (str): language of the compiler (either - 'cc', 'cxx', 'f77' or 'fc') - *search_paths (list of paths): paths where to look for a compiler @@ -276,43 +268,42 @@ def is_accessible_dir(x): # Select accessible directories search_directories = filter(is_accessible_dir, search_paths) - # Get compiler names and the callback to detect their versions - compiler_names = getattr(cls, '{0}_names'.format(compiler_language)) - detect_version = getattr(cls, '{0}_version'.format(compiler_language)) + search_args = [] + for language in ('cc', 'cxx', 'f77', 'fc'): - # Compile all the regular expressions used for files beforehand - prefixes = [''] + cls.prefixes - suffixes = [''] + cls.suffixes - regexp_fmt = r'^({0}){1}({2})$' - search_regexps = [ - re.compile(regexp_fmt.format(prefix, re.escape(name), suffix)) - for prefix, name, suffix in - itertools.product(prefixes, compiler_names, suffixes) - ] + # Get compiler names and the callback to detect their versions + compiler_names = getattr(cls, '{0}_names'.format(language)) + detect_version = getattr(cls, '{0}_version'.format(language)) - # Select only the files matching a regexp - checks = [] - for d in search_directories: - # Only select actual files, use the full path - files = filter( - os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)] - ) - for full_path in files: - file = os.path.basename(full_path) - for regexp in search_regexps: - match = regexp.match(file) - if match: - key = (full_path,) + match.groups() + (detect_version,) - checks.append(key) + # Compile all the regular expressions used for files beforehand. + # This searches for any combination of + # defined for the compiler + prefixes = [''] + cls.prefixes + suffixes = [''] + cls.suffixes + regexp_fmt = r'^({0}){1}({2})$' + search_regexps = [ + re.compile(regexp_fmt.format(prefix, re.escape(name), suffix)) + for prefix, name, suffix in + itertools.product(prefixes, compiler_names, suffixes) + ] - successful = [k for k in mp.parmap(_get_versioned_tuple, checks) - if k is not None] + # Select only the files matching a regexp + for d in search_directories: + # Only select actual files, use the full path + files = filter( + os.path.isfile, [os.path.join(d, f) for f in os.listdir(d)] + ) + for full_path in files: + file = os.path.basename(full_path) + for regexp in search_regexps: + match = regexp.match(file) + if match: + key = (detect_version, full_path, cls, language) \ + + tuple(match.groups()) + search_args.append(key) - # The 'successful' list is ordered like the input paths. - # Reverse it here so that the dict creation (last insert wins) - # does not spoil the intented precedence. - successful.reverse() - return dict(((v, p, s), path) for v, p, s, path in successful) + commands = [detect_version_command(*args) for args in search_args] + return commands def setup_custom_environment(self, pkg, env): """Set any environment variables necessary to use the compiler.""" @@ -330,26 +321,49 @@ def __str__(self): str(self.operating_system))))) -def _get_versioned_tuple(compiler_check_tuple): - full_path, prefix, suffix, detect_version = compiler_check_tuple - try: - version = detect_version(full_path) - if (not version) or (not str(version).strip()): +CompilerKey = collections.namedtuple('CompilerKey', [ + 'os', 'cmp_cls', 'language', 'version', 'prefix', 'suffix' +]) + + +def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix): + """Returns a command that, when invoked, searches for a compiler and + detects its version. + + Args: + callback (callable): function that given the full path to search + returns a tuple of (CompilerKey, full path) or None + path (path): absolute path to search + cmp_cls (Compiler): compiler class for this specific compiler + lang (str): language of the compiler + prefix (str): prefix of the compiler name + suffix (str): suffix of the compiler name + + Returns: + Callable to be invoked. + """ + def _detect_version(): + try: + version = callback(path) + if (not version) or (not str(version).strip()): + tty.debug( + "Couldn't get version for compiler %s" % path) + return None + return CompilerKey( + None, cmp_cls, lang, version, prefix, suffix + ), path + except spack.util.executable.ProcessError as e: tty.debug( - "Couldn't get version for compiler %s" % full_path) + "Couldn't get version for compiler %s" % path, e) return None - return (version, prefix, suffix, full_path) - except ProcessError as e: - tty.debug( - "Couldn't get version for compiler %s" % full_path, e) - return None - except Exception as e: - # Catching "Exception" here is fine because it just - # means something went wrong running a candidate executable. - tty.debug("Error while executing candidate compiler %s" - % full_path, - "%s: %s" % (e.__class__.__name__, e)) - return None + except Exception as e: + # Catching "Exception" here is fine because it just + # means something went wrong running a candidate executable. + tty.debug("Error while executing candidate compiler %s" + % path, + "%s: %s" % (e.__class__.__name__, e)) + return None + return _detect_version class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/operating_systems/cnl.py b/lib/spack/spack/operating_systems/cnl.py index 59b6e980a22..78543366088 100644 --- a/lib/spack/spack/operating_systems/cnl.py +++ b/lib/spack/spack/operating_systems/cnl.py @@ -6,7 +6,6 @@ import re import llnl.util.tty as tty -import llnl.util.multiproc as mp from spack.architecture import OperatingSystem from spack.util.module_cmd import module @@ -40,7 +39,8 @@ def find_compilers(self, *paths): import spack.compilers types = spack.compilers.all_compiler_types() - compiler_lists = mp.parmap( + # TODO: was parmap before + compiler_lists = map( lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types) # ensure all the version calls we made are cached in the parent diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index 4129503dbbf..c279129fbd8 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -23,7 +23,7 @@ import spack.compilers.xl_r import spack.compilers.fj -from spack.compiler import _get_versioned_tuple, Compiler +from spack.compiler import detect_version_command, Compiler, CompilerKey def test_get_compiler_duplicates(config): @@ -46,16 +46,20 @@ def test_all_compilers(config): def test_version_detection_is_empty(): - no_version = lambda x: None - compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', no_version) - assert not _get_versioned_tuple(compiler_check_tuple) + command = detect_version_command( + callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None, + lang='cc', prefix='', suffix=r'\d\d' + ) + assert command() is None def test_version_detection_is_successful(): - version = lambda x: '4.9' - compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', version) - assert _get_versioned_tuple(compiler_check_tuple) == ( - '4.9', '', r'\d\d', '/usr/bin/gcc') + command = detect_version_command( + callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None, + lang='cc', prefix='', suffix=r'\d\d' + ) + correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc' + assert command() == correct def test_compiler_flags_from_config_are_grouped():