diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py index 8c89b7930d1..672413d0fb3 100644 --- a/lib/spack/llnl/util/multiproc.py +++ b/lib/spack/llnl/util/multiproc.py @@ -8,25 +8,46 @@ than multiprocessing.Pool.apply() can. For example, apply() will fail to pickle functions if they're passed indirectly as parameters. """ -from multiprocessing import Process, Pipe, Semaphore, Value +import functools +from multiprocessing import Semaphore, Value -__all__ = ['spawn', 'parmap', 'Barrier'] +__all__ = ['Barrier'] -def spawn(f): - def fun(pipe, x): - pipe.send(f(x)) - pipe.close() - return fun +def deferred(func): + """Package a function call into something that can be invoked + at a later moment. + + Args: + func (callable): callable that must be deferred + + Returns: + Deferred version of the same function + """ + @functools.wraps(func) + def _impl(*args, **kwargs): + def _deferred_call(): + return func(*args, **kwargs) + return _deferred_call + return _impl -def parmap(f, elements): - pipe = [Pipe() for x in elements] - proc = [Process(target=spawn(f), args=(c, x)) - for x, (p, c) in zip(elements, pipe)] - [p.start() for p in proc] - [p.join() for p in proc] - return [p.recv() for (p, c) in pipe] +def invoke(f): + return f() + + +def execute(command_list, executor=map): + """Execute a list of packaged commands and return their result. + + Args: + command_list: list of commands to be executed + executor: object that execute each command. Must have the + same semantic as ``map``. + + Returns: + List of results + """ + return executor(invoke, command_list) class Barrier: diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index c9594e9b4e6..80e70300da4 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -56,15 +56,15 @@ attributes front_os and back_os. The operating system as described earlier, will be responsible for compiler detection. """ -import collections import os import inspect import itertools -import platform as py_platform +import llnl.util.multiproc import llnl.util.tty as tty from llnl.util.lang import memoized, list_modules, key_ordering +import spack.compiler import spack.paths import spack.error as serr from spack.util.naming import mod_to_class @@ -230,7 +230,30 @@ def __repr__(self): return self.__str__() def _cmp_key(self): - return (self.name, self.version) + return self.name, self.version + + def search_compiler_commands(self, *path_hints): + """Returns a list of commands that, when invoked, search for + compilers tied to this OS. + + Args: + *path_hints (list of paths): path where to look for compilers + + Returns: + List of callable functions. + """ + # Turn the path hints into paths that are to be searched + paths = executable_search_paths(path_hints or get_path('PATH')) + + # NOTE: we import spack.compilers here to avoid init order cycles + import spack.compilers + + commands = [] + for compiler_cls in spack.compilers.all_compiler_types(): + commands.extend( + compiler_cls.search_compiler_commands(self, *paths) + ) + return commands def find_compilers(self, *path_hints): """ @@ -238,89 +261,10 @@ def find_compilers(self, *path_hints): This invokes the find() method for each Compiler class, and appends the compilers detected to a list. """ - paths = executable_search_paths(path_hints or get_path('PATH')) - - # Once the paths are cleaned up, do a search for each type of - # compiler. We can spawn a bunch of parallel searches to reduce - # the overhead of spelunking all these directories. - # NOTE: we import spack.compilers here to avoid init order cycles - import spack.compilers - types = spack.compilers.all_compiler_types() - # TODO: was parmap before - compiler_lists = map( - lambda cmp_cls: self.find_compiler(cmp_cls, *paths), - types) - - # ensure all the version calls we made are cached in the parent - # process, as well. This speeds up Spack a lot. - clist = [comp for cl in compiler_lists for comp in cl] - return clist - - def find_compiler(self, cmp_cls, *search_paths): - """Try to find the given type of compiler in the user's - environment. For each set of compilers found, this returns - compiler objects with the cc, cxx, f77, fc paths and the - version filled in. - - This will search for compilers with the names in cc_names, - cxx_names, etc. and it will group them if they have common - prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would - be grouped with g++-mp-4.7 and gfortran-mp-4.7. - """ - # The commands returned here are already sorted by language - commands = cmp_cls.search_compiler_commands(*search_paths) - - def invoke(f): - return f() - - compilers = map(invoke, commands) - - # Remove search with no results - compilers = filter(None, compilers) - - # Skip compilers with unknown version - def has_known_version(compiler_entry): - """Returns True if the key has not an unknown version.""" - compiler_key, _ = compiler_entry - return compiler_key.version != 'unknown' - - compilers = filter(has_known_version, compilers) - - compilers_by_language = collections.defaultdict(dict) - language_key = lambda x: x[0].language - for language, group in itertools.groupby(compilers, language_key): - # The 'successful' list is ordered like the input paths. - # Reverse it here so that the dict creation (last insert wins) - # does not spoil the intended precedence. - compilers_by_language[language] = dict(reversed(list(group))) - - dicts = [compilers_by_language[language] - for language in ('cc', 'cxx', 'f77', 'fc')] - - valid_keys = set(key for d in dicts for key in d) - - compilers = {} - for k in valid_keys: - ver = k.version - - paths = tuple(pn.get(k, None) for pn in dicts) - spec = spack.spec.CompilerSpec(cmp_cls.name, ver) - - if ver in compilers: - prev = compilers[ver] - - # prefer the one with more compilers. - prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc] - newcount = len([p for p in paths if p is not None]) - prevcount = len([p for p in prev_paths if p is not None]) - - # Don't add if it's not an improvement over prev compiler. - if newcount <= prevcount: - continue - - compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths) - - return list(compilers.values()) + commands = self.search_compiler_commands(*path_hints) + compilers = llnl.util.multiproc.execute(commands) + compilers = spack.compiler.discard_invalid(compilers) + return spack.compiler.make_compiler_list(compilers) def to_dict(self): return { diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 19e526a4df8..1e89a3e0289 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -8,7 +8,10 @@ import re import itertools +import platform as py_platform + import llnl.util.lang +import llnl.util.multiproc import llnl.util.tty as tty import spack.error @@ -249,11 +252,12 @@ def fc_version(cls, fc): return cls.default_version(fc) @classmethod - def search_compiler_commands(cls, *search_paths): + def search_compiler_commands(cls, operating_system, *search_paths): """Returns a list of commands that, when invoked, search for compilers in the paths supplied. Args: + operating_system (OperatingSystem): the OS requesting the search *search_paths (list of paths): paths where to look for a compiler @@ -266,7 +270,7 @@ def is_accessible_dir(x): return os.path.isdir(x) and os.access(x, os.R_OK | os.X_OK) # Select accessible directories - search_directories = filter(is_accessible_dir, search_paths) + search_directories = list(filter(is_accessible_dir, search_paths)) search_args = [] for language in ('cc', 'cxx', 'f77', 'fc'): @@ -298,12 +302,15 @@ def is_accessible_dir(x): for regexp in search_regexps: match = regexp.match(file) if match: - key = (detect_version, full_path, cls, language) \ - + tuple(match.groups()) + key = (detect_version, full_path, operating_system, + cls, language) + tuple(match.groups()) search_args.append(key) - commands = [detect_version_command(*args) for args in search_args] - return commands + # The 'successful' list is ordered like the input paths. + # Reverse it here so that the dict creation (last insert wins) + # does not spoil the intended precedence. + return [detect_version_command(*args) + for args in reversed(search_args)] def setup_custom_environment(self, pkg, env): """Set any environment variables necessary to use the compiler.""" @@ -326,44 +333,98 @@ def __str__(self): ]) -def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix): - """Returns a command that, when invoked, searches for a compiler and - detects its version. +@llnl.util.multiproc.deferred +def detect_version_command( + callback, path, operating_system, cmp_cls, lang, prefix, suffix +): + """Search for a compiler and eventually detect its version. Args: callback (callable): function that given the full path to search returns a tuple of (CompilerKey, full path) or None path (path): absolute path to search + operating_system (OperatingSystem): the OS for which we are + looking for a compiler cmp_cls (Compiler): compiler class for this specific compiler lang (str): language of the compiler prefix (str): prefix of the compiler name suffix (str): suffix of the compiler name Returns: - Callable to be invoked. + A (CompilerKey, path) tuple if anything is found, else None """ - def _detect_version(): - try: - version = callback(path) - if (not version) or (not str(version).strip()): - tty.debug( - "Couldn't get version for compiler %s" % path) - return None - return CompilerKey( - None, cmp_cls, lang, version, prefix, suffix - ), path - except spack.util.executable.ProcessError as e: + try: + version = callback(path) + if (not version) or (not str(version).strip()): tty.debug( - "Couldn't get version for compiler %s" % path, e) + "Couldn't get version for compiler %s" % path) return None - except Exception as e: - # Catching "Exception" here is fine because it just - # means something went wrong running a candidate executable. - tty.debug("Error while executing candidate compiler %s" - % path, - "%s: %s" % (e.__class__.__name__, e)) - return None - return _detect_version + return CompilerKey( + operating_system, cmp_cls, lang, version, prefix, suffix + ), path + except spack.util.executable.ProcessError as e: + tty.debug( + "Couldn't get version for compiler %s" % path, e) + return None + except Exception as e: + # Catching "Exception" here is fine because it just + # means something went wrong running a candidate executable. + tty.debug("Error while executing candidate compiler %s" + % path, + "%s: %s" % (e.__class__.__name__, e)) + return None + + +def discard_invalid(compilers): + # Remove search with no results + compilers = filter(None, compilers) + + # Skip compilers with unknown version + def has_known_version(compiler_entry): + """Returns True if the key has not an unknown version.""" + compiler_key, _ = compiler_entry + return compiler_key.version != 'unknown' + + return filter(has_known_version, compilers) + + +def make_compiler_list(compilers): + # Group by (os, compiler type, version), (prefix, suffix), language + def sort_key_fn(item): + key, _ = item + return (key.os, str(key.cmp_cls), key.version), \ + (key.prefix, key.suffix), key.language + + compilers_s = sorted(compilers, key=sort_key_fn) + cmp_cls_d = {str(key.cmp_cls): key.cmp_cls for key, _ in compilers_s} + + compilers_d = {} + for sort_key, group in itertools.groupby(compilers_s, sort_key_fn): + compiler_entry, ps, language = sort_key + by_compiler_entry = compilers_d.setdefault(compiler_entry, {}) + by_ps = by_compiler_entry.setdefault(ps, {}) + by_ps[language] = list(x[1] for x in group).pop() + + # For each (os, compiler type, version) select the compiler + # with most entries and add it to a list + compilers = [] + for compiler_entry, by_compiler_entry in compilers_d.items(): + # Select the (prefix, suffix) match with most entries + max_lang, max_ps = max( + (len(by_compiler_entry[ps]), ps) for ps in by_compiler_entry + ) + + # Add it to the list of compilers + operating_system, cmp_cls_key, version = compiler_entry + cmp_cls = cmp_cls_d[cmp_cls_key] + spec = spack.spec.CompilerSpec(cmp_cls.name, version) + paths = [by_compiler_entry[max_ps].get(language, None) + for language in ('cc', 'cxx', 'f77', 'fc')] + compilers.append( + cmp_cls(spec, operating_system, py_platform.machine(), paths) + ) + + return compilers class CompilerAccessError(spack.error.SpackError): diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 5910cd69844..314016464e5 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -9,6 +9,7 @@ import itertools import os +import llnl.util.multiproc from llnl.util.lang import list_modules import spack.paths @@ -189,9 +190,14 @@ def find_compilers(*paths): Returns: List of compilers found in the supplied paths """ - return list(itertools.chain.from_iterable( - o.find_compilers(*paths) for o in all_os_classes() - )) + search_commands = itertools.chain.from_iterable( + o.search_compiler_commands(*paths) for o in all_os_classes() + ) + # TODO: activate multiprocessing + # with multiprocessing.Pool(processes=None) as p: + compilers = llnl.util.multiproc.execute(search_commands, executor=map) + compilers = spack.compiler.discard_invalid(compilers) + return spack.compiler.make_compiler_list(compilers) def supported_compilers(): diff --git a/lib/spack/spack/test/compilers.py b/lib/spack/spack/test/compilers.py index c279129fbd8..c2139ccd7a6 100644 --- a/lib/spack/spack/test/compilers.py +++ b/lib/spack/spack/test/compilers.py @@ -47,16 +47,16 @@ def test_all_compilers(config): def test_version_detection_is_empty(): command = detect_version_command( - callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None, - lang='cc', prefix='', suffix=r'\d\d' + callback=lambda x: None, path='/usr/bin/gcc', operating_system=None, + cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d' ) assert command() is None def test_version_detection_is_successful(): command = detect_version_command( - callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None, - lang='cc', prefix='', suffix=r'\d\d' + callback=lambda x: '4.9', path='/usr/bin/gcc', operating_system=None, + cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d' ) correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc' assert command() == correct