diff --git a/lib/spack/llnl/util/multiproc.py b/lib/spack/llnl/util/multiproc.py index 672413d0fb3..dc778b6f76e 100644 --- a/lib/spack/llnl/util/multiproc.py +++ b/lib/spack/llnl/util/multiproc.py @@ -36,18 +36,20 @@ def invoke(f): return f() -def execute(command_list, executor=map): +def execute(command_list, map_fn=map, transformation=invoke): """Execute a list of packaged commands and return their result. Args: command_list: list of commands to be executed - executor: object that execute each command. Must have the - same semantic as ``map``. + map_fn: object that execute each command. Must have the + same semantic as ``map`` + transformation: callable invoked on each item to construct + the output list Returns: List of results """ - return executor(invoke, command_list) + return map_fn(transformation, command_list) class Barrier: diff --git a/lib/spack/spack/architecture.py b/lib/spack/spack/architecture.py index 80e70300da4..c90be9f1d72 100644 --- a/lib/spack/spack/architecture.py +++ b/lib/spack/spack/architecture.py @@ -60,7 +60,6 @@ import inspect import itertools -import llnl.util.multiproc import llnl.util.tty as tty from llnl.util.lang import memoized, list_modules, key_ordering @@ -255,17 +254,6 @@ def search_compiler_commands(self, *path_hints): ) return commands - def find_compilers(self, *path_hints): - """ - Return a list of compilers found in the supplied paths. - This invokes the find() method for each Compiler class, - and appends the compilers detected to a list. - """ - commands = self.search_compiler_commands(*path_hints) - compilers = llnl.util.multiproc.execute(commands) - compilers = spack.compiler.discard_invalid(compilers) - return spack.compiler.make_compiler_list(compilers) - def to_dict(self): return { 'name': self.name, diff --git a/lib/spack/spack/compiler.py b/lib/spack/spack/compiler.py index 1e89a3e0289..81213ee7aa9 100644 --- a/lib/spack/spack/compiler.py +++ b/lib/spack/spack/compiler.py @@ -375,7 +375,8 @@ def detect_version_command( return None -def discard_invalid(compilers): +def _discard_invalid(compilers): + """Removes invalid compilers from the list""" # Remove search with no results compilers = filter(None, compilers) @@ -389,6 +390,8 @@ def has_known_version(compiler_entry): def make_compiler_list(compilers): + compilers = _discard_invalid(compilers) + # Group by (os, compiler type, version), (prefix, suffix), language def sort_key_fn(item): key, _ = item @@ -396,6 +399,10 @@ def sort_key_fn(item): (key.prefix, key.suffix), key.language compilers_s = sorted(compilers, key=sort_key_fn) + # This dictionary is needed because a class (NOT an instance of it) + # doesn't have __lt__ or other similar functions defined. Therefore + # we sort on its string representation and need to maintain the map + # to the class here cmp_cls_d = {str(key.cmp_cls): key.cmp_cls for key, _ in compilers_s} compilers_d = {} diff --git a/lib/spack/spack/compilers/__init__.py b/lib/spack/spack/compilers/__init__.py index 314016464e5..0549d2f6988 100644 --- a/lib/spack/spack/compilers/__init__.py +++ b/lib/spack/spack/compilers/__init__.py @@ -7,8 +7,10 @@ system and configuring Spack to use multiple compilers. """ import itertools +import multiprocessing.pool import os + import llnl.util.multiproc from llnl.util.lang import list_modules @@ -193,10 +195,10 @@ def find_compilers(*paths): search_commands = itertools.chain.from_iterable( o.search_compiler_commands(*paths) for o in all_os_classes() ) - # TODO: activate multiprocessing - # with multiprocessing.Pool(processes=None) as p: - compilers = llnl.util.multiproc.execute(search_commands, executor=map) - compilers = spack.compiler.discard_invalid(compilers) + + with multiprocessing.pool.ThreadPool() as tp: + compilers = llnl.util.multiproc.execute(search_commands, map_fn=tp.map) + return spack.compiler.make_compiler_list(compilers)