Finding compilers executes all queries first and then process them
The function calls to find compilers have been reorganized to: 1. Collect all the queries that are needed to detect compiler's version 2. Execute them on demand 3. Process the results and register compilers
This commit is contained in:
Massimiliano Culpo
parent
74a13e665f
commit
db6ef96e87
@ -8,25 +8,46 @@
|
||||
than multiprocessing.Pool.apply() can. For example, apply() will fail
|
||||
to pickle functions if they're passed indirectly as parameters.
|
||||
"""
|
||||
from multiprocessing import Process, Pipe, Semaphore, Value
|
||||
import functools
|
||||
from multiprocessing import Semaphore, Value
|
||||
|
||||
__all__ = ['spawn', 'parmap', 'Barrier']
|
||||
__all__ = ['Barrier']
|
||||
|
||||
|
||||
def spawn(f):
|
||||
def fun(pipe, x):
|
||||
pipe.send(f(x))
|
||||
pipe.close()
|
||||
return fun
|
||||
def deferred(func):
|
||||
"""Package a function call into something that can be invoked
|
||||
at a later moment.
|
||||
|
||||
Args:
|
||||
func (callable): callable that must be deferred
|
||||
|
||||
Returns:
|
||||
Deferred version of the same function
|
||||
"""
|
||||
@functools.wraps(func)
|
||||
def _impl(*args, **kwargs):
|
||||
def _deferred_call():
|
||||
return func(*args, **kwargs)
|
||||
return _deferred_call
|
||||
return _impl
|
||||
|
||||
|
||||
def parmap(f, elements):
|
||||
pipe = [Pipe() for x in elements]
|
||||
proc = [Process(target=spawn(f), args=(c, x))
|
||||
for x, (p, c) in zip(elements, pipe)]
|
||||
[p.start() for p in proc]
|
||||
[p.join() for p in proc]
|
||||
return [p.recv() for (p, c) in pipe]
|
||||
def invoke(f):
|
||||
return f()
|
||||
|
||||
|
||||
def execute(command_list, executor=map):
|
||||
"""Execute a list of packaged commands and return their result.
|
||||
|
||||
Args:
|
||||
command_list: list of commands to be executed
|
||||
executor: object that execute each command. Must have the
|
||||
same semantic as ``map``.
|
||||
|
||||
Returns:
|
||||
List of results
|
||||
"""
|
||||
return executor(invoke, command_list)
|
||||
|
||||
|
||||
class Barrier:
|
||||
|
||||
@ -56,15 +56,15 @@
|
||||
attributes front_os and back_os. The operating system as described earlier,
|
||||
will be responsible for compiler detection.
|
||||
"""
|
||||
import collections
|
||||
import os
|
||||
import inspect
|
||||
import itertools
|
||||
import platform as py_platform
|
||||
|
||||
import llnl.util.multiproc
|
||||
import llnl.util.tty as tty
|
||||
from llnl.util.lang import memoized, list_modules, key_ordering
|
||||
|
||||
import spack.compiler
|
||||
import spack.paths
|
||||
import spack.error as serr
|
||||
from spack.util.naming import mod_to_class
|
||||
@ -230,7 +230,30 @@ def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
def _cmp_key(self):
|
||||
return (self.name, self.version)
|
||||
return self.name, self.version
|
||||
|
||||
def search_compiler_commands(self, *path_hints):
|
||||
"""Returns a list of commands that, when invoked, search for
|
||||
compilers tied to this OS.
|
||||
|
||||
Args:
|
||||
*path_hints (list of paths): path where to look for compilers
|
||||
|
||||
Returns:
|
||||
List of callable functions.
|
||||
"""
|
||||
# Turn the path hints into paths that are to be searched
|
||||
paths = executable_search_paths(path_hints or get_path('PATH'))
|
||||
|
||||
# NOTE: we import spack.compilers here to avoid init order cycles
|
||||
import spack.compilers
|
||||
|
||||
commands = []
|
||||
for compiler_cls in spack.compilers.all_compiler_types():
|
||||
commands.extend(
|
||||
compiler_cls.search_compiler_commands(self, *paths)
|
||||
)
|
||||
return commands
|
||||
|
||||
def find_compilers(self, *path_hints):
|
||||
"""
|
||||
@ -238,89 +261,10 @@ def find_compilers(self, *path_hints):
|
||||
This invokes the find() method for each Compiler class,
|
||||
and appends the compilers detected to a list.
|
||||
"""
|
||||
paths = executable_search_paths(path_hints or get_path('PATH'))
|
||||
|
||||
# Once the paths are cleaned up, do a search for each type of
|
||||
# compiler. We can spawn a bunch of parallel searches to reduce
|
||||
# the overhead of spelunking all these directories.
|
||||
# NOTE: we import spack.compilers here to avoid init order cycles
|
||||
import spack.compilers
|
||||
types = spack.compilers.all_compiler_types()
|
||||
# TODO: was parmap before
|
||||
compiler_lists = map(
|
||||
lambda cmp_cls: self.find_compiler(cmp_cls, *paths),
|
||||
types)
|
||||
|
||||
# ensure all the version calls we made are cached in the parent
|
||||
# process, as well. This speeds up Spack a lot.
|
||||
clist = [comp for cl in compiler_lists for comp in cl]
|
||||
return clist
|
||||
|
||||
def find_compiler(self, cmp_cls, *search_paths):
|
||||
"""Try to find the given type of compiler in the user's
|
||||
environment. For each set of compilers found, this returns
|
||||
compiler objects with the cc, cxx, f77, fc paths and the
|
||||
version filled in.
|
||||
|
||||
This will search for compilers with the names in cc_names,
|
||||
cxx_names, etc. and it will group them if they have common
|
||||
prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
|
||||
be grouped with g++-mp-4.7 and gfortran-mp-4.7.
|
||||
"""
|
||||
# The commands returned here are already sorted by language
|
||||
commands = cmp_cls.search_compiler_commands(*search_paths)
|
||||
|
||||
def invoke(f):
|
||||
return f()
|
||||
|
||||
compilers = map(invoke, commands)
|
||||
|
||||
# Remove search with no results
|
||||
compilers = filter(None, compilers)
|
||||
|
||||
# Skip compilers with unknown version
|
||||
def has_known_version(compiler_entry):
|
||||
"""Returns True if the key has not an unknown version."""
|
||||
compiler_key, _ = compiler_entry
|
||||
return compiler_key.version != 'unknown'
|
||||
|
||||
compilers = filter(has_known_version, compilers)
|
||||
|
||||
compilers_by_language = collections.defaultdict(dict)
|
||||
language_key = lambda x: x[0].language
|
||||
for language, group in itertools.groupby(compilers, language_key):
|
||||
# The 'successful' list is ordered like the input paths.
|
||||
# Reverse it here so that the dict creation (last insert wins)
|
||||
# does not spoil the intended precedence.
|
||||
compilers_by_language[language] = dict(reversed(list(group)))
|
||||
|
||||
dicts = [compilers_by_language[language]
|
||||
for language in ('cc', 'cxx', 'f77', 'fc')]
|
||||
|
||||
valid_keys = set(key for d in dicts for key in d)
|
||||
|
||||
compilers = {}
|
||||
for k in valid_keys:
|
||||
ver = k.version
|
||||
|
||||
paths = tuple(pn.get(k, None) for pn in dicts)
|
||||
spec = spack.spec.CompilerSpec(cmp_cls.name, ver)
|
||||
|
||||
if ver in compilers:
|
||||
prev = compilers[ver]
|
||||
|
||||
# prefer the one with more compilers.
|
||||
prev_paths = [prev.cc, prev.cxx, prev.f77, prev.fc]
|
||||
newcount = len([p for p in paths if p is not None])
|
||||
prevcount = len([p for p in prev_paths if p is not None])
|
||||
|
||||
# Don't add if it's not an improvement over prev compiler.
|
||||
if newcount <= prevcount:
|
||||
continue
|
||||
|
||||
compilers[ver] = cmp_cls(spec, self, py_platform.machine(), paths)
|
||||
|
||||
return list(compilers.values())
|
||||
commands = self.search_compiler_commands(*path_hints)
|
||||
compilers = llnl.util.multiproc.execute(commands)
|
||||
compilers = spack.compiler.discard_invalid(compilers)
|
||||
return spack.compiler.make_compiler_list(compilers)
|
||||
|
||||
def to_dict(self):
|
||||
return {
|
||||
|
||||
@ -8,7 +8,10 @@
|
||||
import re
|
||||
import itertools
|
||||
|
||||
import platform as py_platform
|
||||
|
||||
import llnl.util.lang
|
||||
import llnl.util.multiproc
|
||||
import llnl.util.tty as tty
|
||||
|
||||
import spack.error
|
||||
@ -249,11 +252,12 @@ def fc_version(cls, fc):
|
||||
return cls.default_version(fc)
|
||||
|
||||
@classmethod
|
||||
def search_compiler_commands(cls, *search_paths):
|
||||
def search_compiler_commands(cls, operating_system, *search_paths):
|
||||
"""Returns a list of commands that, when invoked, search for compilers
|
||||
in the paths supplied.
|
||||
|
||||
Args:
|
||||
operating_system (OperatingSystem): the OS requesting the search
|
||||
*search_paths (list of paths): paths where to look for a
|
||||
compiler
|
||||
|
||||
@ -266,7 +270,7 @@ def is_accessible_dir(x):
|
||||
return os.path.isdir(x) and os.access(x, os.R_OK | os.X_OK)
|
||||
|
||||
# Select accessible directories
|
||||
search_directories = filter(is_accessible_dir, search_paths)
|
||||
search_directories = list(filter(is_accessible_dir, search_paths))
|
||||
|
||||
search_args = []
|
||||
for language in ('cc', 'cxx', 'f77', 'fc'):
|
||||
@ -298,12 +302,15 @@ def is_accessible_dir(x):
|
||||
for regexp in search_regexps:
|
||||
match = regexp.match(file)
|
||||
if match:
|
||||
key = (detect_version, full_path, cls, language) \
|
||||
+ tuple(match.groups())
|
||||
key = (detect_version, full_path, operating_system,
|
||||
cls, language) + tuple(match.groups())
|
||||
search_args.append(key)
|
||||
|
||||
commands = [detect_version_command(*args) for args in search_args]
|
||||
return commands
|
||||
# The 'successful' list is ordered like the input paths.
|
||||
# Reverse it here so that the dict creation (last insert wins)
|
||||
# does not spoil the intended precedence.
|
||||
return [detect_version_command(*args)
|
||||
for args in reversed(search_args)]
|
||||
|
||||
def setup_custom_environment(self, pkg, env):
|
||||
"""Set any environment variables necessary to use the compiler."""
|
||||
@ -326,44 +333,98 @@ def __str__(self):
|
||||
])
|
||||
|
||||
|
||||
def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix):
|
||||
"""Returns a command that, when invoked, searches for a compiler and
|
||||
detects its version.
|
||||
@llnl.util.multiproc.deferred
|
||||
def detect_version_command(
|
||||
callback, path, operating_system, cmp_cls, lang, prefix, suffix
|
||||
):
|
||||
"""Search for a compiler and eventually detect its version.
|
||||
|
||||
Args:
|
||||
callback (callable): function that given the full path to search
|
||||
returns a tuple of (CompilerKey, full path) or None
|
||||
path (path): absolute path to search
|
||||
operating_system (OperatingSystem): the OS for which we are
|
||||
looking for a compiler
|
||||
cmp_cls (Compiler): compiler class for this specific compiler
|
||||
lang (str): language of the compiler
|
||||
prefix (str): prefix of the compiler name
|
||||
suffix (str): suffix of the compiler name
|
||||
|
||||
Returns:
|
||||
Callable to be invoked.
|
||||
A (CompilerKey, path) tuple if anything is found, else None
|
||||
"""
|
||||
def _detect_version():
|
||||
try:
|
||||
version = callback(path)
|
||||
if (not version) or (not str(version).strip()):
|
||||
tty.debug(
|
||||
"Couldn't get version for compiler %s" % path)
|
||||
return None
|
||||
return CompilerKey(
|
||||
None, cmp_cls, lang, version, prefix, suffix
|
||||
), path
|
||||
except spack.util.executable.ProcessError as e:
|
||||
try:
|
||||
version = callback(path)
|
||||
if (not version) or (not str(version).strip()):
|
||||
tty.debug(
|
||||
"Couldn't get version for compiler %s" % path, e)
|
||||
"Couldn't get version for compiler %s" % path)
|
||||
return None
|
||||
except Exception as e:
|
||||
# Catching "Exception" here is fine because it just
|
||||
# means something went wrong running a candidate executable.
|
||||
tty.debug("Error while executing candidate compiler %s"
|
||||
% path,
|
||||
"%s: %s" % (e.__class__.__name__, e))
|
||||
return None
|
||||
return _detect_version
|
||||
return CompilerKey(
|
||||
operating_system, cmp_cls, lang, version, prefix, suffix
|
||||
), path
|
||||
except spack.util.executable.ProcessError as e:
|
||||
tty.debug(
|
||||
"Couldn't get version for compiler %s" % path, e)
|
||||
return None
|
||||
except Exception as e:
|
||||
# Catching "Exception" here is fine because it just
|
||||
# means something went wrong running a candidate executable.
|
||||
tty.debug("Error while executing candidate compiler %s"
|
||||
% path,
|
||||
"%s: %s" % (e.__class__.__name__, e))
|
||||
return None
|
||||
|
||||
|
||||
def discard_invalid(compilers):
|
||||
# Remove search with no results
|
||||
compilers = filter(None, compilers)
|
||||
|
||||
# Skip compilers with unknown version
|
||||
def has_known_version(compiler_entry):
|
||||
"""Returns True if the key has not an unknown version."""
|
||||
compiler_key, _ = compiler_entry
|
||||
return compiler_key.version != 'unknown'
|
||||
|
||||
return filter(has_known_version, compilers)
|
||||
|
||||
|
||||
def make_compiler_list(compilers):
|
||||
# Group by (os, compiler type, version), (prefix, suffix), language
|
||||
def sort_key_fn(item):
|
||||
key, _ = item
|
||||
return (key.os, str(key.cmp_cls), key.version), \
|
||||
(key.prefix, key.suffix), key.language
|
||||
|
||||
compilers_s = sorted(compilers, key=sort_key_fn)
|
||||
cmp_cls_d = {str(key.cmp_cls): key.cmp_cls for key, _ in compilers_s}
|
||||
|
||||
compilers_d = {}
|
||||
for sort_key, group in itertools.groupby(compilers_s, sort_key_fn):
|
||||
compiler_entry, ps, language = sort_key
|
||||
by_compiler_entry = compilers_d.setdefault(compiler_entry, {})
|
||||
by_ps = by_compiler_entry.setdefault(ps, {})
|
||||
by_ps[language] = list(x[1] for x in group).pop()
|
||||
|
||||
# For each (os, compiler type, version) select the compiler
|
||||
# with most entries and add it to a list
|
||||
compilers = []
|
||||
for compiler_entry, by_compiler_entry in compilers_d.items():
|
||||
# Select the (prefix, suffix) match with most entries
|
||||
max_lang, max_ps = max(
|
||||
(len(by_compiler_entry[ps]), ps) for ps in by_compiler_entry
|
||||
)
|
||||
|
||||
# Add it to the list of compilers
|
||||
operating_system, cmp_cls_key, version = compiler_entry
|
||||
cmp_cls = cmp_cls_d[cmp_cls_key]
|
||||
spec = spack.spec.CompilerSpec(cmp_cls.name, version)
|
||||
paths = [by_compiler_entry[max_ps].get(language, None)
|
||||
for language in ('cc', 'cxx', 'f77', 'fc')]
|
||||
compilers.append(
|
||||
cmp_cls(spec, operating_system, py_platform.machine(), paths)
|
||||
)
|
||||
|
||||
return compilers
|
||||
|
||||
|
||||
class CompilerAccessError(spack.error.SpackError):
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
import itertools
|
||||
import os
|
||||
|
||||
import llnl.util.multiproc
|
||||
from llnl.util.lang import list_modules
|
||||
|
||||
import spack.paths
|
||||
@ -189,9 +190,14 @@ def find_compilers(*paths):
|
||||
Returns:
|
||||
List of compilers found in the supplied paths
|
||||
"""
|
||||
return list(itertools.chain.from_iterable(
|
||||
o.find_compilers(*paths) for o in all_os_classes()
|
||||
))
|
||||
search_commands = itertools.chain.from_iterable(
|
||||
o.search_compiler_commands(*paths) for o in all_os_classes()
|
||||
)
|
||||
# TODO: activate multiprocessing
|
||||
# with multiprocessing.Pool(processes=None) as p:
|
||||
compilers = llnl.util.multiproc.execute(search_commands, executor=map)
|
||||
compilers = spack.compiler.discard_invalid(compilers)
|
||||
return spack.compiler.make_compiler_list(compilers)
|
||||
|
||||
|
||||
def supported_compilers():
|
||||
|
||||
@ -47,16 +47,16 @@ def test_all_compilers(config):
|
||||
|
||||
def test_version_detection_is_empty():
|
||||
command = detect_version_command(
|
||||
callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None,
|
||||
lang='cc', prefix='', suffix=r'\d\d'
|
||||
callback=lambda x: None, path='/usr/bin/gcc', operating_system=None,
|
||||
cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
|
||||
)
|
||||
assert command() is None
|
||||
|
||||
|
||||
def test_version_detection_is_successful():
|
||||
command = detect_version_command(
|
||||
callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None,
|
||||
lang='cc', prefix='', suffix=r'\d\d'
|
||||
callback=lambda x: '4.9', path='/usr/bin/gcc', operating_system=None,
|
||||
cmp_cls=None, lang='cc', prefix='', suffix=r'\d\d'
|
||||
)
|
||||
correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc'
|
||||
assert command() == correct
|
||||
|
||||
Loading…
Reference in New Issue
Block a user