zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reworked _find_matches_in_path to return lazy commands

Browse Source 
These commands, when invoked, give back a tuple (CompilerKey,path) or
None if the compiler was not found. CompilerKey is a namedtuple
containing all the information needed to identify a compiler.

find_compiler has been changed accordingly.
This commit is contained in:
Massimiliano Culpo 2018-12-20 21:49:16 +01:00
parent cae7e075a6
commit 74a13e665f
No known key found for this signature in database
GPG Key ID: D1ADB1014FF1118C
4 changed files with 122 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
lib/spack/spack/architecture.py
View File

@ -56,12 +56,12 @@
attributes front_os and back_os. The operating system as described earlier,
will be responsible for compiler detection.
"""
import collections
import os
import inspect
import itertools
import platform as py_platform
import llnl.util.multiproc as mp
import llnl.util.tty as tty
from llnl.util.lang import memoized, list_modules, key_ordering
@ -246,7 +246,8 @@ def find_compilers(self, *path_hints):
# NOTE: we import spack.compilers here to avoid init order cycles
import spack.compilers
types = spack.compilers.all_compiler_types()
compiler_lists = mp.parmap(
# TODO: was parmap before
compiler_lists = map(
lambda cmp_cls: self.find_compiler(cmp_cls, *paths),
types)
@ -266,24 +267,41 @@ def find_compiler(self, cmp_cls, *search_paths):
prefixes, suffixes, and versions. e.g., gcc-mp-4.7 would
be grouped with g++-mp-4.7 and gfortran-mp-4.7.
"""
dicts = mp.parmap(
lambda t: cmp_cls._find_matches_in_path(*t),
[('cc',) + tuple(search_paths), ('cxx',) + tuple(search_paths),
('f77',) + tuple(search_paths), ('fc',) + tuple(search_paths)])
# The commands returned here are already sorted by language
commands = cmp_cls.search_compiler_commands(*search_paths)
all_keys = set(key for d in dicts for key in d)
def invoke(f):
return f()
compilers = map(invoke, commands)
# Remove search with no results
compilers = filter(None, compilers)
# Skip compilers with unknown version
def has_known_version(x):
def has_known_version(compiler_entry):
"""Returns True if the key has not an unknown version."""
version, _, _ = x
return version != 'unknown'
compiler_key, _ = compiler_entry
return compiler_key.version != 'unknown'
valid_keys = filter(has_known_version, all_keys)
compilers = filter(has_known_version, compilers)
compilers_by_language = collections.defaultdict(dict)
language_key = lambda x: x[0].language
for language, group in itertools.groupby(compilers, language_key):
# The 'successful' list is ordered like the input paths.
# Reverse it here so that the dict creation (last insert wins)
# does not spoil the intended precedence.
compilers_by_language[language] = dict(reversed(list(group)))
dicts = [compilers_by_language[language]
for language in ('cc', 'cxx', 'f77', 'fc')]
valid_keys = set(key for d in dicts for key in d)
compilers = {}
for k in valid_keys:
ver, _, _ = k
ver = k.version
paths = tuple(pn.get(k, None) for pn in dicts)
spec = spack.spec.CompilerSpec(cmp_cls.name, ver)

86
lib/spack/spack/compiler.py
View File

@ -3,18 +3,18 @@
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import collections
import os
import re
import itertools
import llnl.util.lang
import llnl.util.tty as tty
import llnl.util.multiproc as mp
import spack.error
import spack.spec
import spack.architecture
from spack.util.executable import Executable, ProcessError
import spack.util.executable
__all__ = ['Compiler']
@ -249,19 +249,11 @@ def fc_version(cls, fc):
return cls.default_version(fc)
@classmethod
def _find_matches_in_path(cls, compiler_language, *search_paths):
"""Finds compilers for a given language in the paths supplied.
Looks for all combinations of ``compiler_names`` with the
``prefixes`` and ``suffixes`` defined for this compiler
class. If any compilers match the compiler_names,
prefixes, or suffixes, uses ``detect_version`` to figure
out what version the compiler is.
def search_compiler_commands(cls, *search_paths):
"""Returns a list of commands that, when invoked, search for compilers
in the paths supplied.
Args:
compiler_language (str): language of the compiler (either
'cc', 'cxx', 'f77' or 'fc')
*search_paths (list of paths): paths where to look for a
compiler
@ -276,11 +268,16 @@ def is_accessible_dir(x):
# Select accessible directories
search_directories = filter(is_accessible_dir, search_paths)
# Get compiler names and the callback to detect their versions
compiler_names = getattr(cls, '{0}_names'.format(compiler_language))
detect_version = getattr(cls, '{0}_version'.format(compiler_language))
search_args = []
for language in ('cc', 'cxx', 'f77', 'fc'):
# Compile all the regular expressions used for files beforehand
# Get compiler names and the callback to detect their versions
compiler_names = getattr(cls, '{0}_names'.format(language))
detect_version = getattr(cls, '{0}_version'.format(language))
# Compile all the regular expressions used for files beforehand.
# This searches for any combination of <prefix><name><suffix>
# defined for the compiler
prefixes = [''] + cls.prefixes
suffixes = [''] + cls.suffixes
regexp_fmt = r'^({0}){1}({2})$'
@ -291,7 +288,6 @@ def is_accessible_dir(x):
]
# Select only the files matching a regexp
checks = []
for d in search_directories:
# Only select actual files, use the full path
files = filter(
@ -302,17 +298,12 @@ def is_accessible_dir(x):
for regexp in search_regexps:
match = regexp.match(file)
if match:
key = (full_path,) + match.groups() + (detect_version,)
checks.append(key)
key = (detect_version, full_path, cls, language) \
+ tuple(match.groups())
search_args.append(key)
successful = [k for k in mp.parmap(_get_versioned_tuple, checks)
if k is not None]
# The 'successful' list is ordered like the input paths.
# Reverse it here so that the dict creation (last insert wins)
# does not spoil the intented precedence.
successful.reverse()
return dict(((v, p, s), path) for v, p, s, path in successful)
commands = [detect_version_command(*args) for args in search_args]
return commands
def setup_custom_environment(self, pkg, env):
"""Set any environment variables necessary to use the compiler."""
@ -330,26 +321,49 @@ def __str__(self):
str(self.operating_system)))))
def _get_versioned_tuple(compiler_check_tuple):
full_path, prefix, suffix, detect_version = compiler_check_tuple
CompilerKey = collections.namedtuple('CompilerKey', [
'os', 'cmp_cls', 'language', 'version', 'prefix', 'suffix'
])
def detect_version_command(callback, path, cmp_cls, lang, prefix, suffix):
"""Returns a command that, when invoked, searches for a compiler and
detects its version.
Args:
callback (callable): function that given the full path to search
returns a tuple of (CompilerKey, full path) or None
path (path): absolute path to search
cmp_cls (Compiler): compiler class for this specific compiler
lang (str): language of the compiler
prefix (str): prefix of the compiler name
suffix (str): suffix of the compiler name
Returns:
Callable to be invoked.
"""
def _detect_version():
try:
version = detect_version(full_path)
version = callback(path)
if (not version) or (not str(version).strip()):
tty.debug(
"Couldn't get version for compiler %s" % full_path)
"Couldn't get version for compiler %s" % path)
return None
return (version, prefix, suffix, full_path)
except ProcessError as e:
return CompilerKey(
None, cmp_cls, lang, version, prefix, suffix
), path
except spack.util.executable.ProcessError as e:
tty.debug(
"Couldn't get version for compiler %s" % full_path, e)
"Couldn't get version for compiler %s" % path, e)
return None
except Exception as e:
# Catching "Exception" here is fine because it just
# means something went wrong running a candidate executable.
tty.debug("Error while executing candidate compiler %s"
% full_path,
% path,
"%s: %s" % (e.__class__.__name__, e))
return None
return _detect_version
class CompilerAccessError(spack.error.SpackError):

4
lib/spack/spack/operating_systems/cnl.py
View File

@ -6,7 +6,6 @@
import re
import llnl.util.tty as tty
import llnl.util.multiproc as mp
from spack.architecture import OperatingSystem
from spack.util.module_cmd import module
@ -40,7 +39,8 @@ def find_compilers(self, *paths):
import spack.compilers
types = spack.compilers.all_compiler_types()
compiler_lists = mp.parmap(
# TODO: was parmap before
compiler_lists = map(
lambda cmp_cls: self.find_compiler(cmp_cls, *paths), types)
# ensure all the version calls we made are cached in the parent

20
lib/spack/spack/test/compilers.py
View File

@ -23,7 +23,7 @@
import spack.compilers.xl_r
import spack.compilers.fj
from spack.compiler import _get_versioned_tuple, Compiler
from spack.compiler import detect_version_command, Compiler, CompilerKey
def test_get_compiler_duplicates(config):
@ -46,16 +46,20 @@ def test_all_compilers(config):
def test_version_detection_is_empty():
no_version = lambda x: None
compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', no_version)
assert not _get_versioned_tuple(compiler_check_tuple)
command = detect_version_command(
callback=lambda x: None, path='/usr/bin/gcc', cmp_cls=None,
lang='cc', prefix='', suffix=r'\d\d'
)
assert command() is None
def test_version_detection_is_successful():
version = lambda x: '4.9'
compiler_check_tuple = ('/usr/bin/gcc', '', r'\d\d', version)
assert _get_versioned_tuple(compiler_check_tuple) == (
'4.9', '', r'\d\d', '/usr/bin/gcc')
command = detect_version_command(
callback=lambda x: '4.9', path='/usr/bin/gcc', cmp_cls=None,
lang='cc', prefix='', suffix=r'\d\d'
)
correct = CompilerKey(None, None, 'cc', '4.9', '', r'\d\d'), '/usr/bin/gcc'
assert command() == correct
def test_compiler_flags_from_config_are_grouped():