zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compiler support now uses configuration files.

Browse Source 
- no more need for compiler python files.

- Default compilers are found in user's environment and added
	to ~/.spackconfig automatically

- User can add new compilers by editing configuration file
This commit is contained in:
Todd Gamblin 2014-06-19 10:34:21 -07:00
parent b6740cf1d1
commit 3653cfe6f0
15 changed files with 287 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
lib/spack/env/cc vendored
View File

@ -27,10 +27,11 @@ spack_env_path = get_path("SPACK_ENV_PATH")
spack_debug_log_dir = get_env_var("SPACK_DEBUG_LOG_DIR")
spack_spec = get_env_var("SPACK_SPEC")
spack_cc = get_env_var("SPACK_CC")
spack_cxx = get_env_var("SPACK_CXX")
spack_f77 = get_env_var("SPACK_F77")
spack_fc = get_env_var("SPACK_FC")
compiler_spec = get_env_var("SPACK_COMPILER_SPEC")
spack_cc = get_env_var("SPACK_CC", required=False)
spack_cxx = get_env_var("SPACK_CXX", required=False)
spack_f77 = get_env_var("SPACK_F77", required=False)
spack_fc = get_env_var("SPACK_FC", required=False)
# Figure out what type of operation we're doing
command = os.path.basename(sys.argv[0])
@ -51,17 +52,25 @@ else:
if command in ('cc', 'gcc', 'c89', 'c99', 'clang'):
command = spack_cc
language = "C"
elif command in ('c++', 'CC', 'g++', 'clang++'):
command = spack_cxx
language = "C++"
elif command in ('f77'):
command = spack_f77
language = "Fortran 77"
elif command in ('fc'):
command = spack_fc
language = "Fortran 90"
elif command in ('ld', 'cpp'):
pass # leave it the same. TODO: what's the right thing?
else:
raise Exception("Unknown compiler: %s" % command)
if command is None:
print "ERROR: Compiler '%s' does not support compiling %s programs." % (
compiler_spec, language)
sys.exit(1)
version_args = ['-V', '-v', '--version', '-dumpversion']
if any(arg in sys.argv for arg in version_args):

5
lib/spack/spack/__init__.py
View File

@ -52,11 +52,6 @@
stage_path = join_path(var_path, "stage")
install_path = join_path(prefix, "opt")
#
# Place to look for usable compiler versions.
#
compiler_version_path = join_path(var_path, "compilers")
#
# Set up the packages database.
#

14
lib/spack/spack/build_environment.py
View File

@ -93,10 +93,16 @@ def set_compiler_environment_variables(pkg):
os.environ['FC'] = 'fc'
# Set SPACK compiler variables so that our wrapper knows what to call
os.environ['SPACK_CC'] = compiler.cc.command
os.environ['SPACK_CXX'] = compiler.cxx.command
os.environ['SPACK_F77'] = compiler.f77.command
os.environ['SPACK_FC'] = compiler.fc.command
if compiler.cc:
os.environ['SPACK_CC'] = compiler.cc.command
if compiler.cxx:
os.environ['SPACK_CXX'] = compiler.cxx.command
if compiler.f77:
os.environ['SPACK_F77'] = compiler.f77.command
if compiler.fc:
os.environ['SPACK_FC'] = compiler.fc.command
os.environ['SPACK_COMPILER_SPEC'] = str(pkg.spec.compiler)
def set_build_environment_variables(pkg):

13
lib/spack/spack/cmd/uninstall.py
View File

@ -56,11 +56,14 @@ def uninstall(parser, args):
for spec in specs:
matching_specs = spack.db.get_installed(spec)
if not args.all and len(matching_specs) > 1:
tty.die("%s matches multiple packages." % spec,
"You can either:",
" a) Use spack uninstall -a to uninstall ALL matching specs, or",
" b) use a more specific spec.",
"Matching packages:", *(" " + str(s) for s in matching_specs))
args = ["%s matches multiple packages." % spec,
"Matching packages:"]
args += [" " + str(s) for s in matching_specs]
args += ["You can either:",
" a) Use spack uninstall -a to uninstall ALL matching specs, or",
" b) use a more specific spec."]
tty.die(*args)
if len(matching_specs) == 0:
tty.die("%s does not match any installed packages." % spec)

145
lib/spack/spack/compiler.py
View File

@ -1,17 +1,26 @@
import os
import re
import itertools
from llnl.util.lang import memoized
from llnl.util.filesystem import join_path
import spack.error
import spack.spec
from spack.version import Version
from spack.util.executable import Executable
from spack.util.executable import Executable, which
from spack.compilation import get_path
_default_order = ['']
def _verify_executables(*paths):
for path in paths:
if not os.path.isfile(path) and os.access(path, os.X_OK):
raise InvalidCompilerPathError(path)
raise CompilerAccessError(path)
@memoized
def get_compiler_version(compiler, version_arg):
return compiler(version_arg, return_output=True)
class Compiler(object):
@ -32,27 +41,135 @@ class Compiler(object):
# Subclasses use possible names of Fortran 90 compiler
fc_names = []
# Optional prefix regexes for searching for this type of compiler.
# Prefixes are sometimes used for toolchains, e.g. 'powerpc-bgq-linux-'
prefixes = []
# Optional suffix regexes for searching for this type of compiler.
# Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y'
# version suffix for gcc.
suffixes = []
# Names of generic arguments used by this compiler
arg_version = '-dumpversion'
arg_rpath = '-Wl,-rpath,%s'
def __init__(self, cc, cxx, f77, fc):
_verify_executables(cc, cxx, f77, fc)
def make_exe(exe):
if exe is None:
return None
_verify_executables(exe)
return Executable(exe)
self.cc = make_exe(cc)
self.cxx = make_exe(cxx)
self.f77 = make_exe(f77)
self.fc = make_exe(fc)
def _tuple(self):
return (self.cc, self.cxx, self.f77, self.fc)
self.cc = Executable(cc)
self.cxx = Executable(cxx)
self.f77 = Executable(f77)
self.fc = Executable(fc)
@property
@memoized
def version(self):
v = self.cc(arg_version)
return Version(v)
for comp in self._tuple():
if comp is not None:
v = get_compiler_version(comp, self.arg_version)
return Version(v)
raise InvalidCompilerError()
class InvalidCompilerPathError(spack.error.SpackError):
@property
def spec(self):
return spack.spec.CompilerSpec(self.name, self.version)
@classmethod
def _find_matches_in_path(cls, compiler_names, *path):
"""Try to find compilers with the supplied names in any of the suppled
paths. Searches for all combinations of each name with the
compiler's specified prefixes and suffixes. Compilers are
returned in a dict from (prefix, suffix) tuples to paths to
the compilers with those prefixes and suffixes.
"""
if not path:
path = get_path('PATH')
matches = {}
ps_pairs = [p for p in itertools.product(
[''] + cls.prefixes, [''] + cls.suffixes)]
for directory in path:
files = os.listdir(directory)
for pre_re, suf_re in ps_pairs:
for compiler_name in compiler_names:
regex = r'^(%s)%s(%s)$' % (
pre_re, re.escape(compiler_name), suf_re)
for exe in files:
match = re.match(regex, exe)
if match:
pair = match.groups()
if pair not in matches:
matches[pair] = join_path(directory, exe)
return matches
@classmethod
def find(cls, *path):
"""Try to find this type of compiler in the user's environment. For
each set of compilers found, this returns a 4-tuple with
the cc, cxx, f77, and fc paths.
This will search for compilers with the names in cc_names,
cxx_names, etc. and it will group 4-tuples if they have
common prefixes and suffixes. e.g., gcc-mp-4.7 would be
grouped with g++-mp-4.7 and gfortran-mp-4.7.
Example return values::
[ ('/usr/bin/gcc', '/usr/bin/g++',
'/usr/bin/gfortran', '/usr/bin/gfortran'),
('/usr/bin/gcc-mp-4.5', '/usr/bin/g++-mp-4.5',
'/usr/bin/gfortran-mp-4.5', '/usr/bin/gfortran-mp-4.5') ]
"""
pair_names = [cls._find_matches_in_path(names, *path) for names in (
cls.cc_names, cls.cxx_names, cls.f77_names, cls.fc_names)]
keys = set()
for p in pair_names:
keys.update(p)
return [ tuple(pn[k] if k in pn else None for pn in pair_names)
for k in keys ]
def __repr__(self):
"""Return a string represntation of the compiler toolchain."""
return self.__str__()
def __str__(self):
"""Return a string represntation of the compiler toolchain."""
def p(path):
return ' '.join(path.exe) if path else None
return "%s(%s, %s, %s, %s)" % (
self.name,
p(self.cc), p(self.cxx), p(self.f77), p(self.fc))
class CompilerAccessError(spack.error.SpackError):
def __init__(self, path):
super(InvalidCompilerPathError, self).__init__(
super(CompilerAccessError, self).__init__(
"'%s' is not a valid compiler." % path)
class InvalidCompilerError(spack.error.SpackError):
def __init__(self):
super(InvalidCompilerError, self).__init__(
"Compiler has no executables.")

107
lib/spack/spack/compilers/__init__.py
View File

@ -22,9 +22,6 @@
# along with this program; if not, write to the Free Software Foundation,
# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
#
# This needs to be expanded for full compiler support.
#
import imp
from llnl.util.lang import memoized, list_modules
@ -33,12 +30,14 @@
import spack
import spack.error
import spack.spec
import spack.config
from spack.compiler import Compiler
from spack.util.executable import which
from spack.util.naming import mod_to_class
_imported_compilers_module = 'spack.compiler.versions'
_imported_versions_module = 'spack.compilers'
_imported_compilers_module = 'spack.compilers'
_required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
def _auto_compiler_spec(function):
@ -49,7 +48,40 @@ def converter(cspec_like):
return converter
@memoized
def _get_config():
"""Get a Spack config, but make sure it has compiler configuration
first."""
# If any configuration file has compilers, just stick with the
# ones already configured.
config = spack.config.get_config()
existing = [spack.spec.CompilerSpec(s)
for s in config.get_section_names('compiler')]
if existing:
return config
user_config = spack.config.get_config('user')
compilers = find_default_compilers()
for name, clist in compilers.items():
for compiler in clist:
if compiler.spec not in existing:
add_compiler(user_config, compiler)
user_config.write()
# After writing compilers to the user config, return a full config
# from all files.
return spack.config.get_config()
def add_compiler(config, compiler):
def setup_field(cspec, name, exe):
path = ' '.join(exe.exe) if exe else "None"
config.set_value('compiler', cspec, name, path)
for c in _required_instance_vars:
setup_field(compiler.spec, c, getattr(compiler, c))
def supported_compilers():
"""Return a set of names of compilers supported by Spack.
@ -65,13 +97,13 @@ def supported(compiler_spec):
return compiler_spec.name in supported_compilers()
@memoized
def all_compilers():
"""Return a set of specs for all the compiler versions currently
available to build with. These are instances of CompilerSpec.
"""
return set(spack.spec.CompilerSpec(c)
for c in list_modules(spack.compiler_version_path))
configuration = _get_config()
return [spack.spec.CompilerSpec(s)
for s in configuration.get_section_names('compiler')]
@_auto_compiler_spec
@ -86,20 +118,32 @@ def compilers_for_spec(compiler_spec):
"""This gets all compilers that satisfy the supplied CompilerSpec.
Returns an empty list if none are found.
"""
matches = find(compiler_spec)
config = _get_config()
def get_compiler(cspec):
items = { k:v for k,v in config.items('compiler "%s"' % cspec) }
if not all(n in items for n in _required_instance_vars):
raise InvalidCompilerConfigurationError(cspec)
compilers = []
for cspec in matches:
path = join_path(spack.compiler_version_path, "%s.py" % cspec)
mod = imp.load_source(_imported_versions_module, path)
cls = class_for_compiler_name(cspec.name)
compilers.append(cls(mod.cc, mod.cxx, mod.f77, mod.fc))
compiler_paths = []
for c in _required_instance_vars:
compiler_path = items[c]
if compiler_path != "None":
compiler_paths.append(compiler_path)
else:
compiler_paths.append(None)
return cls(*compiler_paths)
return compilers
matches = find(compiler_spec)
return [get_compiler(cspec) for cspec in matches]
@_auto_compiler_spec
def compiler_for_spec(compiler_spec):
"""Get the compiler that satisfies compiler_spec. compiler_spec must
be concrete."""
assert(compiler_spec.concrete)
compilers = compilers_for_spec(compiler_spec)
assert(len(compilers) == 1)
@ -107,11 +151,32 @@ def compiler_for_spec(compiler_spec):
def class_for_compiler_name(compiler_name):
"""Given a compiler module name, get the corresponding Compiler class."""
assert(supported(compiler_name))
file_path = join_path(spack.compilers_path, compiler_name + ".py")
compiler_mod = imp.load_source(_imported_compilers_module, file_path)
return getattr(compiler_mod, mod_to_class(compiler_name))
cls = getattr(compiler_mod, mod_to_class(compiler_name))
# make a note of the name in the module so we can get to it easily.
cls.name = compiler_name
return cls
def all_compiler_types():
return [class_for_compiler_name(c) for c in supported_compilers()]
def find_default_compilers():
"""Search the user's environment to get default compilers. Each
compiler class can have its own find() class method that can be
customized to locate that type of compiler.
"""
# Compiler name is inserted on load by class_for_compiler_name
return {
Compiler.name : [Compiler(*c) for c in Compiler.find()]
for Compiler in all_compiler_types() }
@memoized
@ -126,3 +191,11 @@ def default_compiler():
gcc = which('gcc', required=True)
version = gcc('-dumpversion', return_output=True)
return spack.spec.CompilerSpec('gcc', version)
class InvalidCompilerConfigurationError(spack.error.SpackError):
def __init__(self, compiler_spec):
super(InvalidCompilerConfigurationError, self).__init__(
"Invalid configuration for [compiler \"%s\"]: " % compiler_spec,
"Compiler configuration must contain entries for all compilers: %s"
% _required_instance_vars)

3
lib/spack/spack/compilers/gcc.py
View File

@ -37,5 +37,8 @@ class Gcc(Compiler):
# Subclasses use possible names of Fortran 90 compiler
fc_names = ['gfortran']
# MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
suffixes = [r'-mp-\d\.\d']
def __init__(self, cc, cxx, f77, fc):
super(Gcc, self).__init__(cc, cxx, f77, fc)

8
lib/spack/spack/concretize.py
View File

@ -110,14 +110,18 @@ def concretize_compiler(self, spec):
build with the compiler that will be used by libraries that
link to this one, to maximize compatibility.
"""
if spec.compiler and spec.compiler.concrete:
all_compilers = spack.compilers.all_compilers()
if (spec.compiler and
spec.compiler.concrete and
spec.compiler in all_compilers):
return
try:
nearest = next(p for p in spec.preorder_traversal(direction='parents')
if p.compiler is not None).compiler
if not nearest.concrete:
if not nearest in all_compilers:
# Take the newest compiler that saisfies the spec
matches = sorted(spack.compilers.find(nearest))
if not matches:

41
lib/spack/spack/config.py
View File

@ -88,7 +88,6 @@
from llnl.util.lang import memoized
import spack
import spack.error
__all__ = [
@ -196,7 +195,7 @@ def string_key_func(*args):
class SpackConfigParser(cp.RawConfigParser):
"""Slightly modified from Python's raw config file parser to accept
leading whitespace.
leading whitespace and preserve comments.
"""
# Slightly modified Python option expression. This one allows
# leading whitespace.
@ -305,6 +304,7 @@ def _read(self, fp, fpname):
cursect = None # None, or a dictionary
optname = None
lineno = 0
comment = 0
e = None # None, or an exception
while True:
line = fp.readline()
@ -312,10 +312,10 @@ def _read(self, fp, fpname):
break
lineno = lineno + 1
# comment or blank line?
if line.strip() == '' or line[0] in '#;':
continue
if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
# no leading whitespace
if ((line.strip() == '' or line[0] in '#;') or
(line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR")):
self._sections["comment-%d" % comment] = line
comment += 1
continue
# a section header or option header?
else:
@ -375,6 +375,10 @@ def _read(self, fp, fpname):
all_sections = [self._defaults]
all_sections.extend(self._sections.values())
for options in all_sections:
# skip comments
if isinstance(options, basestring):
continue
for name, val in options.items():
if isinstance(val, list):
options[name] = '\n'.join(val)
@ -391,17 +395,22 @@ def _write(self, fp):
for (key, value) in self._defaults.items():
fp.write(" %s = %s\n" % (key, str(value).replace('\n', '\n\t')))
fp.write("\n")
for section in self._sections:
# Allow leading whitespace
fp.write("[%s]\n" % section)
for (key, value) in self._sections[section].items():
if key == "__name__":
continue
if (value is not None) or (self._optcre == self.OPTCRE):
key = " = ".join((key, str(value).replace('\n', '\n\t')))
fp.write(" %s\n" % (key))
fp.write("\n")
for section in self._sections:
# Handles comments and blank lines.
if isinstance(self._sections[section], basestring):
fp.write(self._sections[section])
continue
else:
# Allow leading whitespace
fp.write("[%s]\n" % section)
for (key, value) in self._sections[section].items():
if key == "__name__":
continue
if (value is not None) or (self._optcre == self.OPTCRE):
key = " = ".join((key, str(value).replace('\n', '\n\t')))
fp.write(" %s\n" % (key))
class SpackConfigurationError(spack.error.SpackError):

1
lib/spack/spack/packages.py
View File

@ -32,7 +32,6 @@
from llnl.util.filesystem import join_path
from llnl.util.lang import memoized
import spack
import spack.error
import spack.spec
from spack.virtual import ProviderIndex

2
lib/spack/spack/spec.py
View File

@ -233,7 +233,7 @@ def constrain(self, other):
def concrete(self):
"""A CompilerSpec is concrete if its versions are concrete and there
is an available compiler with the right version."""
return self.versions.concrete and self in compilers.all_compilers()
return self.versions.concrete
@property

4
var/spack/compilers/clang@3.3.py
View File

@ -1,4 +0,0 @@
cc = '/usr/bin/clang'
cxx = '/usr/bin/clang++'
f77 = 'gfortran'
fc = 'gfortran'

4
var/spack/compilers/gcc@4.5.4.py
View File

@ -1,4 +0,0 @@
cc = '/Users/gamblin2/macports/bin/gcc-mp-4.5'
cxx = '/Users/gamblin2/macports/bin/g++-mp-4.5'
f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.5'
fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.5'

4
var/spack/compilers/gcc@4.7.3.py
View File

@ -1,4 +0,0 @@
cc = '/Users/gamblin2/macports/bin/gcc-mp-4.7'
cxx = '/Users/gamblin2/macports/bin/g++-mp-4.7'
f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.7'
fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.7'

4
var/spack/compilers/gcc@4.8.1.py
View File

@ -1,4 +0,0 @@
cc = '/Users/gamblin2/macports/bin/gcc-mp-4.8'
cxx = '/Users/gamblin2/macports/bin/g++-mp-4.8'
f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.8'
fc = '/Users/gamblin2/macports/bin/gfortran-mp-4.8'