Compiler support now uses configuration files.

- no more need for compiler python files.

- Default compilers are found in user's environment and added
	to ~/.spackconfig automatically

- User can add new compilers by editing configuration file

lib/spack/env/cc

@@ -27,10 +27,11 @@ spack_env_path      = get_path("SPACK_ENV_PATH")
 spack_debug_log_dir = get_env_var("SPACK_DEBUG_LOG_DIR")
 spack_spec          = get_env_var("SPACK_SPEC")
 
-spack_cc  = get_env_var("SPACK_CC")
+compiler_spec = get_env_var("SPACK_COMPILER_SPEC")
-spack_cxx = get_env_var("SPACK_CXX")
+spack_cc  = get_env_var("SPACK_CC",  required=False)
-spack_f77 = get_env_var("SPACK_F77")
+spack_cxx = get_env_var("SPACK_CXX", required=False)
-spack_fc  = get_env_var("SPACK_FC")
+spack_f77 = get_env_var("SPACK_F77", required=False)
+spack_fc  = get_env_var("SPACK_FC",  required=False)
 
 # Figure out what type of operation we're doing
 command = os.path.basename(sys.argv[0])
@@ -51,17 +52,25 @@ else:
 
 if command in ('cc', 'gcc', 'c89', 'c99', 'clang'):
     command = spack_cc
+    language = "C"
 elif command in ('c++', 'CC', 'g++', 'clang++'):
     command = spack_cxx
+    language = "C++"
 elif command in ('f77'):
     command = spack_f77
+    language = "Fortran 77"
 elif command in ('fc'):
     command = spack_fc
+    language = "Fortran 90"
 elif command in ('ld', 'cpp'):
     pass # leave it the same.  TODO: what's the right thing?
 else:
     raise Exception("Unknown compiler: %s" % command)
 
+if command is None:
+    print "ERROR: Compiler '%s' does not support compiling %s programs." % (
+        compiler_spec, language)
+    sys.exit(1)
 
 version_args = ['-V', '-v', '--version', '-dumpversion']
 if any(arg in sys.argv for arg in version_args):

lib/spack/spack/__init__.py

@@ -52,11 +52,6 @@
 stage_path     = join_path(var_path, "stage")
 install_path   = join_path(prefix, "opt")
 
-#
-# Place to look for usable compiler versions.
-#
-compiler_version_path = join_path(var_path, "compilers")
-
 #
 # Set up the packages database.
 #

lib/spack/spack/build_environment.py

@@ -93,11 +93,17 @@ def set_compiler_environment_variables(pkg):
     os.environ['FC']  = 'fc'
 
     # Set SPACK compiler variables so that our wrapper knows what to call
+    if compiler.cc:
         os.environ['SPACK_CC']  = compiler.cc.command
+    if compiler.cxx:
         os.environ['SPACK_CXX'] = compiler.cxx.command
+    if compiler.f77:
         os.environ['SPACK_F77'] = compiler.f77.command
+    if compiler.fc:
         os.environ['SPACK_FC']  = compiler.fc.command
 
+    os.environ['SPACK_COMPILER_SPEC']  = str(pkg.spec.compiler)
+
 
 def set_build_environment_variables(pkg):
     """This ensures a clean install environment when we build packages.

lib/spack/spack/cmd/uninstall.py

@@ -56,11 +56,14 @@ def uninstall(parser, args):
     for spec in specs:
         matching_specs = spack.db.get_installed(spec)
         if not args.all and len(matching_specs) > 1:
-            tty.die("%s matches multiple packages." % spec,
+            args =  ["%s matches multiple packages." % spec,
-                    "You can either:",
+                     "Matching packages:"]
+            args += ["  " + str(s) for s in matching_specs]
+            args += ["You can either:",
                      "  a) Use spack uninstall -a to uninstall ALL matching specs, or",
-                    " b) use a more specific spec.",
+                     "  b) use a more specific spec."]
-                    "Matching packages:", *("  " + str(s) for s in matching_specs))
+            tty.die(*args)
+
 
         if len(matching_specs) == 0:
             tty.die("%s does not match any installed packages." % spec)

lib/spack/spack/compiler.py

@@ -1,17 +1,26 @@
 import os
+import re
+import itertools
+
 from llnl.util.lang import memoized
+from llnl.util.filesystem import join_path
 
 import spack.error
+import spack.spec
 from spack.version import Version
-from spack.util.executable import Executable
+from spack.util.executable import Executable, which
-
+from spack.compilation import get_path
-
 
+_default_order = ['']
 
 def _verify_executables(*paths):
     for path in paths:
         if not os.path.isfile(path) and os.access(path, os.X_OK):
-            raise InvalidCompilerPathError(path)
+            raise CompilerAccessError(path)
+
+@memoized
+def get_compiler_version(compiler, version_arg):
+    return compiler(version_arg, return_output=True)
 
 
 class Compiler(object):
@@ -32,27 +41,135 @@ class Compiler(object):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = []
 
+    # Optional prefix regexes for searching for this type of compiler.
+    # Prefixes are sometimes used for toolchains, e.g. 'powerpc-bgq-linux-'
+    prefixes = []
+
+    # Optional suffix regexes for searching for this type of compiler.
+    # Suffixes are used by some frameworks, e.g. macports uses an '-mp-X.Y'
+    # version suffix for gcc.
+    suffixes = []
+
     # Names of generic arguments used by this compiler
     arg_version = '-dumpversion'
     arg_rpath   = '-Wl,-rpath,%s'
 
 
     def __init__(self, cc, cxx, f77, fc):
-        _verify_executables(cc, cxx, f77, fc)
+        def make_exe(exe):
+            if exe is None:
+                return None
+            _verify_executables(exe)
+            return Executable(exe)
+
+        self.cc  = make_exe(cc)
+        self.cxx = make_exe(cxx)
+        self.f77 = make_exe(f77)
+        self.fc  = make_exe(fc)
+
+
+    def _tuple(self):
+        return (self.cc, self.cxx, self.f77, self.fc)
 
-        self.cc  = Executable(cc)
-        self.cxx = Executable(cxx)
-        self.f77 = Executable(f77)
-        self.fc  = Executable(fc)
 
     @property
-    @memoized
     def version(self):
-        v = self.cc(arg_version)
+        for comp in self._tuple():
+            if comp is not None:
+                v = get_compiler_version(comp, self.arg_version)
                 return Version(v)
+        raise InvalidCompilerError()
 
 
-class InvalidCompilerPathError(spack.error.SpackError):
+    @property
+    def spec(self):
+        return spack.spec.CompilerSpec(self.name, self.version)
+
+
+    @classmethod
+    def _find_matches_in_path(cls, compiler_names, *path):
+        """Try to find compilers with the supplied names in any of the suppled
+           paths.  Searches for all combinations of each name with the
+           compiler's specified prefixes and suffixes.  Compilers are
+           returned in a dict from (prefix, suffix) tuples to paths to
+           the compilers with those prefixes and suffixes.
+        """
+        if not path:
+            path = get_path('PATH')
+
+        matches = {}
+        ps_pairs = [p for p in itertools.product(
+            [''] + cls.prefixes, [''] + cls.suffixes)]
+
+        for directory in path:
+            files = os.listdir(directory)
+
+            for pre_re, suf_re in ps_pairs:
+                for compiler_name in compiler_names:
+                    regex = r'^(%s)%s(%s)$' % (
+                        pre_re, re.escape(compiler_name), suf_re)
+
+                    for exe in files:
+                        match = re.match(regex, exe)
+                        if match:
+                            pair = match.groups()
+                            if pair not in matches:
+                                matches[pair] = join_path(directory, exe)
+
+        return matches
+
+    @classmethod
+    def find(cls, *path):
+        """Try to find this type of compiler in the user's environment. For
+           each set of compilers found, this returns a 4-tuple with
+           the cc, cxx, f77, and fc paths.
+
+           This will search for compilers with the names in cc_names,
+           cxx_names, etc. and it will group 4-tuples if they have
+           common prefixes and suffixes.  e.g., gcc-mp-4.7 would be
+           grouped with g++-mp-4.7 and gfortran-mp-4.7.
+
+           Example return values::
+
+               [ ('/usr/bin/gcc',      '/usr/bin/g++',
+                  '/usr/bin/gfortran', '/usr/bin/gfortran'),
+
+                 ('/usr/bin/gcc-mp-4.5',      '/usr/bin/g++-mp-4.5',
+                  '/usr/bin/gfortran-mp-4.5', '/usr/bin/gfortran-mp-4.5') ]
+
+        """
+        pair_names = [cls._find_matches_in_path(names, *path) for names in (
+            cls.cc_names, cls.cxx_names, cls.f77_names, cls.fc_names)]
+
+        keys = set()
+        for p in pair_names:
+            keys.update(p)
+
+        return [ tuple(pn[k] if k in pn else None for pn in pair_names)
+                 for k in keys ]
+
+
+    def __repr__(self):
+        """Return a string represntation of the compiler toolchain."""
+        return self.__str__()
+
+
+    def __str__(self):
+        """Return a string represntation of the compiler toolchain."""
+        def p(path):
+            return ' '.join(path.exe) if path else None
+        return "%s(%s, %s, %s, %s)" % (
+            self.name,
+            p(self.cc), p(self.cxx), p(self.f77), p(self.fc))
+
+
+class CompilerAccessError(spack.error.SpackError):
     def __init__(self, path):
-        super(InvalidCompilerPathError, self).__init__(
+        super(CompilerAccessError, self).__init__(
             "'%s' is not a valid compiler." % path)
+
+
+class InvalidCompilerError(spack.error.SpackError):
+    def __init__(self):
+        super(InvalidCompilerError, self).__init__(
+            "Compiler has no executables.")

lib/spack/spack/compilers/__init__.py

@@ -22,9 +22,6 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-#
-# This needs to be expanded for full compiler support.
-#
 import imp
 
 from llnl.util.lang import memoized, list_modules
@@ -33,12 +30,14 @@
 import spack
 import spack.error
 import spack.spec
+import spack.config
+
 from spack.compiler import Compiler
 from spack.util.executable import which
 from spack.util.naming import mod_to_class
 
-_imported_compilers_module = 'spack.compiler.versions'
+_imported_compilers_module = 'spack.compilers'
-_imported_versions_module  = 'spack.compilers'
+_required_instance_vars = ['cc', 'cxx', 'f77', 'fc']
 
 
 def _auto_compiler_spec(function):
@@ -49,7 +48,40 @@ def converter(cspec_like):
     return converter
 
 
-@memoized
+def _get_config():
+    """Get a Spack config, but make sure it has compiler configuration
+       first."""
+    # If any configuration file has compilers, just stick with the
+    # ones already configured.
+    config = spack.config.get_config()
+    existing = [spack.spec.CompilerSpec(s)
+                for s in config.get_section_names('compiler')]
+    if existing:
+        return config
+
+    user_config = spack.config.get_config('user')
+
+    compilers = find_default_compilers()
+    for name, clist in compilers.items():
+        for compiler in clist:
+            if compiler.spec not in existing:
+                add_compiler(user_config, compiler)
+    user_config.write()
+
+    # After writing compilers to the user config, return a full config
+    # from all files.
+    return spack.config.get_config()
+
+
+def add_compiler(config, compiler):
+    def setup_field(cspec, name, exe):
+        path = ' '.join(exe.exe) if exe else "None"
+        config.set_value('compiler', cspec, name, path)
+
+    for c in _required_instance_vars:
+        setup_field(compiler.spec, c, getattr(compiler, c))
+
+
 def supported_compilers():
     """Return a set of names of compilers supported by Spack.
 
@@ -65,13 +97,13 @@ def supported(compiler_spec):
     return compiler_spec.name in supported_compilers()
 
 
-@memoized
 def all_compilers():
     """Return a set of specs for all the compiler versions currently
        available to build with.  These are instances of CompilerSpec.
     """
-    return set(spack.spec.CompilerSpec(c)
+    configuration = _get_config()
-               for c in list_modules(spack.compiler_version_path))
+    return [spack.spec.CompilerSpec(s)
+            for s in configuration.get_section_names('compiler')]
 
 
 @_auto_compiler_spec
@@ -86,20 +118,32 @@ def compilers_for_spec(compiler_spec):
     """This gets all compilers that satisfy the supplied CompilerSpec.
        Returns an empty list if none are found.
     """
-    matches = find(compiler_spec)
+    config = _get_config()
+
+    def get_compiler(cspec):
+        items = { k:v for k,v in config.items('compiler "%s"' % cspec) }
+
+        if not all(n in items for n in _required_instance_vars):
+            raise InvalidCompilerConfigurationError(cspec)
 
-    compilers = []
-    for cspec in matches:
-        path = join_path(spack.compiler_version_path, "%s.py" % cspec)
-        mod  = imp.load_source(_imported_versions_module, path)
         cls  = class_for_compiler_name(cspec.name)
-        compilers.append(cls(mod.cc, mod.cxx, mod.f77, mod.fc))
+        compiler_paths = []
+        for c in _required_instance_vars:
+            compiler_path = items[c]
+            if compiler_path != "None":
+                compiler_paths.append(compiler_path)
+            else:
+                compiler_paths.append(None)
+        return cls(*compiler_paths)
 
-    return compilers
+    matches = find(compiler_spec)
+    return [get_compiler(cspec) for cspec in matches]
 
 
 @_auto_compiler_spec
 def compiler_for_spec(compiler_spec):
+    """Get the compiler that satisfies compiler_spec.  compiler_spec must
+       be concrete."""
     assert(compiler_spec.concrete)
     compilers = compilers_for_spec(compiler_spec)
     assert(len(compilers) == 1)
@@ -107,11 +151,32 @@ def compiler_for_spec(compiler_spec):
 
 
 def class_for_compiler_name(compiler_name):
+    """Given a compiler module name, get the corresponding Compiler class."""
     assert(supported(compiler_name))
 
     file_path = join_path(spack.compilers_path, compiler_name + ".py")
     compiler_mod = imp.load_source(_imported_compilers_module, file_path)
-    return getattr(compiler_mod, mod_to_class(compiler_name))
+    cls = getattr(compiler_mod, mod_to_class(compiler_name))
+
+    # make a note of the name in the module so we can get to it easily.
+    cls.name = compiler_name
+
+    return cls
+
+
+def all_compiler_types():
+    return [class_for_compiler_name(c) for c in supported_compilers()]
+
+
+def find_default_compilers():
+    """Search the user's environment to get default compilers.  Each
+       compiler class can have its own find() class method that can be
+       customized to locate that type of compiler.
+    """
+    # Compiler name is inserted on load by class_for_compiler_name
+    return {
+        Compiler.name : [Compiler(*c) for c in Compiler.find()]
+        for Compiler in all_compiler_types() }
 
 
 @memoized
@@ -126,3 +191,11 @@ def default_compiler():
     gcc = which('gcc', required=True)
     version = gcc('-dumpversion', return_output=True)
     return spack.spec.CompilerSpec('gcc', version)
+
+
+class InvalidCompilerConfigurationError(spack.error.SpackError):
+    def __init__(self, compiler_spec):
+        super(InvalidCompilerConfigurationError, self).__init__(
+            "Invalid configuration for [compiler \"%s\"]: " % compiler_spec,
+            "Compiler configuration must contain entries for all compilers: %s"
+            % _required_instance_vars)

lib/spack/spack/compilers/gcc.py

@@ -37,5 +37,8 @@ class Gcc(Compiler):
     # Subclasses use possible names of Fortran 90 compiler
     fc_names = ['gfortran']
 
+    # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
+    suffixes = [r'-mp-\d\.\d']
+
     def __init__(self, cc, cxx, f77, fc):
         super(Gcc, self).__init__(cc, cxx, f77, fc)

lib/spack/spack/concretize.py

@@ -110,14 +110,18 @@ def concretize_compiler(self, spec):
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
         """
-        if spec.compiler and spec.compiler.concrete:
+        all_compilers = spack.compilers.all_compilers()
+
+        if (spec.compiler and
+            spec.compiler.concrete and
+            spec.compiler in all_compilers):
             return
 
         try:
             nearest = next(p for p in spec.preorder_traversal(direction='parents')
                            if p.compiler is not None).compiler
 
-            if not nearest.concrete:
+            if not nearest in all_compilers:
                 # Take the newest compiler that saisfies the spec
                 matches = sorted(spack.compilers.find(nearest))
                 if not matches:

lib/spack/spack/config.py

@@ -88,7 +88,6 @@
 
 from llnl.util.lang import memoized
 
-import spack
 import spack.error
 
 __all__ = [
@@ -196,7 +195,7 @@ def string_key_func(*args):
 
 class SpackConfigParser(cp.RawConfigParser):
     """Slightly modified from Python's raw config file parser to accept
-    leading whitespace.
+       leading whitespace and preserve comments.
     """
     # Slightly modified Python option expression. This one allows
     # leading whitespace.
@@ -305,6 +304,7 @@ def _read(self, fp, fpname):
         cursect = None                        # None, or a dictionary
         optname = None
         lineno = 0
+        comment = 0
         e = None                              # None, or an exception
         while True:
             line = fp.readline()
@@ -312,10 +312,10 @@ def _read(self, fp, fpname):
                 break
             lineno = lineno + 1
             # comment or blank line?
-            if line.strip() == '' or line[0] in '#;':
+            if ((line.strip() == '' or line[0] in '#;') or
-                continue
+                (line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR")):
-            if line.split(None, 1)[0].lower() == 'rem' and line[0] in "rR":
+                self._sections["comment-%d" % comment] = line
-                # no leading whitespace
+                comment += 1
                 continue
             # a section header or option header?
             else:
@@ -375,6 +375,10 @@ def _read(self, fp, fpname):
         all_sections = [self._defaults]
         all_sections.extend(self._sections.values())
         for options in all_sections:
+            # skip comments
+            if isinstance(options, basestring):
+                continue
+
             for name, val in options.items():
                 if isinstance(val, list):
                     options[name] = '\n'.join(val)
@@ -391,7 +395,14 @@ def _write(self, fp):
             for (key, value) in self._defaults.items():
                 fp.write("    %s = %s\n" % (key, str(value).replace('\n', '\n\t')))
             fp.write("\n")
+
         for section in self._sections:
+            # Handles comments and blank lines.
+            if isinstance(self._sections[section], basestring):
+                fp.write(self._sections[section])
+                continue
+
+            else:
                 # Allow leading whitespace
                 fp.write("[%s]\n" % section)
                 for (key, value) in self._sections[section].items():
@@ -400,8 +411,6 @@ def _write(self, fp):
                     if (value is not None) or (self._optcre == self.OPTCRE):
                         key = " = ".join((key, str(value).replace('\n', '\n\t')))
                     fp.write("    %s\n" % (key))
-            fp.write("\n")
-
 
 
 class SpackConfigurationError(spack.error.SpackError):

lib/spack/spack/packages.py

@@ -32,7 +32,6 @@
 from llnl.util.filesystem import join_path
 from llnl.util.lang import memoized
 
-import spack
 import spack.error
 import spack.spec
 from spack.virtual import ProviderIndex

lib/spack/spack/spec.py

@@ -233,7 +233,7 @@ def constrain(self, other):
     def concrete(self):
         """A CompilerSpec is concrete if its versions are concrete and there
            is an available compiler with the right version."""
-        return self.versions.concrete and self in compilers.all_compilers()
+        return self.versions.concrete
 
 
     @property

var/spack/compilers/clang@3.3.py

@@ -1,4 +0,0 @@
-cc  = '/usr/bin/clang'
-cxx = '/usr/bin/clang++'
-f77 = 'gfortran'
-fc  = 'gfortran'

var/spack/compilers/gcc@4.5.4.py

@@ -1,4 +0,0 @@
-cc  = '/Users/gamblin2/macports/bin/gcc-mp-4.5'
-cxx = '/Users/gamblin2/macports/bin/g++-mp-4.5'
-f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.5'
-fc  = '/Users/gamblin2/macports/bin/gfortran-mp-4.5'

var/spack/compilers/gcc@4.7.3.py

@@ -1,4 +0,0 @@
-cc  = '/Users/gamblin2/macports/bin/gcc-mp-4.7'
-cxx = '/Users/gamblin2/macports/bin/g++-mp-4.7'
-f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.7'
-fc  = '/Users/gamblin2/macports/bin/gfortran-mp-4.7'

var/spack/compilers/gcc@4.8.1.py

@@ -1,4 +0,0 @@
-cc  = '/Users/gamblin2/macports/bin/gcc-mp-4.8'
-cxx = '/Users/gamblin2/macports/bin/g++-mp-4.8'
-f77 = '/Users/gamblin2/macports/bin/gfortran-mp-4.8'
-fc  = '/Users/gamblin2/macports/bin/gfortran-mp-4.8'