Add CompilerSpec class and loading capability.

- spack.spec.Compiler is now spack.spec.CompilerSpec
- Can load a spack.compilers.* module for a particular spec
	- e.g. load Gcc module for gcc@4.7 spec.

lib/spack/docs/packaging_guide.rst

@@ -107,7 +107,7 @@ Package class names
 The **class name** (``Libelf`` in our example) is formed by converting
 words separated by `-` or ``_`` in the file name to camel case.  If
 the name starts with a number, we prefix the class name with
-``Num_``. Here are some examples:
+``_``. Here are some examples:
 
 =================  =================
  Module Name         Class Name
@@ -115,7 +115,7 @@ the name starts with a number, we prefix the class name with
  ``foo_bar``         ``FooBar``
  ``docbook-xml``     ``DocbookXml``
  ``FooBar``          ``Foobar``
- ``3proxy``          ``Num_3proxy``
+ ``3proxy``          ``_3proxy``
 =================  =================
 
 The class name is needed by Spack to properly import a package, but

lib/spack/spack/build_environment.py

@@ -34,6 +34,7 @@
 from llnl.util.filesystem import *
 
 import spack
+from spack.compilers import compiler_for_spec
 from spack.util.executable import Executable, which
 from spack.util.environment import *
 
@@ -79,6 +80,11 @@ def __call__(self, *args, **kwargs):
         super(MakeExecutable, self).__call__(*args, **kwargs)
 
 
+def set_compiler_environment_variables(pkg):
+    assert(pkg.spec.concrete)
+    compiler = compiler_for_spec(pkg.spec.compiler)
+
+
 def set_build_environment_variables(pkg):
     """This ensures a clean install environment when we build packages.
     """

lib/spack/spack/compilers/__init__.py

@@ -25,11 +25,21 @@
 #
 # This needs to be expanded for full compiler support.
 #
+import imp
+
 from llnl.util.lang import memoized, list_modules
+from llnl.util.filesystem import join_path
 
 import spack
+import spack.error
 import spack.spec
+from spack.compiler import Compiler
 from spack.util.executable import which
+from spack.util.naming import mod_to_class
+
+_imported_compilers_module = 'spack.compiler.versions'
+_imported_versions_module  = 'spack.compilers'
+
 
 @memoized
 def supported_compilers():
@@ -41,29 +51,58 @@ def supported_compilers():
     return sorted(c for c in list_modules(spack.compilers_path))
 
 
+def supported(compiler_spec):
+    """Test if a particular compiler is supported."""
+    if not isinstance(compiler_spec, spack.spec.CompilerSpec):
+        compiler_spec = spack.spec.CompilerSpec(compiler_spec)
+    return compiler_spec.name in supported_compilers()
+
+
 def available_compilers():
     """Return a list of specs for all the compiler versions currently
        available to build with.  These are instances of
-       spack.spec.Compiler.
+       CompilerSpec.
     """
-    return [spack.spec.Compiler(c)
+    return [spack.spec.CompilerSpec(c)
             for c in list_modules(spack.compiler_version_path)]
 
 
-def supported(compiler_spec):
+def compiler_for_spec(compiler_spec):
-    """Test if a particular compiler is supported."""
+    """This gets an instance of an actual spack.compiler.Compiler object
-    if not isinstance(compiler_spec, spack.spec.Compiler):
+       from a compiler spec.  The spec needs to be concrete for this to
-        compiler_spec = spack.spec.Compiler(compiler_spec)
+       work; it will raise an error if passed an abstract compiler.
-    return compiler_spec.name in supported_compilers()
+    """
+    matches = [c for c in available_compilers() if c.satisfies(compiler_spec)]
+
+    # TODO: do something when there are zero matches.
+    assert(len(matches) >= 1)
+
+    compiler = matches[0]
+    file_path = join_path(spack.compiler_version_path, "%s.py" % compiler)
+
+    mod = imp.load_source(_imported_versions_module, file_path)
+    compiler_class = class_for_compiler_name(compiler.name)
+
+    return compiler_class(mod.cc, mod.cxx, mod.f77, mod.f90)
+
+
+def class_for_compiler_name(compiler_name):
+    assert(supported(compiler_name))
+
+    file_path = join_path(spack.compilers_path, compiler_name + ".py")
+    compiler_mod = imp.load_source(_imported_compilers_module, file_path)
+    return getattr(compiler_mod, mod_to_class(compiler_name))
 
 
 @memoized
 def default_compiler():
-    """Get the spec for the default compiler supported by Spack.
+    """Get the spec for the default compiler on the system.
        Currently just returns the system's default gcc.
 
-       TODO: provide a better way to specify/find this on startup.
+       TODO: provide a more sensible default.  e.g. on Intel systems
+             we probably want icc.  On Mac OS, clang.  Probably need
+             to inspect the system and figure this out.
     """
     gcc = which('gcc', required=True)
     version = gcc('-dumpversion', return_output=True)
-    return spack.spec.Compiler('gcc', version)
+    return spack.spec.CompilerSpec('gcc', version)

lib/spack/spack/concretize.py

@@ -99,6 +99,15 @@ def concretize_compiler(self, spec):
            this one has a strict compiler requirement.  Otherwise, try to
            build with the compiler that will be used by libraries that
            link to this one, to maximize compatibility.
+
+           TODO: In many cases we probably want to look for installed
+                 versions of each package and use *that* version if we
+                 can link to it.  The policy implemented here will
+                 tend to rebuild a lot of stuff becasue it will prefer
+                 a compiler in the spec to any compiler already-
+                 installed things were built with.  There is likely
+                 some better policy that finds some middle ground
+                 between these two extremes.
         """
         if spec.compiler and spec.compiler.concrete:
             return

lib/spack/spack/package.py

@@ -657,6 +657,7 @@ def do_install(self, **kwargs):
             spack.install_layout.make_path_for_spec(self.spec)
 
             # Set up process's build environment before running install.
+            build_env.set_compiler_environment_variables(self)
             build_env.set_build_environment_variables(self)
             build_env.set_module_variables_for_package(self)
 

lib/spack/spack/packages.py

@@ -22,10 +22,8 @@
 # along with this program; if not, write to the Free Software Foundation,
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
-import re
 import os
 import sys
-import string
 import inspect
 import glob
 import imp
@@ -38,6 +36,7 @@
 import spack.error
 import spack.spec
 from spack.virtual import ProviderIndex
+from spack.util.naming import mod_to_class, validate_module_name
 
 # Name of module under which packages are imported
 _imported_packages_module = 'spack.packages'
@@ -45,42 +44,6 @@
 # Name of the package file inside a package directory
 _package_file_name = 'package.py'
 
-# Valid package names can contain '-' but can't start with it.
-valid_package_re = r'^\w[\w-]*$'
-
-# Don't allow consecutive [_-] in package names
-invalid_package_re = r'[_-][_-]+'
-
-
-def valid_package_name(pkg_name):
-    """Return whether the pkg_name is valid for use in Spack."""
-    return (re.match(valid_package_re, pkg_name) and
-            not re.search(invalid_package_re, pkg_name))
-
-
-def validate_package_name(pkg_name):
-    """Raise an exception if pkg_name is not valid."""
-    if not valid_package_name(pkg_name):
-        raise InvalidPackageNameError(pkg_name)
-
-
-def class_name_for_package_name(pkg_name):
-    """Get a name for the class the package file should contain.  Note that
-       conflicts don't matter because the classes are in different modules.
-    """
-    validate_package_name(pkg_name)
-
-    class_name = pkg_name.replace('_', '-')
-    class_name = string.capwords(class_name, '-')
-    class_name = class_name.replace('-', '')
-
-    # If a class starts with a number, prefix it with Number_ to make it a valid
-    # Python class name.
-    if re.match(r'^[0-9]', class_name):
-        class_name = "Num_%s" % class_name
-
-    return class_name
-
 
 def _autospec(function):
     """Decorator that automatically converts the argument of a single-arg
@@ -143,7 +106,7 @@ def filename_for_package_name(self, pkg_name):
            package doesn't exist yet, so callers will need to ensure
            the package exists before importing.
         """
-        validate_package_name(pkg_name)
+        validate_module_name(pkg_name)
         pkg_dir = self.dirname_for_package_name(pkg_name)
         return join_path(pkg_dir, _package_file_name)
 
@@ -200,7 +163,7 @@ def get_class_for_package_name(self, pkg_name):
         else:
             raise UnknownPackageError(pkg_name)
 
-        class_name = class_name_for_package_name(pkg_name)
+        class_name = mod_to_class(pkg_name)
         try:
             module_name = _imported_packages_module + '.' + pkg_name
             module = imp.load_source(module_name, file_path)
@@ -259,14 +222,6 @@ def quote(string):
         out.write('}\n')
 
 
-class InvalidPackageNameError(spack.error.SpackError):
-    """Raised when we encounter a bad package name."""
-    def __init__(self, name):
-        super(InvalidPackageNameError, self).__init__(
-            "Invalid package name: " + name)
-        self.name = name
-
-
 class UnknownPackageError(spack.error.SpackError):
     """Raised when we encounter a package spack doesn't have."""
     def __init__(self, name):

lib/spack/spack/spec.py

@@ -102,8 +102,7 @@
 import spack
 import spack.parse
 import spack.error
-import spack.compilers
+from spack.compilers import supported as supported_compiler
-import spack.compilers.gcc
 
 from spack.version import *
 from spack.util.string import *
@@ -169,18 +168,30 @@ def __call__(self, match):
 
 
 @key_ordering
-class Compiler(object):
+class CompilerSpec(object):
-    """The Compiler field represents the compiler or range of compiler
+    """The CompilerSpec field represents the compiler or range of compiler
-       versions that a package should be built with.  Compilers have a
+       versions that a package should be built with.  CompilerSpecs have a
        name and a version list. """
     def __init__(self, *args):
         nargs = len(args)
         if nargs == 1:
-            # If there is one argument, it's a spec to parse
+            arg = args[0]
-            c = SpecParser().parse_compiler(args[0])
+            # If there is one argument, it's either another CompilerSpec
+            # to copy or a string to parse
+            if isinstance(arg, basestring):
+                c = SpecParser().parse_compiler(arg)
                 self.name = c.name
                 self.versions = c.versions
 
+            elif isinstance(arg, CompilerSpec):
+                self.name = arg.name
+                self.versions = arg.versions.copy()
+
+            else:
+                raise TypeError(
+                    "Can only build CompilerSpec from string or CompilerSpec." +
+                    "  Found %s" % type(arg))
+
         elif nargs == 2:
             name, version = args
             self.name = name
@@ -197,12 +208,14 @@ def _add_version(self, version):
 
 
     def _autospec(self, compiler_spec_like):
-        if not isinstance(compiler_spec_like, Compiler):
+        if isinstance(compiler_spec_like, CompilerSpec):
-            return Compiler(compiler_spec_like)
             return compiler_spec_like
+        return CompilerSpec(compiler_spec_like)
 
 
     def satisfies(self, other):
+        # TODO: This should not just look for overlapping versions.
+        # TODO: e.g., 4.7.3 should satisfy a requirement for 4.7.
         other = self._autospec(other)
         return (self.name == other.name and
                 self.versions.overlaps(other.versions))
@@ -218,7 +231,7 @@ def constrain(self, other):
 
     @property
     def concrete(self):
-        """A Compiler spec is concrete if its versions are concrete."""
+        """A CompilerSpec is concrete if its versions are concrete."""
         return self.versions.concrete
 
 
@@ -230,7 +243,7 @@ def version(self):
 
 
     def copy(self):
-        clone = Compiler.__new__(Compiler)
+        clone = CompilerSpec.__new__(CompilerSpec)
         clone.name = self.name
         clone.versions = self.versions.copy()
         return clone
@@ -353,7 +366,7 @@ def _add_variant(self, name, enabled):
 
     def _set_compiler(self, compiler):
         """Called by the parser to set the compiler."""
-        if self.compiler: raise DuplicateCompilerError(
+        if self.compiler: raise DuplicateCompilerSpecError(
                 "Spec for '%s' cannot have two compilers." % self.name)
         self.compiler = compiler
 
@@ -808,7 +821,7 @@ def validate_names(self):
 
             # validate compiler in addition to the package name.
             if spec.compiler:
-                if not spack.compilers.supported(spec.compiler):
+                if not supported_compiler(spec.compiler):
                     raise UnsupportedCompilerError(spec.compiler.name)
 
 
@@ -1320,7 +1333,7 @@ def compiler(self):
         self.expect(ID)
         self.check_identifier()
 
-        compiler = Compiler.__new__(Compiler)
+        compiler = CompilerSpec.__new__(CompilerSpec)
         compiler.name = self.token.value
         compiler.versions = VersionList()
         if self.accept(AT):
@@ -1402,10 +1415,10 @@ def __init__(self, message):
         super(DuplicateVariantError, self).__init__(message)
 
 
-class DuplicateCompilerError(SpecError):
+class DuplicateCompilerSpecError(SpecError):
     """Raised when the same compiler occurs in a spec twice."""
     def __init__(self, message):
-        super(DuplicateCompilerError, self).__init__(message)
+        super(DuplicateCompilerSpecError, self).__init__(message)
 
 
 class UnsupportedCompilerError(SpecError):

lib/spack/spack/test/concretize.py

@@ -25,7 +25,7 @@
 import unittest
 
 import spack
-from spack.spec import Spec, Compiler
+from spack.spec import Spec, CompilerSpec
 from spack.test.mock_packages_test import *
 
 class ConcretizeTest(MockPackagesTest):
@@ -169,7 +169,7 @@ def test_compiler_inheritance(self):
         spec = Spec('mpileaks')
         spec.normalize()
 
-        spec['dyninst'].compiler = Compiler('clang')
+        spec['dyninst'].compiler = CompilerSpec('clang')
         spec.concretize()
 
         # TODO: not exactly the syntax I would like.

lib/spack/spack/test/packages.py

@@ -28,6 +28,7 @@
 
 import spack
 import spack.packages as packages
+from spack.util.naming import mod_to_class
 from spack.test.mock_packages_test import *
 
 
@@ -58,8 +59,8 @@ def test_nonexisting_package_filename(self):
 
 
     def test_package_class_names(self):
-        self.assertEqual('Mpich',          packages.class_name_for_package_name('mpich'))
+        self.assertEqual('Mpich',          mod_to_class('mpich'))
-        self.assertEqual('PmgrCollective', packages.class_name_for_package_name('pmgr_collective'))
+        self.assertEqual('PmgrCollective', mod_to_class('pmgr_collective'))
-        self.assertEqual('PmgrCollective', packages.class_name_for_package_name('pmgr-collective'))
+        self.assertEqual('PmgrCollective', mod_to_class('pmgr-collective'))
-        self.assertEqual('Pmgrcollective', packages.class_name_for_package_name('PmgrCollective'))
+        self.assertEqual('Pmgrcollective', mod_to_class('PmgrCollective'))
-        self.assertEqual('Num_3db',        packages.class_name_for_package_name('3db'))
+        self.assertEqual('_3db',        mod_to_class('3db'))

lib/spack/spack/test/spec_syntax.py

@@ -133,12 +133,12 @@ def test_duplicate_depdendence(self):
         self.assertRaises(DuplicateDependencyError, self.check_parse, "x ^y ^y")
 
     def test_duplicate_compiler(self):
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x%intel%intel")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%intel")
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x%intel%gcc")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%intel%gcc")
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x%gcc%intel")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x%gcc%intel")
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%intel%intel")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%intel%intel")
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%intel%gcc")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%intel%gcc")
-        self.assertRaises(DuplicateCompilerError, self.check_parse, "x ^y%gcc%intel")
+        self.assertRaises(DuplicateCompilerSpecError, self.check_parse, "x ^y%gcc%intel")
 
 
     # ================================================================================

lib/spack/spack/util/naming.py

@@ -0,0 +1,61 @@
+# Need this because of spack.util.string
+from __future__ import absolute_import
+import string
+import re
+
+import spack
+
+# Valid module names can contain '-' but can't start with it.
+_valid_module_re = r'^\w[\w-]*$'
+
+
+def mod_to_class(mod_name):
+    """Convert a name from module style to class name style.  Spack mostly
+       follows `PEP-8 <http://legacy.python.org/dev/peps/pep-0008/>`_:
+
+          * Module and package names use lowercase_with_underscores.
+          * Class names use the CapWords convention.
+
+       Regular source code follows these convetions.  Spack is a bit
+       more liberal with its Package names nad Compiler names:
+
+          * They can contain '-' as well as '_', but cannot start with '-'.
+          * They can start with numbers, e.g. "3proxy".
+
+       This function converts from the module convention to the class
+       convention by removing _ and - and converting surrounding
+       lowercase text to CapWords.  If mod_name starts with a number,
+       the class name returned will be prepended with '_' to make a
+       valid Python identifier.
+    """
+    validate_module_name(mod_name)
+
+    class_name = re.sub(r'[-_]+', '-', mod_name)
+    class_name = string.capwords(class_name, '-')
+    class_name = class_name.replace('-', '')
+
+    # If a class starts with a number, prefix it with Number_ to make it a valid
+    # Python class name.
+    if re.match(r'^[0-9]', class_name):
+        class_name = "_%s" % class_name
+
+    return class_name
+
+
+def valid_module_name(mod_name):
+    """Return whether the mod_name is valid for use in Spack."""
+    return bool(re.match(_valid_module_re, mod_name))
+
+
+def validate_module_name(mod_name):
+    """Raise an exception if mod_name is not valid."""
+    if not valid_module_name(mod_name):
+        raise InvalidModuleNameError(mod_name)
+
+
+class InvalidModuleNameError(spack.error.SpackError):
+    """Raised when we encounter a bad module name."""
+    def __init__(self, name):
+        super(InvalidModuleNameError, self).__init__(
+            "Invalid module name: " + name)
+        self.name = name