Merge branch 'features/automaded' into develop

lib/spack/llnl/util/filesystem.py

@@ -38,7 +38,7 @@
 from spack.util.compression import ALLOWED_ARCHIVE_TYPES
 
 
-def filter_file(regex, repl, *filenames):
+def filter_file(regex, repl, *filenames, **kwargs):
     """Like sed, but uses python regular expressions.
 
        Filters every line of file through regex and replaces the file
@@ -49,16 +49,31 @@ def filter_file(regex, repl, *filenames):
        return a suitable replacement string.  If it is a string, it
        can contain ``\1``, ``\2``, etc. to represent back-substitution
        as sed would allow.
+
+       Keyword Options:
+         string[=False]         If True, treat regex as a plain string.
+         backup[=True]          Make a backup files suffixed with ~
+         ignore_absent[=False]  Ignore any files that don't exist.
     """
-    # Keep callables intact
+    string = kwargs.get('string', False)
-    if not hasattr(repl, '__call__'):
+    backup = kwargs.get('backup', True)
-        # Allow strings to use \1, \2, etc. for replacement, like sed
+    ignore_absent = kwargs.get('ignore_absent', False)
+
+    # Allow strings to use \1, \2, etc. for replacement, like sed
+    if not callable(repl):
         unescaped = repl.replace(r'\\', '\\')
         repl = lambda m: re.sub(
             r'\\([0-9])', lambda x: m.group(int(x.group(1))), unescaped)
 
+    if string:
+        regex = re.escape(regex)
+
     for filename in filenames:
         backup = filename + "~"
+
+        if ignore_absent and not os.path.exists(filename):
+            continue
+
         shutil.copy(filename, backup)
         try:
             with closing(open(backup)) as infile:
@@ -71,6 +86,10 @@ def filter_file(regex, repl, *filenames):
             shutil.move(backup, filename)
             raise
 
+        finally:
+            if not backup:
+                shutil.rmtree(backup, ignore_errors=True)
+
 
 def change_sed_delimiter(old_delim, new_delim, *filenames):
     """Find all sed search/replace commands and change the delimiter.

lib/spack/spack/__init__.py

@@ -137,9 +137,9 @@
 # TODO: it's not clear where all the stuff that needs to be included in packages
 #       should live.  This file is overloaded for spack core vs. for packages.
 #
-__all__ = ['Package', 'Version', 'when']
+__all__ = ['Package', 'Version', 'when', 'ver']
 from spack.package import Package
-from spack.version import Version
+from spack.version import Version, ver
 from spack.multimethod import when
 
 import llnl.util.filesystem

lib/spack/spack/cmd/checksum.py

@@ -117,7 +117,5 @@ def checksum(parser, args):
     if not version_hashes:
         tty.die("Could not fetch any available versions for %s." % pkg.name)
 
-    dict_string = ["    '%s' : '%s'," % (v, h) for v, h in version_hashes]
+    version_lines = ["    version('%s', '%s')" % (v, h) for v, h in version_hashes]
-    dict_string = ['{'] + dict_string + ["}"]
+    tty.msg("Checksummed new versions of %s:" % pkg.name, *version_lines)
-
-    tty.msg("Checksummed new versions of %s:" % pkg.name, *dict_string)

lib/spack/spack/compilers/gcc.py

@@ -56,7 +56,7 @@ def fc_version(cls, fc):
         return get_compiler_version(
             fc, '-dumpversion',
             # older gfortran versions don't have simple dumpversion output.
-            r'(?:GNU Fortran \(GCC\))?(\d+\.\d+\.\d+)')
+            r'(?:GNU Fortran \(GCC\))?(\d+\.\d+(?:\.\d+)?)')
 
 
     @classmethod

lib/spack/spack/package.py

@@ -859,7 +859,7 @@ def find_versions_of_archive(archive_url, **kwargs):
     list_depth = kwargs.get('list_depth', 1)
 
     if not list_url:
-        list_url = os.path.dirname(archive_url)
+        list_url = url.find_list_url(archive_url)
 
     # This creates a regex from the URL with a capture group for the
     # version part of the URL.  The capture group is converted to a

lib/spack/spack/url.py

@@ -78,6 +78,26 @@ def __init__(self, path):
             "Couldn't parse package name in: " + path, path)
 
 
+def find_list_url(url):
+    """Finds a good list URL for the supplied URL.  This depends on
+       the site.  By default, just assumes that a good list URL is the
+       dirname of an archive path.  For github URLs, this returns the
+       URL of the project's releases page.
+    """
+
+    url_types = [
+        # e.g. https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz
+        (r'^(https://github.com/[^/]+/[^/]+)/archive/', lambda m: m.group(1) + '/releases')
+        ]
+
+    for pattern, fun in url_types:
+        match = re.search(pattern, url)
+        if match:
+            return fun(match)
+    else:
+        return os.path.dirname(url)
+
+
 def parse_version_string_with_indices(path):
     """Try to extract a version string from a filename or URL.  This is taken
        largely from Homebrew's Version class."""

var/spack/packages/automaded/package.py

@@ -0,0 +1,51 @@
+##############################################################################
+# Copyright (c) 2013, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://scalability-llnl.github.io/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+class Automaded(Package):
+    """AutomaDeD (Automata-based Debugging for Dissimilar parallel
+       tasks) is a tool for automatic diagnosis of performance and
+       correctness problems in MPI applications. It creates
+       control-flow models of each MPI process and, when a failure
+       occurs, these models are leveraged to find the origin of
+       problems automatically. MPI calls are intercepted (using
+       wrappers) to create the models. When an MPI application hangs,
+       AutomaDeD creates a progress-dependence graph that helps
+       finding the process (or group of processes) that caused the hang.
+    """
+
+    homepage = "https://github.com/scalability-llnl/AutomaDeD"
+    url      = "https://github.com/scalability-llnl/AutomaDeD/archive/v1.0.tar.gz"
+
+    version('1.0', '16a3d4def2c4c77d0bc4b21de8b3ab03')
+
+    depends_on('mpi')
+    depends_on('boost')
+    depends_on('callpath')
+
+    def install(self, spec, prefix):
+        cmake("-DSTATE_TRACKER_WITH_CALLPATH=ON", *std_cmake_args)
+        make()
+        make("install")

var/spack/packages/callpath/package.py

@@ -31,6 +31,7 @@ class Callpath(Package):
     homepage = "https://github.com/scalability-llnl/callpath"
     url      = "https://github.com/scalability-llnl/callpath/archive/v1.0.1.tar.gz"
 
+    version('1.0.2', 'b1994d5ee7c7db9d27586fc2dcf8f373')
     version('1.0.1', '0047983d2a52c5c335f8ba7f5bab2325')
 
     depends_on("dyninst")

var/spack/packages/mpich/package.py

@@ -23,6 +23,7 @@
 # Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ##############################################################################
 from spack import *
+import os
 
 class Mpich(Package):
     """MPICH is a high performance and widely portable implementation of
@@ -38,8 +39,41 @@ class Mpich(Package):
     provides('mpi@:1', when='@1:')
 
     def install(self, spec, prefix):
-        configure(
+        config_args = ["--prefix=" + prefix,
-            "--prefix=" + prefix,
+                       "--enable-shared"]
-            "--enable-shared")
+
+        # TODO: Spack should make it so that you can't actually find
+        # these compilers if they're "disabled" for the current
+        # compiler configuration.
+        if not self.compiler.f77:
+            config_args.append("--disable-f77")
+
+        if not self.compiler.fc:
+            config_args.append("--disable-fc")
+
+        configure(*config_args)
         make()
         make("install")
+
+        self.filter_compilers()
+
+
+    def filter_compilers(self):
+        """Run after install to make the MPI compilers use the
+           compilers that Spack built the package with.
+
+           If this isn't done, they'll have CC, CXX, F77, and FC set
+           to Spack's generic cc, c++, f77, and f90.  We want them to
+           be bound to whatever compiler they were built with.
+        """
+        bin = self.prefix.bin
+        mpicc  = os.path.join(bin, 'mpicc')
+        mpicxx = os.path.join(bin, 'mpicxx')
+        mpif77 = os.path.join(bin, 'mpif77')
+        mpif90 = os.path.join(bin, 'mpif90')
+
+        kwargs = { 'ignore_absent' : True, 'backup' : False, 'string' : True }
+        filter_file('CC="cc"',   'CC="%s"'  % self.compiler.cc,  mpicc,  **kwargs)
+        filter_file('CXX="c++"', 'CXX="%s"' % self.compiler.cxx, mpicxx, **kwargs)
+        filter_file('F77="f77"', 'F77="%s"' % self.compiler.f77, mpif77, **kwargs)
+        filter_file('FC="f90"',  'FC="%s"'  % self.compiler.fc,  mpif90, **kwargs)

var/spack/packages/openmpi/package.py

@@ -10,22 +10,32 @@ class Openmpi(Package):
     """
 
     homepage = "http://www.open-mpi.org"
-    url      = "http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.5.tar.bz2"
 
-    version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475')
+    version('1.8.2', 'ab538ed8e328079d566fc797792e016e',
+            url='http://www.open-mpi.org/software/ompi/v1.8/downloads/openmpi-1.8.2.tar.gz')
+
+    version('1.6.5', '03aed2a4aa4d0b27196962a2a65fc475',
+            url = "http://www.open-mpi.org/software/ompi/v1.6/downloads/openmpi-1.6.5.tar.bz2")
+    patch('ad_lustre_rwcontig_open_source.patch', when="@1.6.5")
+    patch('llnl-platforms.patch', when="@1.6.5")
 
     provides('mpi@:2')
 
-    patch('ad_lustre_rwcontig_open_source.patch')
-    patch('llnl-platforms.patch')
-
     def install(self, spec, prefix):
-        configure("--prefix=%s" % prefix,
+        config_args = ["--prefix=%s" % prefix]
-                  "--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas")
 
-        # TODO: implement variants next, so we can have LLNL and LANL options.
+        # TODO: use variants for this, e.g. +lanl, +llnl, etc.
-        # use above for LANL builds, but for LLNL builds, we need this
+        # use this for LANL builds, but for LLNL builds, we need:
-        #     "--with-platform=contrib/platform/llnl/optimized")
+        #     "--with-platform=contrib/platform/llnl/optimized"
+        if self.version == ver("1.6.5"):
+            confg_args.append("--with-platform=contrib/platform/lanl/tlcc2/optimized-nopanasas")
 
+        # TODO: Spack should make it so that you can't actually find
+        # these compilers if they're "disabled" for the current
+        # compiler configuration.
+        if not self.compiler.f77 and not self.compiler.fc:
+            config_args.append("--enable-mpi-fortran=no")
+
+        configure(*config_args)
         make()
         make("install")