Merge remote-tracking branch 'upstream/develop' into pkg-graphviz

lib/spack/docs/packaging_guide.rst

@@ -1844,6 +1844,20 @@ dedicated process.
 
 .. _prefix-objects:
 
+
+Failing the build
+----------------------
+
+Sometimes you don't want a package to successfully install unless some
+condition is true.  You can explicitly cause the build to fail from
+``install()`` by raising an ``InstallError``, for example:
+
+.. code-block:: python
+
+   if spec.architecture.startswith('darwin'):
+       raise InstallError('This package does not build on Mac OS X!')
+
+
 Prefix objects
 ----------------------
 
@@ -2160,6 +2174,62 @@ package, this allows us to avoid race conditions in the library's
 build system.
 
 
+.. _sanity-checks:
+
+Sanity checking an intallation
+--------------------------------
+
+By default, Spack assumes that a build has failed if nothing is
+written to the install prefix, and that it has succeeded if anything
+(a file, a directory, etc.)  is written to the install prefix after
+``install()`` completes.
+
+Consider a simple autotools build like this:
+
+.. code-block:: python
+
+   def install(self, spec, prefix):
+       configure("--prefix=" + prefix)
+       make()
+       make("install")
+
+If you are using using standard autotools or CMake, ``configure`` and
+``make`` will not write anything to the install prefix.  Only ``make
+install`` writes the files, and only once the build is already
+complete.  Not all builds are like this.  Many builds of scientific
+software modify the install prefix *before* ``make install``. Builds
+like this can falsely report that they were successfully installed if
+an error occurs before the install is complete but after files have
+been written to the ``prefix``.
+
+
+``sanity_check_is_file`` and ``sanity_check_is_dir``
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+You can optionally specify *sanity checks* to deal with this problem.
+Add properties like this to your package:
+
+.. code-block:: python
+
+   class MyPackage(Package):
+       ...
+
+       sanity_check_is_file = ['include/libelf.h']
+       sanity_check_is_dir  = [lib]
+
+       def install(self, spec, prefix):
+           configure("--prefix=" + prefix)
+           make()
+           make("install")
+
+Now, after ``install()`` runs, Spack will check whether
+``$prefix/include/libelf.h`` exists and is a file, and whether
+``$prefix/lib`` exists and is a directory.  If the checks fail, then
+the build will fail and the install prefix will be removed.  If they
+succeed, Spack considers the build succeeful and keeps the prefix in
+place.
+
+
 .. _file-manipulation:
 
 File manipulation functions

lib/spack/spack/__init__.py

@@ -188,3 +188,10 @@
 import spack.util.executable
 from spack.util.executable import *
 __all__ += spack.util.executable.__all__
+
+from spack.package import \
+    install_dependency_symlinks, flatten_dependencies, DependencyConflictError, \
+    InstallError, ExternalPackageError
+__all__ += [
+    'install_dependency_symlinks', 'flatten_dependencies', 'DependencyConflictError',
+    'InstallError', 'ExternalPackageError']

lib/spack/spack/compilers/gcc.py

@@ -40,7 +40,8 @@ class Gcc(Compiler):
     fc_names = ['gfortran']
 
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
-    suffixes = [r'-mp-\d\.\d']
+    # Homebrew and Linuxes may build gcc with -X, -X.Y suffixes
+    suffixes = [r'-mp-\d\.\d', r'-\d\.\d', r'-\d']
 
     # Named wrapper links within spack.build_env_path
     link_paths = {'cc'  : 'gcc/gcc',

lib/spack/spack/package.py

@@ -318,16 +318,17 @@ class SomePackage(Package):
     """Most packages are NOT extendable.  Set to True if you want extensions."""
     extendable = False
 
-    """List of prefix-relative file paths. If these do not exist after
+    """List of prefix-relative file paths (or a single path). If these do
-       install, or if they exist but are not files, sanity checks fail.
+       not exist after install, or if they exist but are not files,
+       sanity checks fail.
     """
-    sanity_check_files = []
+    sanity_check_is_file = []
 
-    """List of prefix-relative directory paths. If these do not exist
+    """List of prefix-relative directory paths (or a single path). If
-       after install, or if they exist but are not directories, sanity
+       these do not exist after install, or if they exist but are not
-       checks will fail.
+       directories, sanity checks will fail.
     """
-    sanity_check_dirs = []
+    sanity_check_is_dir = []
 
 
     def __init__(self, spec):
@@ -966,14 +967,17 @@ def build_process():
     def sanity_check_prefix(self):
         """This function checks whether install succeeded."""
         def check_paths(path_list, filetype, predicate):
+            if isinstance(path_list, basestring):
+                path_list = [path_list]
+
             for path in path_list:
                 abs_path = os.path.join(self.prefix, path)
                 if not predicate(abs_path):
                     raise InstallError("Install failed for %s. No such %s in prefix: %s"
                                        % (self.name, filetype, path))
 
-        check_paths(self.sanity_check_files, 'file', os.path.isfile)
+        check_paths(self.sanity_check_is_file, 'file', os.path.isfile)
-        check_paths(self.sanity_check_dirs, 'directory', os.path.isdir)
+        check_paths(self.sanity_check_is_dir, 'directory', os.path.isdir)
 
         installed = set(os.listdir(self.prefix))
         installed.difference_update(spack.install_layout.hidden_file_paths)
@@ -1239,6 +1243,27 @@ def rpath_args(self):
         return " ".join("-Wl,-rpath,%s" % p for p in self.rpath)
 
 
+def install_dependency_symlinks(pkg, spec, prefix):
+    """Execute a dummy install and flatten dependencies"""
+    flatten_dependencies(spec, prefix)
+
+def flatten_dependencies(spec, flat_dir):
+    """Make each dependency of spec present in dir via symlink."""
+    for dep in spec.traverse(root=False):
+        name = dep.name
+
+        dep_path = spack.install_layout.path_for_spec(dep)
+        dep_files = LinkTree(dep_path)
+
+        os.mkdir(flat_dir+'/'+name)
+
+        conflict = dep_files.find_conflict(flat_dir+'/'+name)
+        if conflict:
+            raise DependencyConflictError(conflict)
+
+        dep_files.merge(flat_dir+'/'+name)
+
+
 def validate_package_url(url_string):
     """Determine whether spack can handle a particular URL or not."""
     url = urlparse(url_string)
@@ -1326,6 +1351,10 @@ def __init__(self, message, long_msg=None):
         super(InstallError, self).__init__(message, long_msg)
 
 
+class ExternalPackageError(InstallError):
+    """Raised by install() when a package is only for external use."""
+
+
 class PackageStillNeededError(InstallError):
     """Raised when package is still needed by another on uninstall."""
     def __init__(self, spec, dependents):
@@ -1376,3 +1405,11 @@ def __init__(self, path):
 class ActivationError(ExtensionError):
     def __init__(self, msg, long_msg=None):
         super(ActivationError, self).__init__(msg, long_msg)
+
+
+class DependencyConflictError(spack.error.SpackError):
+    """Raised when the dependencies cannot be flattened as asked for."""
+    def __init__(self, conflict):
+        super(DependencyConflictError, self).__init__(
+            "%s conflicts with another file in the flattened directory." %(
+                conflict))

lib/spack/spack/url.py

@@ -142,7 +142,7 @@ def split_url_extension(path):
 
 def downloaded_file_extension(path):
     """This returns the type of archive a URL refers to.  This is
-       sometimes confusing becasue of URLs like:
+       sometimes confusing because of URLs like:
 
            (1) https://github.com/petdance/ack/tarball/1.93_02
 

lib/spack/spack/util/compression.py

@@ -27,13 +27,12 @@
 from itertools import product
 from spack.util.executable import which
 
-# Supported archvie extensions.
+# Supported archive extensions.
 PRE_EXTS = ["tar"]
 EXTS     = ["gz", "bz2", "xz", "Z", "zip", "tgz"]
 
-# Add EXTS last so that .tar.gz is matched *before* tar.gz
+# Add PRE_EXTS and EXTS last so that .tar.gz is matched *before* .tar or .gz
-ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + EXTS
+ALLOWED_ARCHIVE_TYPES = [".".join(l) for l in product(PRE_EXTS, EXTS)] + PRE_EXTS + EXTS
-
 
 def allowed_archive(path):
     return any(path.endswith(t) for t in ALLOWED_ARCHIVE_TYPES)

share/spack/setup-env.sh

@@ -141,7 +141,7 @@ function _spack_pathadd {
     fi
 
     # Do the actual prepending here.
-    eval "_pa_oldvalue=\$${_pa_varname}"
+    eval "_pa_oldvalue=\${${_pa_varname}:-}"
 
     if [ -d "$_pa_new_path" ] && [[ ":$_pa_oldvalue:" != *":$_pa_new_path:"* ]]; then
         if [ -n "$_pa_oldvalue" ]; then

var/spack/repos/builtin/packages/curl/package.py

@@ -7,6 +7,7 @@ class Curl(Package):
     homepage = "http://curl.haxx.se"
     url      = "http://curl.haxx.se/download/curl-7.46.0.tar.bz2"
 
+    version('7.47.1', '9ea3123449439bbd960cd25cf98796fb')
     version('7.46.0', '9979f989a2a9930d10f1b3deeabc2148')
     version('7.45.0', '62c1a352b28558f25ba6209214beadc8')
     version('7.44.0', '6b952ca00e5473b16a11f05f06aa8dae')

var/spack/repos/builtin/packages/libelf/package.py

@@ -38,8 +38,7 @@ class Libelf(Package):
 
     provides('elf')
 
-    sanity_check_files = ['include/libelf.h']
+    sanity_check_is_file = 'include/libelf.h'
-    sanity_check_dirs  = ['lib']
 
     def install(self, spec, prefix):
         configure("--prefix=" + prefix,

var/spack/repos/builtin/packages/netlib-scalapack/package.py

@@ -1,4 +1,5 @@
 from spack import *
+import sys
 
 class NetlibScalapack(Package):
     """ScaLAPACK is a library of high-performance linear algebra routines for parallel distributed memory machines"""
@@ -41,8 +42,8 @@ def install(self, spec, prefix):
             make("install")
 
     def setup_dependent_environment(self, module, spec, dependent_spec):
-        # TODO treat OS that are not Linux...
+        lib_dsuffix = '.dylib' if sys.platform == 'darwin' else '.so'
-        lib_suffix = '.so' if '+shared' in spec['scalapack'] else '.a'
+        lib_suffix = lib_dsuffix if '+shared' in spec['scalapack'] else '.a'
 
         spec['scalapack'].fc_link = '-L%s -lscalapack' % spec['scalapack'].prefix.lib
         spec['scalapack'].cc_link = spec['scalapack'].fc_link

var/spack/repos/builtin/packages/openblas/package.py

@@ -1,10 +1,12 @@
 from spack import *
+import sys
 
 class Openblas(Package):
     """OpenBLAS: An optimized BLAS library"""
     homepage = "http://www.openblas.net"
     url      = "http://github.com/xianyi/OpenBLAS/archive/v0.2.15.tar.gz"
 
+    version('0.2.16', 'fef46ab92463bdbb1479dcec594ef6dc')
     version('0.2.15', 'b1190f3d3471685f17cfd1ec1d252ac9')
 
     # virtual dependency
@@ -15,13 +17,14 @@ def install(self, spec, prefix):
         make('libs', 'netlib', 'shared', 'CC=cc', 'FC=f77')
         make('install', "PREFIX='%s'" % prefix)
 
+        lib_dsuffix = 'dylib' if sys.platform == 'darwin' else 'so'
         # Blas virtual package should provide blas.a and libblas.a
         with working_dir(prefix.lib):
             symlink('libopenblas.a', 'blas.a')
             symlink('libopenblas.a', 'libblas.a')
-            symlink('libopenblas.so', 'libblas.so')
+            symlink('libopenblas.%s' % lib_dsuffix, 'libblas.%s' % lib_dsuffix)
 
         # Lapack virtual package should provide liblapack.a
         with working_dir(prefix.lib):
             symlink('libopenblas.a', 'liblapack.a')
-            symlink('libopenblas.so', 'liblapack.so')
+            symlink('libopenblas.%s' % lib_dsuffix, 'liblapack.%s' % lib_dsuffix)

var/spack/repos/builtin/packages/papi/package.py

@@ -13,6 +13,7 @@ class Papi(Package):
     homepage = "http://icl.cs.utk.edu/papi/index.html"
 
     url      = "http://icl.cs.utk.edu/projects/papi/downloads/papi-5.4.1.tar.gz"
+    version('5.4.3', '3211b5a5bb389fe692370f5cf4cc2412')
     version('5.4.1', '9134a99219c79767a11463a76b0b01a2')
     version('5.3.0', '367961dd0ab426e5ae367c2713924ffb')
 

var/spack/repos/builtin/packages/superlu-dist/package.py

@@ -0,0 +1,63 @@
+from spack import *
+
+class SuperluDist(Package):
+    """A general purpose library for the direct solution of large, sparse, nonsymmetric systems of linear equations on high performance machines."""
+    homepage = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/"
+    url      = "http://crd-legacy.lbl.gov/~xiaoye/SuperLU/superlu_dist_4.1.tar.gz"
+
+    version('4.3', 'ee66c84e37b4f7cc557771ccc3dc43ae')
+    version('4.2', 'ae9fafae161f775fbac6eba11e530a65')
+    version('4.1', '4edee38cc29f687bd0c8eb361096a455')
+    version('4.0', 'c0b98b611df227ae050bc1635c6940e0')
+
+    depends_on ('mpi')
+    depends_on ('blas')
+    depends_on ('lapack')
+    depends_on ('parmetis')
+    depends_on ('metis')
+
+    def install(self, spec, prefix):
+        makefile_inc = []
+        makefile_inc.extend([
+            'PLAT         = _mac_x',
+            'DSuperLUroot = %s' % self.stage.source_path, #self.stage.path, prefix
+            'DSUPERLULIB  = $(DSuperLUroot)/lib/libsuperlu_dist.a',
+            'BLASDEF      = -DUSE_VENDOR_BLAS',
+            'BLASLIB      = -L%s -llapack %s -lblas' % (spec['lapack'].prefix.lib, spec['blas'].prefix.lib), # FIXME: avoid hardcoding blas/lapack lib names
+            'METISLIB     = -L%s -lmetis' % spec['metis'].prefix.lib,
+            'PARMETISLIB  = -L%s -lparmetis' % spec['parmetis'].prefix.lib,
+            'FLIBS        =',
+            'LIBS         = $(DSUPERLULIB) $(BLASLIB) $(PARMETISLIB) $(METISLIB)',
+            'ARCH         = ar',
+            'ARCHFLAGS    = cr',
+            'RANLIB       = true',
+            'CC           = mpicc', # FIXME avoid hardcoding MPI compiler names
+            'CFLAGS       = -fPIC -std=c99 -O2 -I%s -I%s' %(spec['parmetis'].prefix.include, spec['metis'].prefix.include),
+            'NOOPTS       = -fPIC -std=c99',
+            'FORTRAN      = mpif77',
+            'F90FLAGS     = -O2',
+            'LOADER       = mpif77',
+            'LOADOPTS     =',
+            'CDEFS        = -DAdd_'
+            ])
+
+        #with working_dir('src'):
+        with open('make.inc', 'w') as fh:
+            fh.write('\n'.join(makefile_inc))
+
+        make("lib", parallel=False)
+
+        # FIXME:
+        # cd "EXAMPLE" do
+        # system "make"
+
+        # need to install by hand
+        headers_location = join_path(self.prefix.include,'superlu_dist')
+        mkdirp(headers_location)
+        # FIXME: fetch all headers in the folder automatically
+        for header in ['Cnames.h','cublas_utils.h','dcomplex.h','html_mainpage.h','machines.h','old_colamd.h','psymbfact.h','superlu_ddefs.h','superlu_defs.h','superlu_enum_consts.h','superlu_zdefs.h','supermatrix.h','util_dist.h']:
+            superludist_header = join_path(self.stage.source_path, 'SRC/',header)
+            install(superludist_header, headers_location)
+
+        superludist_lib = join_path(self.stage.source_path, 'lib/libsuperlu_dist.a')
+        install(superludist_lib,self.prefix.lib)