Add a SConsPackage base class (#4936)

* Add a SConsPackage base class

* Make Matlab extendable

* Most dependencies are actually required

* Cantera requires older version of fmt

lib/spack/docs/packaging_guide.rst

@@ -2121,6 +2121,9 @@ The classes that are currently provided by Spack are:
     | :py:class:`.QMakePackage`     | Specialized class for packages   |
     |                               | build using QMake                |
     +-------------------------------+----------------------------------+
+    | :py:class:`.SConsPackage`     | Specialized class for packages   |
+    |                               | built using SCons                |
+    +-------------------------------+----------------------------------+
     | :py:class:`.WafPackage`       | Specialized class for packages   |
     |                               | built using Waf                  |
     +-------------------------------+----------------------------------+

lib/spack/spack/__init__.py

@@ -173,6 +173,7 @@
 from spack.build_systems.autotools import AutotoolsPackage
 from spack.build_systems.cmake import CMakePackage
 from spack.build_systems.qmake import QMakePackage
+from spack.build_systems.scons import SConsPackage
 from spack.build_systems.waf import WafPackage
 from spack.build_systems.python import PythonPackage
 from spack.build_systems.r import RPackage
@@ -187,6 +188,7 @@
     'AutotoolsPackage',
     'CMakePackage',
     'QMakePackage',
+    'SConsPackage',
     'WafPackage',
     'PythonPackage',
     'RPackage',

lib/spack/spack/build_systems/scons.py

@@ -0,0 +1,92 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+
+import inspect
+
+from spack.directives import depends_on
+from spack.package import PackageBase, run_after
+
+
+class SConsPackage(PackageBase):
+    """Specialized class for packages built using SCons.
+
+    See http://scons.org/documentation.html for more information.
+
+    This class provides the following phases that can be overridden:
+
+    1. :py:meth:`~.SConsPackage.build`
+    2. :py:meth:`~.SConsPackage.install`
+
+    Packages that use SCons as a build system are less uniform than packages
+    that use other build systems. Developers can add custom subcommands or
+    variables that control the build. You will likely need to override
+    :py:meth:`~.SConsPackage.build_args` to pass the appropriate variables.
+    """
+    #: Phases of a SCons package
+    phases = ['build', 'install']
+
+    #: To be used in UI queries that require to know which
+    #: build-system class we are using
+    build_system_class = 'SConsPackage'
+
+    #: Callback names for build-time test
+    build_time_test_callbacks = ['test']
+
+    depends_on('scons', type='build')
+
+    def build_args(self, spec, prefix):
+        """Arguments to pass to build."""
+        return []
+
+    def build(self, spec, prefix):
+        """Build the package."""
+        args = self.build_args(spec, prefix)
+
+        inspect.getmodule(self).scons(*args)
+
+    def install_args(self, spec, prefix):
+        """Arguments to pass to install."""
+        return []
+
+    def install(self, spec, prefix):
+        """Install the package."""
+        args = self.install_args(spec, prefix)
+
+        inspect.getmodule(self).scons('install', *args)
+
+    # Testing
+
+    def test(self):
+        """Run unit tests after build.
+
+        By default, does nothing. Override this if you want to
+        add package-specific tests.
+        """
+        pass
+
+    run_after('build')(PackageBase._run_default_build_time_test_callbacks)
+
+    # Check that self.prefix is there after installation
+    run_after('install')(PackageBase.sanity_check_prefix)

lib/spack/spack/cmd/build.py

@@ -35,6 +35,7 @@
     AutotoolsPackage: 'build',
     CMakePackage: 'build',
     QMakePackage: 'build',
+    SConsPackage: 'build',
     WafPackage: 'build',
     PythonPackage: 'build',
     PerlPackage: 'build',

lib/spack/spack/cmd/create.py

@@ -208,15 +208,14 @@ def qmake_args(self):
 class SconsPackageTemplate(PackageTemplate):
     """Provides appropriate overrides for SCons-based packages"""
 
-    dependencies = """\
+    base_class_name = 'SConsPackage'
-    # FIXME: Add additional dependencies if required.
-    depends_on('scons', type='build')"""
 
     body = """\
-    def install(self, spec, prefix):
+    def build_args(self, spec, prefix):
-        # FIXME: Add logic to build and install here.
+        # FIXME: Add arguments to pass to build.
-        scons('prefix={0}'.format(prefix))
+        # FIXME: If not needed delete this function
-        scons('install')"""
+        args = []
+        return args"""
 
 
 class WafPackageTemplate(PackageTemplate):

var/spack/repos/builtin/packages/cantera/package.py

@@ -26,132 +26,147 @@
 import os
 
 
-class Cantera(Package):
+class Cantera(SConsPackage):
     """Cantera is a suite of object-oriented software tools for problems
     involving chemical kinetics, thermodynamics, and/or transport processes."""
 
     homepage = "http://www.cantera.org/docs/sphinx/html/index.html"
-    url      = "https://github.com/Cantera/cantera/archive/v2.2.1.tar.gz"
+    url      = "https://github.com/Cantera/cantera/archive/v2.3.0.tar.gz"
 
+    version('2.3.0', 'aebbd8d891cb1623604245398502b72e')
     version('2.2.1', '9d1919bdef39ddec54485fc8a741a3aa')
 
-    variant('lapack',     default=True,
-            description='Build with external BLAS/LAPACK libraries')
-    variant('threadsafe', default=True,
-            description='Build threadsafe, requires Boost')
-    variant('sundials',   default=True,
-            description='Build with external Sundials')
     variant('python',     default=False,
             description='Build the Cantera Python module')
     variant('matlab',     default=False,
             description='Build the Cantera Matlab toolbox')
 
     # Required dependencies
-    depends_on('scons', type='build')
+    depends_on('fmt@3.0.0:3.0.2', when='@2.3.0:')
-
+    depends_on('googletest',      when='@2.3.0:')
-    # Recommended dependencies
+    depends_on('eigen',           when='@2.3.0:')
-    depends_on('blas',      when='+lapack')
+    depends_on('boost')
-    depends_on('lapack',    when='+lapack')
+    depends_on('sundials')  # must be compiled with -fPIC
-    depends_on('boost',     when='+threadsafe')
+    depends_on('blas')
-    depends_on('sundials',  when='+sundials')  # must be compiled with -fPIC
+    depends_on('lapack')
 
     # Python module dependencies
     extends('python', when='+python')
+    depends_on('py-cython', when='+python', type='build')
     depends_on('py-numpy',  when='+python', type=('build', 'run'))
     depends_on('py-scipy',  when='+python', type=('build', 'run'))
-    depends_on('py-cython', when='+python', type=('build', 'run'))
     depends_on('py-3to2',   when='+python', type=('build', 'run'))
     # TODO: these "when" specs don't actually work
     # depends_on('py-unittest2',     when='+python^python@2.6', type=('build', 'run'))  # noqa
     # depends_on('py-unittest2py3k', when='+python^python@3.1', type=('build', 'run'))  # noqa
 
     # Matlab toolbox dependencies
-    # TODO: add Matlab package
+    extends('matlab', when='+matlab')
-    # TODO: allow packages to extend multiple other packages
-    # extends('matlab',   when='+matlab')
 
-    def install(self, spec, prefix):
+    def build_args(self, spec, prefix):
-        # Required options
+        # Valid args can be found by running `scons help`
-        options = [
+
+        # Required args
+        args = [
+            'build',
             'prefix={0}'.format(prefix),
-            'CC={0}'.format(os.environ['CC']),
+            'VERBOSE=yes',
-            'CXX={0}'.format(os.environ['CXX']),
+            'CC={0}'.format(spack_cc),
-            'F77={0}'.format(os.environ['F77']),
+            'CXX={0}'.format(spack_cxx),
-            'FORTRAN={0}'.format(os.environ['FC']),
+            'FORTRAN={0}'.format(spack_fc),
             'cc_flags={0}'.format(self.compiler.pic_flag),
             # Allow Spack environment variables to propagate through to SCons
             'env_vars=all'
         ]
 
-        # BLAS/LAPACK support
+        if spec.satisfies('@:2.2.1'):
-        if '+lapack' in spec:
+            args.append('F77={0}'.format(spack_f77))
-            lapack_blas = spec['lapack'].libs + spec['blas'].libs
+
-            options.extend([
+        # fmt support
-                'blas_lapack_libs={0}'.format(','.join(lapack_blas.names)),
+        if spec.satisfies('@2.3.0:'):
-                'blas_lapack_dir={0}'.format(spec['lapack'].prefix.lib)
+            args.append('system_fmt=y')
+
+        # Googletest support
+        if spec.satisfies('@2.3.0:'):
+            args.append('system_googletest=y')
+
+        # Eigen support
+        if spec.satisfies('@2.3.0:'):
+            args.extend([
+                'system_eigen=y',
+                'extra_inc_dirs={0}'.format(
+                    join_path(spec['eigen'].prefix.include, 'eigen{0}'.format(
+                        spec['eigen'].version.up_to(1)))),
             ])
 
-        # Threadsafe build, requires Boost
+        # BLAS/LAPACK support
-        if '+threadsafe' in spec:
+        lapack_blas = spec['lapack'].libs + spec['blas'].libs
-            options.extend([
+        args.extend([
+            'blas_lapack_libs={0}'.format(','.join(lapack_blas.names)),
+            'blas_lapack_dir={0}'.format(spec['lapack'].prefix.lib)
+        ])
+
+        # Boost support
+        if spec.satisfies('@2.3.0:'):
+            args.append('boost_inc_dir={0}'.format(
+                spec['boost'].prefix.include))
+        else:
+            args.extend([
                 'build_thread_safe=yes',
                 'boost_inc_dir={0}'.format(spec['boost'].prefix.include),
-                'boost_lib_dir={0}'.format(spec['boost'].prefix.lib)
+                'boost_lib_dir={0}'.format(spec['boost'].prefix.lib),
             ])
-        else:
-            options.append('build_thread_safe=no')
 
         # Sundials support
-        if '+sundials' in spec:
+        if spec.satisfies('@2.3.0:'):
-            options.extend([
+            args.append('system_sundials=y')
-                'use_sundials=y',
-                'sundials_include={0}'.format(spec['sundials'].prefix.include),
-                'sundials_libdir={0}'.format(spec['sundials'].prefix.lib),
-                'sundials_license={0}'.format(
-                    join_path(spec['sundials'].prefix, 'LICENSE'))
-            ])
         else:
-            options.append('use_sundials=n')
+            args.extend([
+                'use_sundials=y',
+                'sundials_license={0}'.format(
+                    spec['sundials'].prefix.LICENSE)
+            ])
+
+        args.extend([
+            'sundials_include={0}'.format(spec['sundials'].prefix.include),
+            'sundials_libdir={0}'.format(spec['sundials'].prefix.lib),
+        ])
 
         # Python module
         if '+python' in spec:
-            options.extend([
+            args.extend([
                 'python_package=full',
                 'python_cmd={0}'.format(spec['python'].command.path),
-                'python_array_home={0}'.format(spec['py-numpy'].prefix)
             ])
-            if spec['python'].satisfies('@3'):
+            if spec['python'].satisfies('@3:'):
-                options.extend([
+                args.extend([
                     'python3_package=y',
                     'python3_cmd={0}'.format(spec['python'].command.path),
-                    'python3_array_home={0}'.format(spec['py-numpy'].prefix)
                 ])
             else:
-                options.append('python3_package=n')
+                args.append('python3_package=n')
         else:
-            options.append('python_package=none')
+            args.append('python_package=none')
-            options.append('python3_package=n')
+            args.append('python3_package=n')
 
         # Matlab toolbox
         if '+matlab' in spec:
-            options.extend([
+            args.extend([
                 'matlab_toolbox=y',
                 'matlab_path={0}'.format(spec['matlab'].prefix)
             ])
         else:
-            options.append('matlab_toolbox=n')
+            args.append('matlab_toolbox=n')
 
-        scons('build', *options)
+        return args
 
-        if '+python' in spec:
+    def test(self):
+        if '+python' in self.spec:
             # Tests will always fail if Python dependencies aren't built
             # In addition, 3 of the tests fail when run in parallel
             scons('test', parallel=False)
 
-        scons('install')
+    @run_after('install')
-
-        self.filter_compilers()
-
     def filter_compilers(self):
         """Run after install to tell the Makefile and SConstruct files to use
         the compilers that Spack built the package with.

var/spack/repos/builtin/packages/fmt/package.py

@@ -0,0 +1,47 @@
+##############################################################################
+# Copyright (c) 2013-2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/llnl/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+
+class Fmt(CMakePackage):
+    """fmt (formerly cppformat) is an open-source formatting library.
+    It can be used as a safe alternative to printf or as a fast alternative
+    to C++ IOStreams."""
+
+    homepage = "http://fmtlib.net/latest/index.html"
+    url      = "https://github.com/fmtlib/fmt/releases/download/4.0.0/fmt-4.0.0.zip"
+
+    version('4.0.0', '605b5abee11b83195191234f4f414cf1')
+    version('3.0.2', 'b190a7b8f2a5e522ee70cf339a53d3b2')
+    version('3.0.1', '14505463b838befe1513b09cae112715')
+    version('3.0.0', 'c099561e70fa194bb03b3fd5de2d3fd0')
+
+    depends_on('cmake@2.8.12:', type='build')
+
+    def cmake_args(self):
+        return [
+            '-DCMAKE_C_FLAGS={0}'.format(self.compiler.pic_flag),
+            '-DCMAKE_CXX_FLAGS={0}'.format(self.compiler.pic_flag),
+        ]

var/spack/repos/builtin/packages/kahip/package.py

@@ -29,7 +29,7 @@
 import re
 
 
-class Kahip(Package):
+class Kahip(SConsPackage):
     """KaHIP - Karlsruhe High Quality Partitioning - is a family of graph
     partitioning programs. It includes KaFFPa (Karlsruhe Fast Flow
     Partitioner), which is a multilevel graph partitioning algorithm,
@@ -46,17 +46,25 @@ class Kahip(Package):
     url       = 'http://algo2.iti.kit.edu/schulz/software_releases/KaHIP_2.00.tar.gz'
 
     version('develop', git='https://github.com/schulzchristian/KaHIP.git')
-    version('2.00', '9daeda32f43c90570ed436d5d93c8a872b1a14d8')
+    version('2.00', '0a66b0a604ad72cfb7e3dce00e2c9fdfac82b855')
 
     depends_on('argtable')
     depends_on('mpi')  # Note: upstream package only tested on openmpi
-    depends_on('scons', type='build')
 
-    phases = ['build', 'install']
+    conflicts('%clang')
 
-    #
+    def patch(self):
-    # - End of definitions / setup -
+        """Internal compile.sh scripts hardcode number of cores to build with.
-    #
+        Filter these out so Spack can control it."""
+
+        files = [
+            'compile.sh',
+            'parallel/modified_kahip/compile.sh',
+            'parallel/parallel_src/compile.sh',
+        ]
+
+        for f in files:
+            filter_file('NCORES=.*', 'NCORES={0}'.format(make_jobs), f)
 
     def build(self, spec, prefix):
         """Build using the KaHIP compile.sh script. Uses scons internally."""

var/spack/repos/builtin/packages/matlab/package.py

@@ -66,6 +66,8 @@ class Matlab(Package):
     license_vars     = ['LM_LICENSE_FILE']
     license_url      = 'https://www.mathworks.com/help/install/index.html'
 
+    extendable = True
+
     def url_for_version(self, version):
         return "file://{0}/matlab_{1}_glnxa64.zip".format(os.getcwd(), version)
 

var/spack/repos/builtin/packages/serf/package.py

@@ -25,31 +25,67 @@
 from spack import *
 
 
-class Serf(Package):
+class Serf(SConsPackage):
     """Apache Serf - a high performance C-based HTTP client library
     built upon the Apache Portable Runtime (APR) library"""
 
     homepage  = 'https://serf.apache.org/'
-    url       = 'https://archive.apache.org/dist/serf/serf-1.3.8.tar.bz2'
+    url       = 'https://archive.apache.org/dist/serf/serf-1.3.9.tar.bz2'
 
+    version('1.3.9', '26015c63e3bbb108c1689bf2090e4c26351db674')
     version('1.3.8', '1d45425ca324336ce2f4ae7d7b4cfbc5567c5446')
 
+    variant('debug', default=False,
+            description='Enable debugging info and strict compile warnings')
+
+    depends_on('scons@2.3.0:', type='build')
+
     depends_on('apr')
     depends_on('apr-util')
-    depends_on('scons', type='build')
-    depends_on('expat')
     depends_on('openssl')
     depends_on('zlib')
 
-    def install(self, spec, prefix):
+    def build_args(self, spec, prefix):
-        options = ['PREFIX=%s' % prefix]
+        args = [
-        options.append('APR=%s' % spec['apr'].prefix)
+            'PREFIX={0}'.format(prefix),
-        options.append('APU=%s' % spec['apr-util'].prefix)
+            'APR={0}'.format(spec['apr'].prefix),
-        options.append('OPENSSL=%s' % spec['openssl'].prefix)
+            'APU={0}'.format(spec['apr-util'].prefix),
-        options.append('LINKFLAGS=-L%s/lib -L%s/lib' %
+            'OPENSSL={0}'.format(spec['openssl'].prefix),
-                       (spec['expat'].prefix, spec['zlib'].prefix))
+            'ZLIB={0}'.format(spec['zlib'].prefix),
-        options.append('CPPFLAGS=-I%s/include -I%s/include' %
+        ]
-                       (spec['expat'].prefix, spec['zlib'].prefix))
 
-        scons(*options)
+        if '+debug' in spec:
-        scons('install')
+            args.append('DEBUG=yes')
+        else:
+            args.append('DEBUG=no')
+
+        # SCons doesn't pass Spack environment variables to the
+        # execution environment. Therefore, we can't use Spack's compiler
+        # wrappers. Use the actual compilers. SCons seems to RPATH things
+        # on its own anyway.
+        args.append('CC={0}'.format(self.compiler.cc))
+
+        return args
+
+    def test(self):
+        # FIXME: Several test failures:
+        #
+        # There were 14 failures:
+        # 1) test_ssl_trust_rootca
+        # 2) test_ssl_certificate_chain_with_anchor
+        # 3) test_ssl_certificate_chain_all_from_server
+        # 4) test_ssl_no_servercert_callback_allok
+        # 5) test_ssl_large_response
+        # 6) test_ssl_large_request
+        # 7) test_ssl_client_certificate
+        # 8) test_ssl_future_server_cert
+        # 9) test_setup_ssltunnel
+        # 10) test_ssltunnel_basic_auth
+        # 11) test_ssltunnel_basic_auth_server_has_keepalive_off
+        # 12) test_ssltunnel_basic_auth_proxy_has_keepalive_off
+        # 13) test_ssltunnel_basic_auth_proxy_close_conn_on_200resp
+        # 14) test_ssltunnel_digest_auth
+        #
+        # These seem to be related to:
+        # https://groups.google.com/forum/#!topic/serf-dev/YEFTTdF1Qwc
+        scons('check')