Parquet: new packages (#8601)

* parquet: new package

Includes the following dependencies:

* arrow
* flatbuffers

Changes for compilation:

* snappy
* thrift
* zstd

* parquet: improve recipes (including dependencies)

* arrow: remove unused import in recipe

* parquet: fix line length for flake8/py27

* parquet, arrow: fix py26 format strings

* Address review comments.

* arrow: simplify recipe

var/spack/repos/builtin/packages/arrow/package.py

@@ -0,0 +1,74 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+
+class Arrow(CMakePackage):
+    """A cross-language development platform for in-memory data.
+
+    This package contains the C++ bindings.
+    """
+
+    homepage = "http://arrow.apache.org"
+    url      = "https://github.com/apache/arrow/archive/apache-arrow-0.9.0.tar.gz"
+
+    version('0.9.0', 'ebbd36c362b9e1d398ca612f6d2531ec')
+    version('0.8.0', '56436f6f61ccc68686b7e0ea30bf4d09')
+
+    depends_on('boost@1.60:')
+    depends_on('cmake@3.2.0:', type='build')
+    depends_on('flatbuffers@1.8.0 build_type=Release')  # only Release contains flatc
+    depends_on('rapidjson')
+    depends_on('snappy~shared')
+    depends_on('zlib+pic')
+    depends_on('zstd+pic')
+
+    variant('build_type', default='Release',
+            description='CMake build type',
+            values=('Debug', 'FastDebug', 'Release'))
+
+    root_cmakelists_dir = 'cpp'
+
+    def patch(self):
+        """Prevent `-isystem /usr/include` from appearing, since this confuses gcc.
+        """
+        filter_file(r'(include_directories\()SYSTEM ',
+                    r'\1',
+                    'cpp/cmake_modules/ThirdpartyToolchain.cmake')
+
+    def cmake_args(self):
+        args = [
+            "-DARROW_USE_SSE=ON",
+            "-DARROW_BUILD_SHARED=ON",
+            "-DARROW_BUILD_STATIC=OFF",
+            "-DARROW_BUILD_TESTS=OFF",
+            "-DARROW_WITH_BROTLI=OFF",
+            "-DARROW_WITH_LZ4=OFF",
+        ]
+        for dep in ('flatbuffers', 'rapidjson', 'snappy', 'zlib', 'zstd'):
+            args.append("-D{0}_HOME={1}".format(dep.upper(),
+                                                self.spec[dep].prefix))
+        args.append("-DZLIB_LIBRARIES={0}".format(self.spec['zlib'].libs))
+        return args

var/spack/repos/builtin/packages/flatbuffers/package.py

@@ -0,0 +1,36 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+
+class Flatbuffers(CMakePackage):
+    """Memory Efficient Serialization Library
+    """
+
+    homepage = "http://google.github.io/flatbuffers/"
+    url      = "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz"
+
+    version('1.9.0', '8be7513bf960034f6873326d09521a4b')
+    version('1.8.0', '276cab8303c4189cbe3b8a70e0515d65')

var/spack/repos/builtin/packages/parquet/package.py

@@ -0,0 +1,52 @@
+##############################################################################
+# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://github.com/spack/spack
+# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU Lesser General Public License (as
+# published by the Free Software Foundation) version 2.1, February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU Lesser General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public
+# License along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack import *
+
+
+class Parquet(CMakePackage):
+    """C++ bindings for the Apache Parquet columnar data format.
+    """
+
+    homepage = "https://github.com/apache/parquet-cpp"
+    url = "https://github.com/apache/parquet-cpp/archive/apache-parquet-cpp-1.4.0.tar.gz"
+
+    version('1.4.0', '3a3659e65052ef5a76fb88e4922283b9')
+
+    depends_on('arrow')
+    depends_on('boost')
+    depends_on('cmake@3.2.0:', type='build')
+    depends_on('pkgconfig', type='build')
+    depends_on('thrift+pic')
+
+    variant('build_type', default='Release',
+            description='CMake build type',
+            values=('Debug', 'FastDebug', 'Release'))
+
+    def cmake_args(self):
+        args = ['-DPARQUET_USE_SSE=OFF', '-DPARQUET_BUILD_TESTS=OFF']
+        for dep in ('arrow', 'thrift'):
+            args.append("-D{0}_HOME={1}".format(dep.upper(),
+                                                self.spec[dep].prefix))
+        return args

var/spack/repos/builtin/packages/snappy/package.py

@@ -34,6 +34,7 @@ class Snappy(CMakePackage):
     version('1.1.7', 'ee9086291c9ae8deb4dac5e0b85bf54a')
 
     variant('shared', default=True, description='Build shared libraries')
+    variant('pic', default=True, description='Build position independent code')
 
     def cmake_args(self):
         spec = self.spec
@@ -47,6 +48,12 @@ def cmake_args(self):
 
         return args
 
+    def flag_handler(self, name, flags):
+        flags = list(flags)
+        if '+pic' in self.spec and name in ('cflags', 'cxxflags'):
+            flags.append(self.compiler.pic_flag)
+        return (None, None, flags)
+
     @run_after('install')
     def install_pkgconfig(self):
         mkdirp(self.prefix.lib.pkgconfig)

var/spack/repos/builtin/packages/thrift/package.py

@@ -38,6 +38,7 @@ class Thrift(Package):
     homepage = "http://thrift.apache.org"
     url      = "http://apache.mirrors.ionfish.org/thrift/0.9.2/thrift-0.9.2.tar.gz"
 
+    version('0.11.0', '0be59730ebce071eceaf6bfdb8d3a20e')
     version('0.10.0', '795c5dd192e310ffff38cfd9430d6b29')
     version('0.9.3', '88d667a8ae870d5adeca8cb7d6795442')
     version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678')
@@ -45,6 +46,8 @@ class Thrift(Package):
     # Currently only support for c-family and python
     variant('c', default=True,
             description="Build support for C-family languages")
+    variant('pic', default=True,
+            description='Build position independent code')
     variant('python', default=True,
             description="Build support for python")
 
@@ -63,6 +66,11 @@ class Thrift(Package):
     depends_on('zlib', when='+c')
     depends_on('libevent', when='+c')
 
+    def setup_environment(self, spack_env, run_env):
+        if '+pic' in self.spec:
+            spack_env.append_flags('CFLAGS', self.compiler.pic_flag)
+            spack_env.append_flags('CXXFLAGS', self.compiler.pic_flag)
+
     def install(self, spec, prefix):
         env['PY_PREFIX'] = prefix
         env['JAVA_HOME'] = spec['java'].prefix
@@ -73,6 +81,7 @@ def install(self, spec, prefix):
         options.append('--with-boost=%s' % spec['boost'].prefix)
         options.append('--enable-tests=no')
 
+        options.append('--with-nodejs=no')
         options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no'))
         options.append('--with-python=%s' %
                        ('yes' if '+python' in spec else 'no'))

var/spack/repos/builtin/packages/zstd/package.py

@@ -36,5 +36,11 @@ class Zstd(MakefilePackage):
     version('1.3.0', '888660a850e33c2dcc7c4f9d0b04d347')
     version('1.1.2', '4c57a080d194bdaac83f2d3251fc7ffc')
 
+    variant('pic', default=True, description='Build position independent code')
+
+    def setup_environment(self, spack_env, run_env):
+        if '+pic' in self.spec:
+            spack_env.append_flags('CFLAGS', self.compiler.pic_flag)
+
     def install(self, spec, prefix):
         make('install', 'PREFIX={0}'.format(prefix))