zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Parquet: new packages (#8601)

Browse Source 
* parquet: new package

Includes the following dependencies:

* arrow
* flatbuffers

Changes for compilation:

* snappy
* thrift
* zstd

* parquet: improve recipes (including dependencies)

* arrow: remove unused import in recipe

* parquet: fix line length for flake8/py27

* parquet, arrow: fix py26 format strings

* Address review comments.

* arrow: simplify recipe
This commit is contained in:
Matthias Wolf
2018-07-04 14:24:39 +02:00
committed by Adam J. Stewart
parent aab5488154
commit 81668c524b
6 changed files with 184 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

74
var/spack/repos/builtin/packages/arrow/package.py Normal file
View File

@@ -0,0 +1,74 @@
##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Arrow(CMakePackage):
"""A cross-language development platform for in-memory data.
This package contains the C++ bindings.
"""
homepage = "http://arrow.apache.org"
url = "https://github.com/apache/arrow/archive/apache-arrow-0.9.0.tar.gz"
version('0.9.0', 'ebbd36c362b9e1d398ca612f6d2531ec')
version('0.8.0', '56436f6f61ccc68686b7e0ea30bf4d09')
depends_on('boost@1.60:')
depends_on('cmake@3.2.0:', type='build')
depends_on('flatbuffers@1.8.0 build_type=Release') # only Release contains flatc
depends_on('rapidjson')
depends_on('snappy~shared')
depends_on('zlib+pic')
depends_on('zstd+pic')
variant('build_type', default='Release',
description='CMake build type',
values=('Debug', 'FastDebug', 'Release'))
root_cmakelists_dir = 'cpp'
def patch(self):
"""Prevent `-isystem /usr/include` from appearing, since this confuses gcc.
"""
filter_file(r'(include_directories\()SYSTEM ',
r'\1',
'cpp/cmake_modules/ThirdpartyToolchain.cmake')
def cmake_args(self):
args = [
"-DARROW_USE_SSE=ON",
"-DARROW_BUILD_SHARED=ON",
"-DARROW_BUILD_STATIC=OFF",
"-DARROW_BUILD_TESTS=OFF",
"-DARROW_WITH_BROTLI=OFF",
"-DARROW_WITH_LZ4=OFF",
]
for dep in ('flatbuffers', 'rapidjson', 'snappy', 'zlib', 'zstd'):
args.append("-D{0}_HOME={1}".format(dep.upper(),
self.spec[dep].prefix))
args.append("-DZLIB_LIBRARIES={0}".format(self.spec['zlib'].libs))
return args

36
var/spack/repos/builtin/packages/flatbuffers/package.py Normal file
View File

@@ -0,0 +1,36 @@
##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Flatbuffers(CMakePackage):
"""Memory Efficient Serialization Library
"""
homepage = "http://google.github.io/flatbuffers/"
url = "https://github.com/google/flatbuffers/archive/v1.9.0.tar.gz"
version('1.9.0', '8be7513bf960034f6873326d09521a4b')
version('1.8.0', '276cab8303c4189cbe3b8a70e0515d65')

52
var/spack/repos/builtin/packages/parquet/package.py Normal file
View File

@@ -0,0 +1,52 @@
##############################################################################
# Copyright (c) 2013-2018, Lawrence Livermore National Security, LLC.
# Produced at the Lawrence Livermore National Laboratory.
#
# This file is part of Spack.
# Created by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
# LLNL-CODE-647188
#
# For details, see https://github.com/spack/spack
# Please also see the NOTICE and LICENSE files for our notice and the LGPL.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License (as
# published by the Free Software Foundation) version 2.1, February 1999.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
# conditions of the GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
##############################################################################
from spack import *
class Parquet(CMakePackage):
"""C++ bindings for the Apache Parquet columnar data format.
"""
homepage = "https://github.com/apache/parquet-cpp"
url = "https://github.com/apache/parquet-cpp/archive/apache-parquet-cpp-1.4.0.tar.gz"
version('1.4.0', '3a3659e65052ef5a76fb88e4922283b9')
depends_on('arrow')
depends_on('boost')
depends_on('cmake@3.2.0:', type='build')
depends_on('pkgconfig', type='build')
depends_on('thrift+pic')
variant('build_type', default='Release',
description='CMake build type',
values=('Debug', 'FastDebug', 'Release'))
def cmake_args(self):
args = ['-DPARQUET_USE_SSE=OFF', '-DPARQUET_BUILD_TESTS=OFF']
for dep in ('arrow', 'thrift'):
args.append("-D{0}_HOME={1}".format(dep.upper(),
self.spec[dep].prefix))
return args

7
var/spack/repos/builtin/packages/snappy/package.py
View File

@@ -34,6 +34,7 @@ class Snappy(CMakePackage):
version('1.1.7', 'ee9086291c9ae8deb4dac5e0b85bf54a')
variant('shared', default=True, description='Build shared libraries')
variant('pic', default=True, description='Build position independent code')
def cmake_args(self):
spec = self.spec
@@ -47,6 +48,12 @@ def cmake_args(self):
return args
def flag_handler(self, name, flags):
flags = list(flags)
if '+pic' in self.spec and name in ('cflags', 'cxxflags'):
flags.append(self.compiler.pic_flag)
return (None, None, flags)
@run_after('install')
def install_pkgconfig(self):
mkdirp(self.prefix.lib.pkgconfig)

9
var/spack/repos/builtin/packages/thrift/package.py
View File

@@ -38,6 +38,7 @@ class Thrift(Package):
homepage = "http://thrift.apache.org"
url = "http://apache.mirrors.ionfish.org/thrift/0.9.2/thrift-0.9.2.tar.gz"
version('0.11.0', '0be59730ebce071eceaf6bfdb8d3a20e')
version('0.10.0', '795c5dd192e310ffff38cfd9430d6b29')
version('0.9.3', '88d667a8ae870d5adeca8cb7d6795442')
version('0.9.2', '89f63cc4d0100912f4a1f8a9dee63678')
@@ -45,6 +46,8 @@ class Thrift(Package):
# Currently only support for c-family and python
variant('c', default=True,
description="Build support for C-family languages")
variant('pic', default=True,
description='Build position independent code')
variant('python', default=True,
description="Build support for python")
@@ -63,6 +66,11 @@ class Thrift(Package):
depends_on('zlib', when='+c')
depends_on('libevent', when='+c')
def setup_environment(self, spack_env, run_env):
if '+pic' in self.spec:
spack_env.append_flags('CFLAGS', self.compiler.pic_flag)
spack_env.append_flags('CXXFLAGS', self.compiler.pic_flag)
def install(self, spec, prefix):
env['PY_PREFIX'] = prefix
env['JAVA_HOME'] = spec['java'].prefix
@@ -73,6 +81,7 @@ def install(self, spec, prefix):
options.append('--with-boost=%s' % spec['boost'].prefix)
options.append('--enable-tests=no')
options.append('--with-nodejs=no')
options.append('--with-c=%s' % ('yes' if '+c' in spec else 'no'))
options.append('--with-python=%s' %
('yes' if '+python' in spec else 'no'))

6
var/spack/repos/builtin/packages/zstd/package.py
View File

@@ -36,5 +36,11 @@ class Zstd(MakefilePackage):
version('1.3.0', '888660a850e33c2dcc7c4f9d0b04d347')
version('1.1.2', '4c57a080d194bdaac83f2d3251fc7ffc')
variant('pic', default=True, description='Build position independent code')
def setup_environment(self, spack_env, run_env):
if '+pic' in self.spec:
spack_env.append_flags('CFLAGS', self.compiler.pic_flag)
def install(self, spec, prefix):
make('install', 'PREFIX={0}'.format(prefix))