fdb: new package with deps (#23175)

var/spack/repos/builtin/packages/ecbuild/package.py

@@ -0,0 +1,26 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Ecbuild(CMakePackage):
+    """ecBuild is the ECMWF build system. It is built on top of CMake and
+    consists of a set of macros as well as a wrapper around CMake,"""
+
+    homepage = 'https://github.com/ecmwf/ecbuild'
+    url = 'https://github.com/ecmwf/ecbuild/archive/refs/tags/3.6.1.tar.gz'
+
+    maintainers = ['skosukhin']
+
+    version('3.6.1', sha256='796ccceeb7af01938c2f74eab0724b228e9bf1978e32484aa3e227510f69ac59')
+
+    # Some of the tests (ECBUILD-415 and test_ecbuild_regex_escape) fail with
+    # cmake@2.20.0 and it is not yet clear why. For now, we simply limit the
+    # version of cmake to the latest '3.19.x':
+    depends_on('cmake@3.11:3.19', type=('build', 'run'))
+
+    # Some of the installed scripts require running Perl:
+    depends_on('perl', type=('build', 'run'))

var/spack/repos/builtin/packages/eckit/package.py

@@ -0,0 +1,137 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Eckit(CMakePackage):
+    """ecKit is a cross-platform c++ toolkit that supports development of tools
+    and applications at ECMWF."""
+
+    homepage = 'https://github.com/ecmwf/eckit'
+    url = 'https://github.com/ecmwf/eckit/archive/refs/tags/1.16.0.tar.gz'
+
+    maintainers = ['skosukhin']
+
+    version('1.16.0', sha256='9e09161ea6955df693d3c9ac70131985eaf7cf24a9fa4d6263661c6814ebbaf1')
+
+    variant('tools', default=True, description='Build the command line tools')
+    variant('mpi', default=True, description='Enable MPI support')
+    variant('admin', default=True,
+            description='Build utilities for administration tools')
+    variant('sql', default=True, description='Build SQL engine')
+    variant('linalg',
+            values=any_combination_of('eigen', 'armadillo', 'mkl', 'lapack'),
+            description='List of supported linear algebra backends')
+    variant('compression',
+            values=any_combination_of('bzip2', 'snappy', 'lz4', 'aec'),
+            description='List of supported compression backends')
+    variant('xxhash', default=True,
+            description='Enable xxHash support for hashing')
+    variant('ssl', default=False,
+            description='Enable MD4 and SHA1 support with OpenSSL')
+    variant('curl', default=False,
+            description='Enable URL data transferring with cURL')
+    variant('jemalloc', default=False,
+            description='Link against jemalloc memory allocator')
+    variant('unicode', default=True,
+            description='Enable support for Unicode characters in Yaml/JSON'
+                        'parsers')
+    variant('aio', default=True, description='Enable asynchronous IO')
+
+    depends_on('cmake@3.12:', type='build')
+    depends_on('ecbuild@3.5:', type='build')
+
+    depends_on('mpi', when='+mpi')
+
+    depends_on('yacc', type='build', when='+admin')
+    depends_on('flex', type='build', when='+admin')
+    depends_on('ncurses', when='+admin')
+
+    depends_on('yacc', type='build', when='+sql')
+    depends_on('flex', type='build', when='+sql')
+
+    depends_on('eigen', when='linalg=eigen')
+    depends_on('armadillo', when='linalg=armadillo')
+    depends_on('mkl', when='linalg=mkl')
+    depends_on('lapack', when='linalg=lapack')
+
+    depends_on('bzip2', when='compression=bzip2')
+    depends_on('snappy', when='compression=snappy')
+    depends_on('lz4', when='compression=lz4')
+    depends_on('libaec', when='compression=aec')
+
+    depends_on('openssl', when='+ssl')
+
+    depends_on('curl', when='+curl')
+
+    depends_on('jemalloc', when='+jemalloc')
+
+    # The package enables LAPACK backend (together with MKL backend)
+    # when='linalg=mkl'. This leads to two identical installations when:
+    #   eckit linalg=mkl
+    #   eckit linalg=mkl,lapack
+    # We prevent that by introducing the following conflict:
+    conflicts('linalg=lapack', when='linalg=mkl',
+              msg='"linalg=lapack" is implied when "linalg=mkl" and '
+                  'must not be specified additionally')
+
+    def cmake_args(self):
+        args = [
+            # Some features that we want to build are experimental:
+            self.define('ENABLE_EXPERIMENTAL', True),
+            self.define_from_variant('ENABLE_BUILD_TOOLS', 'tools'),
+            # We let ecBuild find the MPI library. We could help it by setting
+            # CMAKE_C_COMPILER to mpicc but that might give CMake a wrong
+            # impression that no additional flags are needed to link to
+            # libpthread, which will lead to problems with libraries that are
+            # linked with the C++ compiler. We could additionally set
+            # CMAKE_CXX_COMPILER to mpicxx. That would solve the problem with
+            # libpthread but lead to overlinking to MPI libraries, which we
+            # currently prefer to avoid since ecBuild does the job in all known
+            # cases.
+            self.define_from_variant('ENABLE_MPI', 'mpi'),
+            self.define_from_variant('ENABLE_ECKIT_CMD', 'admin'),
+            self.define_from_variant('ENABLE_ECKIT_SQL', 'sql'),
+            self.define('ENABLE_EIGEN', 'linalg=eigen' in self.spec),
+            self.define('ENABLE_ARMADILLO', 'linalg=armadillo' in self.spec),
+            self.define('ENABLE_MKL', 'linalg=mkl' in self.spec),
+            self.define('ENABLE_BZIP2', 'compression=bzip2' in self.spec),
+            self.define('ENABLE_SNAPPY', 'compression=snappy' in self.spec),
+            self.define('ENABLE_LZ4', 'compression=lz4' in self.spec),
+            self.define('ENABLE_AEC', 'compression=aec' in self.spec),
+            self.define_from_variant('ENABLE_XXHASH', 'xxhash'),
+            self.define_from_variant('ENABLE_SSL', 'ssl'),
+            self.define_from_variant('ENABLE_CURL', 'curl'),
+            self.define_from_variant('ENABLE_JEMALLOC', 'jemalloc'),
+            self.define_from_variant('ENABLE_UNICODE', 'unicode'),
+            self.define_from_variant('ENABLE_AIO', 'aio'),
+            self.define('ENABLE_TESTS', self.run_tests),
+            # Unconditionally disable additional unit/performance tests, since
+            # they download additional data (~1.6GB):
+            self.define('ENABLE_EXTRA_TESTS', False),
+            # No reason to check for doxygen and generate the documentation
+            # since it is not installed:
+            self.define('ENABLE_DOCS', False),
+            # Disable features that are currently not needed:
+            self.define('ENABLE_CUDA', False),
+            self.define('ENABLE_VIENNACL', False),
+            # Ceph/Rados storage support requires https://github.com/ceph/ceph
+            # and will be added later:
+            self.define('ENABLE_RADOS', False),
+            # rsync support requires https://github.com/librsync/librsync and
+            # will be added later:
+            self.define('ENABLE_RSYNC', False),
+            # Disable "prototyping code that may never see the light of day":
+            self.define('ENABLE_SANDBOX', False)
+        ]
+
+        if 'linalg=mkl' not in self.spec:
+            # ENABLE_LAPACK is ignored if MKL backend is enabled
+            # (the LAPACK backend is still built though):
+            args.append(
+                self.define('ENABLE_LAPACK', 'linalg=lapack' in self.spec))
+
+        return args

var/spack/repos/builtin/packages/fdb/metkit_1.7.0.patch

@@ -0,0 +1,11 @@
+--- a/src/fdb5/tools/fdb-hammer.cc
++++ b/src/fdb5/tools/fdb-hammer.cc
+@@ -22,7 +22,7 @@
+ #include "eckit/option/SimpleOption.h"
+ #include "eckit/option/VectorOption.h"
+ 
+-#include "metkit/grib/GribHandle.h"
++#include "metkit/codes/GribHandle.h"
+ 
+ #include "fdb5/grib/GribArchiver.h"
+ #include "fdb5/io/HandleGatherer.h"

var/spack/repos/builtin/packages/fdb/package.py

@@ -0,0 +1,73 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Fdb(CMakePackage):
+    """FDB (Fields DataBase) is a domain-specific object store developed at
+    ECMWF for storing, indexing and retrieving GRIB data."""
+
+    homepage = 'https://github.com/ecmwf/fdb'
+    url = 'https://github.com/ecmwf/fdb/archive/refs/tags/5.7.8.tar.gz'
+
+    maintainers = ['skosukhin']
+
+    version('5.7.8', sha256='6adac23c0d1de54aafb3c663d077b85d0f804724596623b381ff15ea4a835f60')
+
+    variant('tools', default=True, description='Build the command line tools')
+    variant(
+        'backends',
+        values=any_combination_of(
+            # FDB backend in indexed filesystem with table-of-contents with
+            # additional support for Lustre filesystem stripping control:
+            'lustre',
+            # Backends that will be added later:
+            # FDB backend in persistent memory (NVRAM):
+            # 'pmem',  # (requires https://github.com/ecmwf/pmem)
+            # FDB backend in CEPH object store (using Rados):
+            # 'rados'  # (requires eckit with RADOS support)
+        ), description='List of supported backends')
+
+    depends_on('cmake@3.12:', type='build')
+    depends_on('ecbuild@3.4:', type='build')
+
+    depends_on('eckit@1.16:')
+    depends_on('eckit+admin', when='+tools')
+
+    depends_on('eccodes@2.10:')
+    depends_on('metkit@1.5:+grib')
+
+    depends_on('lustre', when='backends=lustre')
+
+    # Starting version 1.7.0, metkit installs GribHandle.h to another directory.
+    # That is accounted for only starting version 5.8.0:
+    patch('metkit_1.7.0.patch', when='@:5.7.10+tools^metkit@1.7.0:')
+
+    # Download test data before running a test:
+    patch('https://github.com/ecmwf/fdb/commit/86e06b60f9a2d76a389a5f49bedd566d4c2ad2b2.patch',
+          sha256='e2254577e6d84a61d394eddcf42f894582f5daaf58d8962c609e41be0e3471b3',
+          when='@5.7.1:5.7.10+tools')
+
+    def cmake_args(self):
+        enable_build_tools = '+tools' in self.spec
+
+        args = [
+            self.define('ENABLE_FDB_BUILD_TOOLS', enable_build_tools),
+            self.define('ENABLE_BUILD_TOOLS', enable_build_tools),
+            # We cannot disable the FDB backend in indexed filesystem with
+            # table-of-contents because some default test programs and tools
+            # cannot be built without it:
+            self.define('ENABLE_TOCFDB', True),
+            self.define('ENABLE_LUSTRE', 'backends=lustre' in self.spec),
+            self.define('ENABLE_PMEMFDB', False),
+            self.define('ENABLE_RADOSFDB', False),
+            # The tests download additional data (~10MB):
+            self.define('ENABLE_TESTS', self.run_tests),
+            # We do not need any experimental features:
+            self.define('ENABLE_EXPERIMENTAL', False),
+            self.define('ENABLE_SANDBOX', False)
+        ]
+        return args

var/spack/repos/builtin/packages/metkit/package.py

@@ -0,0 +1,51 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Metkit(CMakePackage):
+    """Toolkit for manipulating and describing meteorological objects,
+    implementing the MARS language and associated processing and semantics."""
+
+    homepage = 'https://github.com/ecmwf/metkit'
+    url = 'https://github.com/ecmwf/metkit/archive/refs/tags/1.7.0.tar.gz'
+
+    maintainers = ['skosukhin']
+
+    version('1.7.0', sha256='8c34f6d8ea5381bd1bcfb22462349d03e1592e67d8137e76b3cecf134a9d338c')
+
+    variant('tools', default=True, description='Build the command line tools')
+    variant('grib', default=True, description='Enable support for GRIB format')
+    variant('odb', default=False, description='Enable support for ODB data')
+
+    depends_on('cmake@3.12:', type='build')
+    depends_on('ecbuild@3.4:', type='build')
+
+    depends_on('eckit@1.16:')
+
+    depends_on('eccodes@2.5:', when='+grib')
+
+    depends_on('odc', when='+odb')
+
+    conflicts('+tools', when='~grib~odb',
+              msg='None of the command line tools is built when both '
+                  'GRIB format and ODB data support are disabled')
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant('ENABLE_BUILD_TOOLS', 'tools'),
+            self.define_from_variant('ENABLE_GRIB', 'grib'),
+            self.define_from_variant('ENABLE_ODC', 'odb'),
+            # The tests download additional data (~4KB):
+            self.define('ENABLE_TESTS', self.run_tests),
+            # The library does not really implement support for BUFR format:
+            self.define('ENABLE_BUFR', False),
+            # The library does not really implement support for NetCDF format:
+            self.define('ENABLE_NETCDF', False),
+            # We do not need any experimental features:
+            self.define('ENABLE_EXPERIMENTAL', False)
+        ]
+        return args

var/spack/repos/builtin/packages/odc/package.py

@@ -0,0 +1,33 @@
+# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack import *
+
+
+class Odc(CMakePackage):
+    """ECMWF encoding and decoding of observational data in ODB2 format."""
+
+    homepage = 'https://github.com/ecmwf/odc'
+    url = 'https://github.com/ecmwf/odc/archive/refs/tags/1.3.0.tar.gz'
+
+    maintainers = ['skosukhin']
+
+    version('1.3.0', sha256='97a4f10765b341cc8ccbbf203f5559cb1b838cbd945f48d4cecb1bc4305e6cd6')
+
+    variant('fortran', default=False,
+            description='Enable the Fortran interface')
+
+    depends_on('ecbuild@3.4:', type='build')
+    depends_on('cmake@3.12:', type='build')
+
+    depends_on('eckit@1.4:+sql')
+
+    def cmake_args(self):
+        args = [
+            self.define_from_variant('ENABLE_FORTRAN', 'fortran'),
+            # The tests download additional data (~650MB):
+            self.define('ENABLE_TESTS', self.run_tests)
+        ]
+        return args