openmpi: options for local transport (#16831)
This commit is contained in:
parent
506f31b7a7
commit
01b277ad0f
31
var/spack/repos/builtin/packages/fca/package.py
Normal file
31
var/spack/repos/builtin/packages/fca/package.py
Normal file
@ -0,0 +1,31 @@
|
||||
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class Fca(Package):
|
||||
"""Legacy interface for Mellanox Fabric Collective Accelerator (FCA). FCA
|
||||
is a MPI-integrated software package that utilizes CORE-Direct technology
|
||||
for implementing the MPI collective communications."""
|
||||
|
||||
homepage = 'https://www.mellanox.com/products/fca'
|
||||
has_code = False
|
||||
|
||||
version('2.5.2431')
|
||||
|
||||
# FCA needs to be added as an external package to SPACK. For this, the
|
||||
# config file packages.yaml needs to be adjusted:
|
||||
#
|
||||
# fca:
|
||||
# version: [2.5.2431]
|
||||
# paths:
|
||||
# fca@2.5.2431: /opt/mellanox/fca (path to your FCA installation)
|
||||
# buildable: False
|
||||
|
||||
def install(self, spec, prefix):
|
||||
raise InstallError(
|
||||
self.spec.format('{name} is not installable, you need to specify '
|
||||
'it as an external package in packages.yaml'))
|
31
var/spack/repos/builtin/packages/hcoll/package.py
Normal file
31
var/spack/repos/builtin/packages/hcoll/package.py
Normal file
@ -0,0 +1,31 @@
|
||||
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class Hcoll(Package):
|
||||
"""Modern interface for Mellanox Fabric Collective Accelerator (FCA). FCA
|
||||
is a MPI-integrated software package that utilizes CORE-Direct technology
|
||||
for implementing the MPI collective communications."""
|
||||
|
||||
homepage = 'https://www.mellanox.com/products/fca'
|
||||
has_code = False
|
||||
|
||||
version('3.9.1927')
|
||||
|
||||
# HCOLL needs to be added as an external package to SPACK. For this, the
|
||||
# config file packages.yaml needs to be adjusted:
|
||||
#
|
||||
# hcoll:
|
||||
# version: [3.9.1927]
|
||||
# paths:
|
||||
# hcoll@3.9.1927: /opt/mellanox/hcoll (path to your HCOLL installation)
|
||||
# buildable: False
|
||||
|
||||
def install(self, spec, prefix):
|
||||
raise InstallError(
|
||||
self.spec.format('{name} is not installable, you need to specify '
|
||||
'it as an external package in packages.yaml'))
|
46
var/spack/repos/builtin/packages/knem/package.py
Normal file
46
var/spack/repos/builtin/packages/knem/package.py
Normal file
@ -0,0 +1,46 @@
|
||||
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class Knem(AutotoolsPackage):
|
||||
"""KNEM is a Linux kernel module enabling high-performance intra-node MPI
|
||||
communication for large messages."""
|
||||
|
||||
homepage = "http://knem.gforge.inria.fr"
|
||||
url = "http://gforge.inria.fr/frs/download.php/37186/knem-1.1.3.tar.gz"
|
||||
list_url = "http://knem.gforge.inria.fr/download"
|
||||
|
||||
maintainers = ['skosukhin']
|
||||
|
||||
version('1.1.3', sha256='50d3c4a20c140108b8ce47aaafd0ade0927d6f507e1b5cc690dd6bddeef30f60')
|
||||
|
||||
variant('hwloc', default=True,
|
||||
description='Enable hwloc in the user-space tools')
|
||||
|
||||
depends_on('hwloc', when='+hwloc')
|
||||
depends_on('pkgconfig', type='build', when='+hwloc')
|
||||
|
||||
# The support for hwloc was added in 0.9.1:
|
||||
conflicts('+hwloc', when='@:0.9.0')
|
||||
|
||||
# Ideally, we should list all non-Linux-based platforms here:
|
||||
conflicts('platform=darwin')
|
||||
|
||||
# All compilers except for gcc are in conflict:
|
||||
for __compiler in spack.compilers.supported_compilers():
|
||||
if __compiler != 'gcc':
|
||||
conflicts('%{0}'.format(__compiler),
|
||||
msg='Linux kernel module must be compiled with gcc')
|
||||
|
||||
@run_before('build')
|
||||
def override_kernel_compiler(self):
|
||||
# Override the compiler for kernel module source files. We need
|
||||
# this additional argument for all installation phases.
|
||||
make.add_default_arg('CC={0}'.format(spack_cc))
|
||||
|
||||
def configure_args(self):
|
||||
return self.enable_or_disable('hwloc')
|
@ -182,19 +182,25 @@ class Openmpi(AutotoolsPackage):
|
||||
variant(
|
||||
'fabrics',
|
||||
values=disjoint_sets(
|
||||
('auto',), ('psm', 'psm2', 'verbs', 'mxm', 'ucx', 'libfabric')
|
||||
('auto',),
|
||||
('psm', 'psm2', 'verbs',
|
||||
'mxm', 'ucx', 'ofi',
|
||||
'fca', 'hcoll',
|
||||
'xpmem', 'cma', 'knem') # shared memory transports
|
||||
).with_non_feature_values('auto', 'none'),
|
||||
description="List of fabrics that are enabled; "
|
||||
"'auto' lets openmpi determine",
|
||||
"'auto' lets openmpi determine",
|
||||
)
|
||||
|
||||
variant(
|
||||
'schedulers',
|
||||
values=disjoint_sets(
|
||||
('auto',), ('alps', 'lsf', 'tm', 'slurm', 'sge', 'loadleveler')
|
||||
('auto',),
|
||||
('alps', 'lsf', 'tm',
|
||||
'slurm', 'sge', 'loadleveler')
|
||||
).with_non_feature_values('auto', 'none'),
|
||||
description="List of schedulers for which support is enabled; "
|
||||
"'auto' lets openmpi determine",
|
||||
"'auto' lets openmpi determine",
|
||||
)
|
||||
|
||||
# Additional support options
|
||||
@ -259,39 +265,59 @@ class Openmpi(AutotoolsPackage):
|
||||
depends_on('zlib', when='@3.0.0:')
|
||||
depends_on('valgrind~mpi', when='+memchecker')
|
||||
|
||||
depends_on('opa-psm2', when='fabrics=psm2')
|
||||
depends_on('rdma-core', when='fabrics=verbs')
|
||||
depends_on('mxm', when='fabrics=mxm')
|
||||
depends_on('binutils+libiberty', when='fabrics=mxm')
|
||||
depends_on('ucx', when='fabrics=ucx')
|
||||
depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple')
|
||||
depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx')
|
||||
depends_on('libfabric', when='fabrics=libfabric')
|
||||
depends_on('opa-psm2', when='fabrics=psm2')
|
||||
depends_on('mxm', when='fabrics=mxm')
|
||||
depends_on('binutils+libiberty', when='fabrics=mxm')
|
||||
depends_on('rdma-core', when='fabrics=verbs')
|
||||
depends_on('libfabric', when='fabrics=ofi')
|
||||
depends_on('fca', when='fabrics=fca')
|
||||
depends_on('hcoll', when='fabrics=hcoll')
|
||||
depends_on('xpmem', when='fabrics=xpmem')
|
||||
depends_on('knem', when='fabrics=knem')
|
||||
|
||||
depends_on('slurm', when='schedulers=slurm')
|
||||
depends_on('lsf', when='schedulers=lsf')
|
||||
depends_on('openpbs', when='schedulers=tm')
|
||||
depends_on('slurm', when='schedulers=slurm')
|
||||
|
||||
# CUDA support was added in 1.7
|
||||
conflicts('+cuda', when='@:1.6')
|
||||
# PMI support was added in 1.5.5
|
||||
conflicts('+pmi', when='@:1.5.4')
|
||||
# RPATH support in the wrappers was added in 1.7.4
|
||||
conflicts('+wrapper-rpath', when='@:1.7.3')
|
||||
|
||||
conflicts('+cuda', when='@:1.6') # CUDA support was added in 1.7
|
||||
conflicts('fabrics=psm2', when='@:1.8') # PSM2 support was added in 1.10.0
|
||||
conflicts('fabrics=mxm', when='@:1.5.3') # MXM support was added in 1.5.4
|
||||
conflicts('+pmi', when='@:1.5.4') # PMI support was added in 1.5.5
|
||||
conflicts('schedulers=slurm ~pmi', when='@1.5.4:',
|
||||
msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.')
|
||||
conflicts('schedulers=loadleveler', when='@3.0.0:',
|
||||
msg='The loadleveler scheduler is not supported with '
|
||||
'openmpi(>=3.0.0).')
|
||||
conflicts('+cxx', when='@5:',
|
||||
msg='C++ MPI bindings are removed in 5.0.X release')
|
||||
conflicts('+cxx_exceptions', when='@5:',
|
||||
msg='C++ exceptions are removed in 5.0.X release')
|
||||
|
||||
filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share')
|
||||
conflicts('fabrics=libfabric', when='@:1.8') # libfabric support was added in 1.10.0
|
||||
# It may be worth considering making libfabric an exclusive fabrics choice
|
||||
# PSM2 support was added in 1.10.0
|
||||
conflicts('fabrics=psm2', when='@:1.8')
|
||||
# MXM support was added in 1.5.4
|
||||
conflicts('fabrics=mxm', when='@:1.5.3')
|
||||
# libfabric (OFI) support was added in 1.10.0
|
||||
conflicts('fabrics=ofi', when='@:1.8')
|
||||
# fca support was added in 1.5.0 and removed in 5.0.0
|
||||
conflicts('fabrics=fca', when='@:1.4,5:')
|
||||
# hcoll support was added in 1.7.3:
|
||||
conflicts('fabrics=hcoll', when='@:1.7.2')
|
||||
# xpmem support was added in 1.7
|
||||
conflicts('fabrics=xpmem', when='@:1.6')
|
||||
# cma support was added in 1.7
|
||||
conflicts('fabrics=cma', when='@:1.6')
|
||||
# knem support was added in 1.5
|
||||
conflicts('fabrics=knem', when='@:1.4')
|
||||
|
||||
# RPATH support in the wrappers was added in 1.7.4
|
||||
conflicts('+wrapper-rpath', when='@:1.7.3')
|
||||
conflicts('schedulers=slurm ~pmi', when='@1.5.4:',
|
||||
msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.')
|
||||
conflicts('schedulers=loadleveler', when='@3.0.0:',
|
||||
msg='The loadleveler scheduler is not supported with '
|
||||
'openmpi(>=3.0.0).')
|
||||
|
||||
filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share')
|
||||
|
||||
def url_for_version(self, version):
|
||||
url = "http://www.open-mpi.org/software/ompi/v{0}/downloads/openmpi-{1}.tar.bz2"
|
||||
@ -343,6 +369,17 @@ def setup_dependent_package(self, module, dependent_spec):
|
||||
join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
|
||||
]
|
||||
|
||||
# Most of the following with_or_without methods might seem redundant
|
||||
# because Spack compiler wrapper adds the required -I and -L flags, which
|
||||
# is enough for the configure script to find them. However, we also need
|
||||
# the flags in Libtool (lib/*.la) and pkg-config (lib/pkgconfig/*.pc).
|
||||
# Therefore, we pass the prefixes explicitly.
|
||||
|
||||
def with_or_without_psm2(self, activated):
|
||||
if not activated:
|
||||
return '--without-psm2'
|
||||
return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
|
||||
|
||||
def with_or_without_verbs(self, activated):
|
||||
# Up through version 1.6, this option was named --with-openib.
|
||||
# In version 1.7, it was renamed to be --with-verbs.
|
||||
@ -351,16 +388,54 @@ def with_or_without_verbs(self, activated):
|
||||
return '--without-{0}'.format(opt)
|
||||
return '--with-{0}={1}'.format(opt, self.spec['rdma-core'].prefix)
|
||||
|
||||
def with_or_without_mxm(self, activated):
|
||||
if not activated:
|
||||
return '--without-mxm'
|
||||
return '--with-mxm={0}'.format(self.spec['mxm'].prefix)
|
||||
|
||||
def with_or_without_ucx(self, activated):
|
||||
if not activated:
|
||||
return '--without-ucx'
|
||||
return '--with-ucx={0}'.format(self.spec['ucx'].prefix)
|
||||
|
||||
def with_or_without_ofi(self, activated):
|
||||
# Up through version 3.0.3 this option was name --with-libfabric.
|
||||
# In version 3.0.4, the old name was deprecated in favor of --with-ofi.
|
||||
opt = 'ofi' if self.spec.satisfies('@3.0.4:') else 'libfabric'
|
||||
if not activated:
|
||||
return '--without-{0}'.format(opt)
|
||||
return '--with-{0}={1}'.format(opt, self.spec['libfabric'].prefix)
|
||||
|
||||
def with_or_without_fca(self, activated):
|
||||
if not activated:
|
||||
return '--without-fca'
|
||||
return '--with-fca={0}'.format(self.spec['fca'].prefix)
|
||||
|
||||
def with_or_without_hcoll(self, activated):
|
||||
if not activated:
|
||||
return '--without-hcoll'
|
||||
return '--with-hcoll={0}'.format(self.spec['hcoll'].prefix)
|
||||
|
||||
def with_or_without_xpmem(self, activated):
|
||||
if not activated:
|
||||
return '--without-xpmem'
|
||||
return '--with-xpmem={0}'.format(self.spec['xpmem'].prefix)
|
||||
|
||||
def with_or_without_knem(self, activated):
|
||||
if not activated:
|
||||
return '--without-knem'
|
||||
return '--with-knem={0}'.format(self.spec['knem'].prefix)
|
||||
|
||||
def with_or_without_lsf(self, activated):
|
||||
if not activated:
|
||||
return '--without-lsf'
|
||||
return '--with-lsf={0}'.format(self.spec['lsf'].prefix)
|
||||
|
||||
def with_or_without_tm(self, activated):
|
||||
if not activated:
|
||||
return '--without-tm'
|
||||
return '--with-tm={0}'.format(self.spec['openpbs'].prefix)
|
||||
|
||||
def with_or_without_psm2(self, activated):
|
||||
if not activated:
|
||||
return '--without-psm2'
|
||||
return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
|
||||
|
||||
@run_before('autoreconf')
|
||||
def die_without_fortran(self):
|
||||
# Until we can pass variants such as +fortran through virtual
|
||||
@ -433,8 +508,13 @@ def configure_args(self):
|
||||
|
||||
# Fabrics
|
||||
if 'fabrics=auto' not in spec:
|
||||
config_args.extend(self.with_or_without('fabrics',
|
||||
activation_value='prefix'))
|
||||
config_args.extend(self.with_or_without('fabrics'))
|
||||
|
||||
if spec.satisfies('@2.0.0'):
|
||||
if 'fabrics=xpmem' in spec and 'platform=cray' in spec:
|
||||
config_args.append('--with-cray-xpmem')
|
||||
else:
|
||||
config_args.append('--without-cray-xpmem')
|
||||
|
||||
# Schedulers
|
||||
if 'schedulers=auto' not in spec:
|
||||
|
91
var/spack/repos/builtin/packages/xpmem/package.py
Normal file
91
var/spack/repos/builtin/packages/xpmem/package.py
Normal file
@ -0,0 +1,91 @@
|
||||
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
class Xpmem(AutotoolsPackage):
|
||||
"""XPMEM is a Linux kernel module that enables a process to map the memory
|
||||
of another process into its virtual address space."""
|
||||
|
||||
# The README file of the repository says that the development was
|
||||
# transferred to a new repository on GitLab: http://gitlab.com/hjelmn/xpmem
|
||||
# However, it looks like that the repository on GitHub has a more recent
|
||||
# version of the codebase.
|
||||
homepage = "https://github.com/hjelmn/xpmem"
|
||||
url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz"
|
||||
git = "https://github.com/hjelmn/xpmem.git"
|
||||
|
||||
maintainers = ['skosukhin']
|
||||
|
||||
version('master', branch='master')
|
||||
|
||||
# Versions starting 2.6.4 are neither tagged nor released in the repo
|
||||
# (the choice of commits is based on the commit history of
|
||||
# 'kernel/xpmem_private.h'):
|
||||
version('2.6.5', commit='4efeed9cbaabe971f3766d67cb108e2c3316d4b8')
|
||||
version('2.6.4', commit='522054850e4d1479d69f50f7190d1548bf9749fd')
|
||||
|
||||
# Released versions:
|
||||
version('2.6.3', sha256='ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54')
|
||||
version('2.6.2', sha256='2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b')
|
||||
|
||||
variant('kernel-module', default=True,
|
||||
description='Enable building the kernel module')
|
||||
|
||||
depends_on('autoconf', type='build')
|
||||
depends_on('automake', type='build')
|
||||
depends_on('libtool', type='build')
|
||||
depends_on('m4', type='build')
|
||||
|
||||
# It will become possible to disable the kernel module only starting 2.6.6:
|
||||
# https://github.com/hjelmn/xpmem/pull/24
|
||||
conflicts('~kernel-module', when='@:2.6.5')
|
||||
|
||||
# Ideally, we should list all non-Linux-based platforms here:
|
||||
conflicts('+kernel-module', when='platform=darwin')
|
||||
|
||||
# All compilers except for gcc are in conflict with +kernel-module:
|
||||
for __compiler in spack.compilers.supported_compilers():
|
||||
if __compiler != 'gcc':
|
||||
conflicts('+kernel-module',
|
||||
when='%{0}'.format(__compiler),
|
||||
msg='Linux kernel module must be compiled with gcc')
|
||||
|
||||
def autoreconf(self, spec, prefix):
|
||||
Executable('./autogen.sh')()
|
||||
|
||||
@run_before('build')
|
||||
def override_kernel_compiler(self):
|
||||
# Override the compiler for kernel module source files. We need
|
||||
# this additional argument for all installation phases.
|
||||
if '+kernel-module' in self.spec:
|
||||
make.add_default_arg('CC={0}'.format(spack_cc))
|
||||
|
||||
def configure_args(self):
|
||||
args = []
|
||||
|
||||
if '~kernel-module' in self.spec:
|
||||
# The kernel module is enabled by default. An attempt of explicit
|
||||
# enabling with '--enable-kernel-module' disables the module.
|
||||
args.extend('--disable-kernel-module')
|
||||
|
||||
if self.spec.satisfies('@:2.6.5'):
|
||||
fmt = self.spec.format
|
||||
# The following arguments will not be needed starting 2.6.6:
|
||||
# https://github.com/hjelmn/xpmem/pull/18
|
||||
args.extend([
|
||||
fmt('--with-default-prefix={prefix}'),
|
||||
fmt('--with-module={prefix.share}/Modules/{name}/{version}')])
|
||||
|
||||
return args
|
||||
|
||||
@when('@:2.6.5')
|
||||
def install(self, spec, prefix):
|
||||
with working_dir(self.build_directory):
|
||||
# Override the hardcoded prefix for 'cray-xpmem.conf'
|
||||
make('ldsoconfdir={0}'.format(
|
||||
self.spec.prefix.etc.join('ld.so.conf.d')),
|
||||
*self.install_targets)
|
Loading…
Reference in New Issue
Block a user