zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

openmpi: options for local transport (#16831)

Browse Source
This commit is contained in:
Sergey Kosukhin
2020-06-17 21:41:24 +02:00
committed by GitHub
parent 506f31b7a7
commit 01b277ad0f
5 changed files with 310 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
var/spack/repos/builtin/packages/fca/package.py Normal file
View File

@@ -0,0 +1,31 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class Fca(Package):
"""Legacy interface for Mellanox Fabric Collective Accelerator (FCA). FCA
is a MPI-integrated software package that utilizes CORE-Direct technology
for implementing the MPI collective communications."""
homepage = 'https://www.mellanox.com/products/fca'
has_code = False
version('2.5.2431')
# FCA needs to be added as an external package to SPACK. For this, the
# config file packages.yaml needs to be adjusted:
#
# fca:
# version: [2.5.2431]
# paths:
# fca@2.5.2431: /opt/mellanox/fca (path to your FCA installation)
# buildable: False
def install(self, spec, prefix):
raise InstallError(
self.spec.format('{name} is not installable, you need to specify '
'it as an external package in packages.yaml'))

31
var/spack/repos/builtin/packages/hcoll/package.py Normal file
View File

@@ -0,0 +1,31 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class Hcoll(Package):
"""Modern interface for Mellanox Fabric Collective Accelerator (FCA). FCA
is a MPI-integrated software package that utilizes CORE-Direct technology
for implementing the MPI collective communications."""
homepage = 'https://www.mellanox.com/products/fca'
has_code = False
version('3.9.1927')
# HCOLL needs to be added as an external package to SPACK. For this, the
# config file packages.yaml needs to be adjusted:
#
# hcoll:
# version: [3.9.1927]
# paths:
# hcoll@3.9.1927: /opt/mellanox/hcoll (path to your HCOLL installation)
# buildable: False
def install(self, spec, prefix):
raise InstallError(
self.spec.format('{name} is not installable, you need to specify '
'it as an external package in packages.yaml'))

46
var/spack/repos/builtin/packages/knem/package.py Normal file
View File

@@ -0,0 +1,46 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class Knem(AutotoolsPackage):
"""KNEM is a Linux kernel module enabling high-performance intra-node MPI
communication for large messages."""
homepage = "http://knem.gforge.inria.fr"
url = "http://gforge.inria.fr/frs/download.php/37186/knem-1.1.3.tar.gz"
list_url = "http://knem.gforge.inria.fr/download"
maintainers = ['skosukhin']
version('1.1.3', sha256='50d3c4a20c140108b8ce47aaafd0ade0927d6f507e1b5cc690dd6bddeef30f60')
variant('hwloc', default=True,
description='Enable hwloc in the user-space tools')
depends_on('hwloc', when='+hwloc')
depends_on('pkgconfig', type='build', when='+hwloc')
# The support for hwloc was added in 0.9.1:
conflicts('+hwloc', when='@:0.9.0')
# Ideally, we should list all non-Linux-based platforms here:
conflicts('platform=darwin')
# All compilers except for gcc are in conflict:
for __compiler in spack.compilers.supported_compilers():
if __compiler != 'gcc':
conflicts('%{0}'.format(__compiler),
msg='Linux kernel module must be compiled with gcc')
@run_before('build')
def override_kernel_compiler(self):
# Override the compiler for kernel module source files. We need
# this additional argument for all installation phases.
make.add_default_arg('CC={0}'.format(spack_cc))
def configure_args(self):
return self.enable_or_disable('hwloc')

142
var/spack/repos/builtin/packages/openmpi/package.py
View File

@@ -182,19 +182,25 @@ class Openmpi(AutotoolsPackage):
variant(
'fabrics',
values=disjoint_sets(
('auto',), ('psm', 'psm2', 'verbs', 'mxm', 'ucx', 'libfabric')
('auto',),
('psm', 'psm2', 'verbs',
'mxm', 'ucx', 'ofi',
'fca', 'hcoll',
'xpmem', 'cma', 'knem') # shared memory transports
).with_non_feature_values('auto', 'none'),
description="List of fabrics that are enabled; "
"'auto' lets openmpi determine",
"'auto' lets openmpi determine",
)
variant(
'schedulers',
values=disjoint_sets(
('auto',), ('alps', 'lsf', 'tm', 'slurm', 'sge', 'loadleveler')
('auto',),
('alps', 'lsf', 'tm',
'slurm', 'sge', 'loadleveler')
).with_non_feature_values('auto', 'none'),
description="List of schedulers for which support is enabled; "
"'auto' lets openmpi determine",
"'auto' lets openmpi determine",
)
# Additional support options
@@ -259,39 +265,59 @@ class Openmpi(AutotoolsPackage):
depends_on('zlib', when='@3.0.0:')
depends_on('valgrind~mpi', when='+memchecker')
depends_on('opa-psm2', when='fabrics=psm2')
depends_on('rdma-core', when='fabrics=verbs')
depends_on('mxm', when='fabrics=mxm')
depends_on('binutils+libiberty', when='fabrics=mxm')
depends_on('ucx', when='fabrics=ucx')
depends_on('ucx +thread_multiple', when='fabrics=ucx +thread_multiple')
depends_on('ucx +thread_multiple', when='@3.0.0: fabrics=ucx')
depends_on('libfabric', when='fabrics=libfabric')
depends_on('opa-psm2', when='fabrics=psm2')
depends_on('mxm', when='fabrics=mxm')
depends_on('binutils+libiberty', when='fabrics=mxm')
depends_on('rdma-core', when='fabrics=verbs')
depends_on('libfabric', when='fabrics=ofi')
depends_on('fca', when='fabrics=fca')
depends_on('hcoll', when='fabrics=hcoll')
depends_on('xpmem', when='fabrics=xpmem')
depends_on('knem', when='fabrics=knem')
depends_on('slurm', when='schedulers=slurm')
depends_on('lsf', when='schedulers=lsf')
depends_on('openpbs', when='schedulers=tm')
depends_on('slurm', when='schedulers=slurm')
# CUDA support was added in 1.7
conflicts('+cuda', when='@:1.6')
# PMI support was added in 1.5.5
conflicts('+pmi', when='@:1.5.4')
# RPATH support in the wrappers was added in 1.7.4
conflicts('+wrapper-rpath', when='@:1.7.3')
conflicts('+cuda', when='@:1.6') # CUDA support was added in 1.7
conflicts('fabrics=psm2', when='@:1.8') # PSM2 support was added in 1.10.0
conflicts('fabrics=mxm', when='@:1.5.3') # MXM support was added in 1.5.4
conflicts('+pmi', when='@:1.5.4') # PMI support was added in 1.5.5
conflicts('schedulers=slurm ~pmi', when='@1.5.4:',
msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.')
conflicts('schedulers=loadleveler', when='@3.0.0:',
msg='The loadleveler scheduler is not supported with '
'openmpi(>=3.0.0).')
conflicts('+cxx', when='@5:',
msg='C++ MPI bindings are removed in 5.0.X release')
conflicts('+cxx_exceptions', when='@5:',
msg='C++ exceptions are removed in 5.0.X release')
filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share')
conflicts('fabrics=libfabric', when='@:1.8') # libfabric support was added in 1.10.0
# It may be worth considering making libfabric an exclusive fabrics choice
# PSM2 support was added in 1.10.0
conflicts('fabrics=psm2', when='@:1.8')
# MXM support was added in 1.5.4
conflicts('fabrics=mxm', when='@:1.5.3')
# libfabric (OFI) support was added in 1.10.0
conflicts('fabrics=ofi', when='@:1.8')
# fca support was added in 1.5.0 and removed in 5.0.0
conflicts('fabrics=fca', when='@:1.4,5:')
# hcoll support was added in 1.7.3:
conflicts('fabrics=hcoll', when='@:1.7.2')
# xpmem support was added in 1.7
conflicts('fabrics=xpmem', when='@:1.6')
# cma support was added in 1.7
conflicts('fabrics=cma', when='@:1.6')
# knem support was added in 1.5
conflicts('fabrics=knem', when='@:1.4')
# RPATH support in the wrappers was added in 1.7.4
conflicts('+wrapper-rpath', when='@:1.7.3')
conflicts('schedulers=slurm ~pmi', when='@1.5.4:',
msg='+pmi is required for openmpi(>=1.5.5) to work with SLURM.')
conflicts('schedulers=loadleveler', when='@3.0.0:',
msg='The loadleveler scheduler is not supported with '
'openmpi(>=3.0.0).')
filter_compiler_wrappers('openmpi/*-wrapper-data*', relative_root='share')
def url_for_version(self, version):
url = "http://www.open-mpi.org/software/ompi/v{0}/downloads/openmpi-{1}.tar.bz2"
@@ -343,6 +369,17 @@ def setup_dependent_package(self, module, dependent_spec):
join_path(self.prefix.lib, 'libmpi.{0}'.format(dso_suffix))
]
# Most of the following with_or_without methods might seem redundant
# because Spack compiler wrapper adds the required -I and -L flags, which
# is enough for the configure script to find them. However, we also need
# the flags in Libtool (lib/*.la) and pkg-config (lib/pkgconfig/*.pc).
# Therefore, we pass the prefixes explicitly.
def with_or_without_psm2(self, activated):
if not activated:
return '--without-psm2'
return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
def with_or_without_verbs(self, activated):
# Up through version 1.6, this option was named --with-openib.
# In version 1.7, it was renamed to be --with-verbs.
@@ -351,16 +388,54 @@ def with_or_without_verbs(self, activated):
return '--without-{0}'.format(opt)
return '--with-{0}={1}'.format(opt, self.spec['rdma-core'].prefix)
def with_or_without_mxm(self, activated):
if not activated:
return '--without-mxm'
return '--with-mxm={0}'.format(self.spec['mxm'].prefix)
def with_or_without_ucx(self, activated):
if not activated:
return '--without-ucx'
return '--with-ucx={0}'.format(self.spec['ucx'].prefix)
def with_or_without_ofi(self, activated):
# Up through version 3.0.3 this option was name --with-libfabric.
# In version 3.0.4, the old name was deprecated in favor of --with-ofi.
opt = 'ofi' if self.spec.satisfies('@3.0.4:') else 'libfabric'
if not activated:
return '--without-{0}'.format(opt)
return '--with-{0}={1}'.format(opt, self.spec['libfabric'].prefix)
def with_or_without_fca(self, activated):
if not activated:
return '--without-fca'
return '--with-fca={0}'.format(self.spec['fca'].prefix)
def with_or_without_hcoll(self, activated):
if not activated:
return '--without-hcoll'
return '--with-hcoll={0}'.format(self.spec['hcoll'].prefix)
def with_or_without_xpmem(self, activated):
if not activated:
return '--without-xpmem'
return '--with-xpmem={0}'.format(self.spec['xpmem'].prefix)
def with_or_without_knem(self, activated):
if not activated:
return '--without-knem'
return '--with-knem={0}'.format(self.spec['knem'].prefix)
def with_or_without_lsf(self, activated):
if not activated:
return '--without-lsf'
return '--with-lsf={0}'.format(self.spec['lsf'].prefix)
def with_or_without_tm(self, activated):
if not activated:
return '--without-tm'
return '--with-tm={0}'.format(self.spec['openpbs'].prefix)
def with_or_without_psm2(self, activated):
if not activated:
return '--without-psm2'
return '--with-psm2={0}'.format(self.spec['opa-psm2'].prefix)
@run_before('autoreconf')
def die_without_fortran(self):
# Until we can pass variants such as +fortran through virtual
@@ -433,8 +508,13 @@ def configure_args(self):
# Fabrics
if 'fabrics=auto' not in spec:
config_args.extend(self.with_or_without('fabrics',
activation_value='prefix'))
config_args.extend(self.with_or_without('fabrics'))
if spec.satisfies('@2.0.0'):
if 'fabrics=xpmem' in spec and 'platform=cray' in spec:
config_args.append('--with-cray-xpmem')
else:
config_args.append('--without-cray-xpmem')
# Schedulers
if 'schedulers=auto' not in spec:

91
var/spack/repos/builtin/packages/xpmem/package.py Normal file
View File

@@ -0,0 +1,91 @@
# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack import *
class Xpmem(AutotoolsPackage):
"""XPMEM is a Linux kernel module that enables a process to map the memory
of another process into its virtual address space."""
# The README file of the repository says that the development was
# transferred to a new repository on GitLab: http://gitlab.com/hjelmn/xpmem
# However, it looks like that the repository on GitHub has a more recent
# version of the codebase.
homepage = "https://github.com/hjelmn/xpmem"
url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz"
git = "https://github.com/hjelmn/xpmem.git"
maintainers = ['skosukhin']
version('master', branch='master')
# Versions starting 2.6.4 are neither tagged nor released in the repo
# (the choice of commits is based on the commit history of
# 'kernel/xpmem_private.h'):
version('2.6.5', commit='4efeed9cbaabe971f3766d67cb108e2c3316d4b8')
version('2.6.4', commit='522054850e4d1479d69f50f7190d1548bf9749fd')
# Released versions:
version('2.6.3', sha256='ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54')
version('2.6.2', sha256='2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b')
variant('kernel-module', default=True,
description='Enable building the kernel module')
depends_on('autoconf', type='build')
depends_on('automake', type='build')
depends_on('libtool', type='build')
depends_on('m4', type='build')
# It will become possible to disable the kernel module only starting 2.6.6:
# https://github.com/hjelmn/xpmem/pull/24
conflicts('~kernel-module', when='@:2.6.5')
# Ideally, we should list all non-Linux-based platforms here:
conflicts('+kernel-module', when='platform=darwin')
# All compilers except for gcc are in conflict with +kernel-module:
for __compiler in spack.compilers.supported_compilers():
if __compiler != 'gcc':
conflicts('+kernel-module',
when='%{0}'.format(__compiler),
msg='Linux kernel module must be compiled with gcc')
def autoreconf(self, spec, prefix):
Executable('./autogen.sh')()
@run_before('build')
def override_kernel_compiler(self):
# Override the compiler for kernel module source files. We need
# this additional argument for all installation phases.
if '+kernel-module' in self.spec:
make.add_default_arg('CC={0}'.format(spack_cc))
def configure_args(self):
args = []
if '~kernel-module' in self.spec:
# The kernel module is enabled by default. An attempt of explicit
# enabling with '--enable-kernel-module' disables the module.
args.extend('--disable-kernel-module')
if self.spec.satisfies('@:2.6.5'):
fmt = self.spec.format
# The following arguments will not be needed starting 2.6.6:
# https://github.com/hjelmn/xpmem/pull/18
args.extend([
fmt('--with-default-prefix={prefix}'),
fmt('--with-module={prefix.share}/Modules/{name}/{version}')])
return args
@when('@:2.6.5')
def install(self, spec, prefix):
with working_dir(self.build_directory):
# Override the hardcoded prefix for 'cray-xpmem.conf'
make('ldsoconfdir={0}'.format(
self.spec.prefix.etc.join('ld.so.conf.d')),
*self.install_targets)