zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mpich: gather in a single place env modifications needed by mpich derivatives (#48276)

Browse Source 
* mpich: gather in a single place env modifications needed by mpich derivatives

MPICH, and its derivatives, share a lot of copy/paste code to setup the
environment during the different stages of the package life-cycle.

This commit gathers the common modifications in a single place (a mixin class),
living in the Mpich package, and makes derivatives import, and reuse, it.

* Fix docs for Python < 3.13
This commit is contained in:
Massimiliano Culpo
2024-12-30 23:42:16 +01:00
committed by GitHub
parent a7373b8a31
commit 71512f5682
8 changed files with 98 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
lib/spack/docs/packaging_guide.rst
View File

@@ -4900,15 +4900,15 @@ If your package has a virtual dependency like ``mpi``, then referring to
``spec["mpi"]`` within ``install()`` will get you the concrete ``mpi``
implementation in your dependency DAG. That is a spec object just like
the one passed to install, only the MPI implementations all set some
additional properties on it to help you out. E.g., in mvapich2, you'll
additional properties on it to help you out. E.g., in openmpi, you'll
find this:
.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/mvapich2/package.py
:pyobject: Mvapich2.setup_dependent_package
.. literalinclude:: _spack_root/var/spack/repos/builtin/packages/openmpi/package.py
:pyobject: Openmpi.setup_dependent_package
That code allows the mvapich2 package to associate an ``mpicc`` property
with the ``mvapich2`` node in the DAG, so that dependents can access it.
``openmpi`` and ``mpich`` do similar things. So, no matter what MPI
That code allows the ``openmpi`` package to associate an ``mpicc`` property
with the ``openmpi`` node in the DAG, so that dependents can access it.
``mvapich2`` and ``mpich`` do similar things. So, no matter what MPI
you're using, spec["mpi"].mpicc gets you the location of the MPI
compilers. This allows us to have a fairly simple polymorphic interface
for information about virtual dependencies like MPI.

30
var/spack/repos/builtin/packages/cray-mpich/package.py
View File

@@ -6,10 +6,11 @@
import os
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
from spack.util.module_cmd import get_path_args_from_module_line, module
class CrayMpich(Package):
class CrayMpich(MpichEnvironmentModifications, Package):
"""Cray's MPICH is a high performance and widely portable implementation of
the Message Passing Interface (MPI) standard."""
@@ -72,31 +73,18 @@ def external_prefix(self):
def setup_run_environment(self, env):
if self.spec.satisfies("+wrappers"):
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
elif spack_cc is not None:
env.set("MPICC", spack_cc)
env.set("MPICXX", spack_cxx)
env.set("MPIF77", spack_fc)
env.set("MPIF90", spack_fc)
self.setup_mpi_wrapper_variables(env)
return
def setup_dependent_build_environment(self, env, dependent_spec):
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
env.set("MPICC", self.compiler.cc)
env.set("MPICXX", self.compiler.cxx)
env.set("MPIFC", self.compiler.fc)
env.set("MPIF77", self.compiler.f77)
def setup_dependent_package(self, module, dependent_spec):
spec = self.spec
if spec.satisfies("+wrappers"):
spec.mpicc = join_path(self.prefix.bin, "mpicc")
spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
spec.mpifc = join_path(self.prefix.bin, "mpif90")
spec.mpif77 = join_path(self.prefix.bin, "mpif77")
MpichEnvironmentModifications.setup_dependent_package(self, module, dependent_spec)
elif spack_cc is not None:
spec.mpicc = spack_cc
spec.mpicxx = spack_cxx

30
var/spack/repos/builtin/packages/cray-mvapich2/package.py
View File

@@ -4,9 +4,10 @@
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
class CrayMvapich2(Package):
class CrayMvapich2(MpichEnvironmentModifications, Package):
"""Cray/HPE packaging of MVAPICH2 for HPE Apollo systems"""
homepage = "https://docs.nersc.gov/development/compilers/wrappers/"
@@ -29,29 +30,10 @@ class CrayMvapich2(Package):
requires("platform=linux", msg="Cray MVAPICH2 is only available on Cray")
def setup_run_environment(self, env):
if spack_cc is None:
return
env.set("MPICC", spack_cc)
env.set("MPICXX", spack_cxx)
env.set("MPIF77", spack_fc)
env.set("MPIF90", spack_fc)
def setup_dependent_build_environment(self, env, dependent_spec):
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_dependent_package(self, module, dependent_spec):
spec = self.spec
dependent_module = dependent_spec.package.module
spec.mpicc = dependent_module.spack_cc
spec.mpicxx = dependent_module.spack_cxx
spec.mpifc = dependent_module.spack_fc
spec.mpif77 = dependent_module.spack_f77
env.set("MPICC", self.compiler.cc)
env.set("MPICXX", self.compiler.cxx)
env.set("MPIFC", self.compiler.fc)
env.set("MPIF77", self.compiler.f77)
def install(self, spec, prefix):
raise InstallError(

83
var/spack/repos/builtin/packages/mpich/package.py
View File

@@ -8,10 +8,58 @@
import sys
import spack.compilers
import spack.package_base
from spack.package import *
class Mpich(AutotoolsPackage, CudaPackage, ROCmPackage):
class MpichEnvironmentModifications(spack.package_base.PackageBase):
"""Collects the environment modifications that are usually needed for the life-cycle of
MPICH, and derivatives.
"""
def setup_dependent_build_environment(self, env, dependent_spec):
dependent_module = dependent_spec.package.module
for var_name, attr_name in (
("MPICH_CC", "spack_cc"),
("MPICH_CXX", "spack_cxx"),
("MPICH_FC", "spack_fc"),
("MPICH_F90", "spack_fc"),
("MPICH_F77", "spack_f77"),
):
if not hasattr(dependent_module, attr_name):
continue
env.set(var_name, getattr(dependent_module, attr_name))
def setup_build_environment(self, env):
env.unset("F90")
env.unset("F90FLAGS")
def setup_run_environment(self, env):
self.setup_mpi_wrapper_variables(env)
def setup_dependent_package(self, module, dependent_spec):
spec = self.spec
spec.mpicc = join_path(self.prefix.bin, "mpicc")
spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
# Some derived packages define the "fortran" variant, most don't. Checking on the
# presence of ~fortran makes us default to add fortran wrappers if the variant is
# not declared.
if spec.satisfies("~fortran"):
return
spec.mpifc = join_path(self.prefix.bin, "mpif90")
spec.mpif77 = join_path(self.prefix.bin, "mpif77")
def setup_mpi_wrapper_variables(self, env):
# Because MPI implementations provide compilers, they have to add to
# their run environments the code to make the compilers available.
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
class Mpich(MpichEnvironmentModifications, AutotoolsPackage, CudaPackage, ROCmPackage):
"""MPICH is a high performance and widely portable implementation of
the Message Passing Interface (MPI) standard."""
@@ -448,39 +496,11 @@ def flag_handler(self, name, flags):
return flags, None, None
def setup_build_environment(self, env):
env.unset("F90")
env.unset("F90FLAGS")
MpichEnvironmentModifications.setup_build_environment(self, env)
if "pmi=cray" in self.spec:
env.set("CRAY_PMI_INCLUDE_OPTS", "-I" + self.spec["cray-pmi"].headers.directories[0])
env.set("CRAY_PMI_POST_LINK_OPTS", "-L" + self.spec["cray-pmi"].libs.directories[0])
def setup_run_environment(self, env):
# Because MPI implementations provide compilers, they have to add to
# their run environments the code to make the compilers available.
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpic++"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
def setup_dependent_build_environment(self, env, dependent_spec):
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_dependent_package(self, module, dependent_spec):
spec = self.spec
spec.mpicc = join_path(self.prefix.bin, "mpicc")
spec.mpicxx = join_path(self.prefix.bin, "mpic++")
if "+fortran" in spec:
spec.mpifc = join_path(self.prefix.bin, "mpif90")
spec.mpif77 = join_path(self.prefix.bin, "mpif77")
def autoreconf(self, spec, prefix):
"""Not needed usually, configure should be already there"""
# If configure exists nothing needs to be done
@@ -543,8 +563,7 @@ def configure_args(self):
)
)
if "~fortran" in spec:
config_args.append("--disable-fortran")
config_args.extend(self.enable_or_disable("fortran"))
if "+slurm" in spec:
config_args.append("--with-slurm=yes")

48
var/spack/repos/builtin/packages/mvapich/package.py
View File

@@ -8,9 +8,10 @@
import sys
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
class Mvapich(AutotoolsPackage):
class Mvapich(MpichEnvironmentModifications, AutotoolsPackage):
"""Mvapich is a High-Performance MPI Library for clusters with diverse
networks (InfiniBand, Omni-Path, Ethernet/iWARP, and RoCE) and computing
platforms (x86 (Intel and AMD), ARM and OpenPOWER)"""
@@ -28,9 +29,9 @@ class Mvapich(AutotoolsPackage):
# Prefer the latest stable release
version("3.0", sha256="ee076c4e672d18d6bf8dd2250e4a91fa96aac1db2c788e4572b5513d86936efb")
depends_on("c", type="build") # generated
depends_on("cxx", type="build") # generated
depends_on("fortran", type="build") # generated
depends_on("c", type="build")
depends_on("cxx", type="build")
depends_on("fortran", type="build")
provides("mpi")
provides("mpi@:3.1")
@@ -207,48 +208,15 @@ def flag_handler(self, name, flags):
return (flags, None, None)
def setup_build_environment(self, env):
# mvapich2 configure fails when F90 and F90FLAGS are set
env.unset("F90")
env.unset("F90FLAGS")
def setup_run_environment(self, env):
env.set("MPI_ROOT", self.prefix)
# Because MPI functions as a compiler, we need to treat it as one and
# add its compiler paths to the run environment.
self.setup_compiler_environment(env)
self.setup_mpi_wrapper_variables(env)
def setup_dependent_build_environment(self, env, dependent_spec):
self.setup_compiler_environment(env)
# use the Spack compiler wrappers under MPI
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_compiler_environment(self, env):
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
def setup_dependent_package(self, module, dependent_spec):
self.spec.mpicc = join_path(self.prefix.bin, "mpicc")
self.spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
self.spec.mpifc = join_path(self.prefix.bin, "mpif90")
self.spec.mpif77 = join_path(self.prefix.bin, "mpif77")
@run_before("configure")
def die_without_fortran(self):
# Until we can pass variants such as +fortran through virtual
# dependencies depends_on('mpi'), require Fortran compiler to
# avoid delayed build errors in dependents.
if (self.compiler.f77 is None) or (self.compiler.fc is None):
raise InstallError("Mvapich2 requires both C and Fortran compilers!")
self.setup_mpi_wrapper_variables(env)
MpichEnvironmentModifications.setup_dependent_build_environment(self, env, dependent_spec)
def configure_args(self):
spec = self.spec

31
var/spack/repos/builtin/packages/mvapich2-gdr/package.py
View File

@@ -6,9 +6,10 @@
import sys
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
class Mvapich2Gdr(AutotoolsPackage):
class Mvapich2Gdr(MpichEnvironmentModifications, AutotoolsPackage):
"""MVAPICH2-GDR is an optimized version of the MVAPICH2 MPI library for
GPU-enabled HPC and Deep Learning Applications. MVAPICH2-GDR is not
installable from source and is only available through a binary mirror.
@@ -146,11 +147,6 @@ def process_manager_options(self):
)
return opts
def setup_build_environment(self, env):
# mvapich2 configure fails when F90 and F90FLAGS are set
env.unset("F90")
env.unset("F90FLAGS")
def setup_run_environment(self, env):
if "pmi_version=pmi1" in self.spec:
env.set("SLURM_MPI_TYPE", "pmi1")
@@ -161,28 +157,11 @@ def setup_run_environment(self, env):
# Because MPI functions as a compiler, we need to treat it as one and
# add its compiler paths to the run environment.
self.setup_compiler_environment(env)
self.setup_mpi_wrapper_variables(env)
def setup_dependent_build_environment(self, env, dependent_spec):
self.setup_compiler_environment(env)
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_compiler_environment(self, env):
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
def setup_dependent_package(self, module, dependent_spec):
self.spec.mpicc = join_path(self.prefix.bin, "mpicc")
self.spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
self.spec.mpifc = join_path(self.prefix.bin, "mpif90")
self.spec.mpif77 = join_path(self.prefix.bin, "mpif77")
self.setup_mpi_wrapper_variables(env)
MpichEnvironmentModifications.setup_dependent_build_environment(self, env, dependent_spec)
def configure_args(self):
spec = self.spec

47
var/spack/repos/builtin/packages/mvapich2/package.py
View File

@@ -9,9 +9,10 @@
import spack.compilers
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
class Mvapich2(AutotoolsPackage):
class Mvapich2(MpichEnvironmentModifications, AutotoolsPackage):
"""Mvapich2 is a High-Performance MPI Library for clusters with diverse
networks (InfiniBand, Omni-Path, Ethernet/iWARP, and RoCE) and computing
platforms (x86 (Intel and AMD), ARM and OpenPOWER)"""
@@ -40,9 +41,9 @@ class Mvapich2(AutotoolsPackage):
version("2.2", sha256="791a6fc2b23de63b430b3e598bf05b1b25b82ba8bf7e0622fc81ba593b3bb131")
version("2.1", sha256="49f3225ad17d2f3b6b127236a0abdc979ca8a3efb8d47ab4b6cd4f5252d05d29")
depends_on("c", type="build") # generated
depends_on("cxx", type="build") # generated
depends_on("fortran", type="build") # generated
depends_on("c", type="build")
depends_on("cxx", type="build")
depends_on("fortran", type="build")
provides("mpi")
provides("mpi@:3.1", when="@2.3:")
@@ -358,11 +359,6 @@ def flag_handler(self, name, flags):
return (flags, None, None)
def setup_build_environment(self, env):
# mvapich2 configure fails when F90 and F90FLAGS are set
env.unset("F90")
env.unset("F90FLAGS")
def setup_run_environment(self, env):
if "process_managers=slurm" in self.spec:
if "pmi_version=pmi1" in self.spec:
@@ -373,40 +369,13 @@ def setup_run_environment(self, env):
env.set("SLURM_MPI_TYPE", "pmix")
env.set("MPI_ROOT", self.prefix)
# Because MPI functions as a compiler, we need to treat it as one and
# add its compiler paths to the run environment.
self.setup_compiler_environment(env)
self.setup_mpi_wrapper_variables(env)
def setup_dependent_build_environment(self, env, dependent_spec):
self.setup_compiler_environment(env)
# use the Spack compiler wrappers under MPI
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_compiler_environment(self, env):
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
def setup_dependent_package(self, module, dependent_spec):
self.spec.mpicc = join_path(self.prefix.bin, "mpicc")
self.spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
self.spec.mpifc = join_path(self.prefix.bin, "mpif90")
self.spec.mpif77 = join_path(self.prefix.bin, "mpif77")
@run_before("configure")
def die_without_fortran(self):
# Until we can pass variants such as +fortran through virtual
# dependencies depends_on('mpi'), require Fortran compiler to
# avoid delayed build errors in dependents.
if (self.compiler.f77 is None) or (self.compiler.fc is None):
raise InstallError("Mvapich2 requires both C and Fortran compilers!")
self.setup_mpi_wrapper_variables(env)
MpichEnvironmentModifications.setup_dependent_build_environment(self, env, dependent_spec)
def configure_args(self):
spec = self.spec

34
var/spack/repos/builtin/packages/mvapich2x/package.py
View File

@@ -6,9 +6,10 @@
import sys
from spack.package import *
from spack.pkg.builtin.mpich import MpichEnvironmentModifications
class Mvapich2x(AutotoolsPackage):
class Mvapich2x(MpichEnvironmentModifications, AutotoolsPackage):
"""MVAPICH2-X is the advanced version of the MVAPICH2 MPI library with
enhanced features (UMR, ODP, DC, Core-Direct, SHARP, XPMEM), OSU INAM
(InifniBand Network Monitoring and Analysis),PGAS (OpenSHMEM, UPC, UPC++,
@@ -198,11 +199,6 @@ def construct_cflags(self):
cflags = cflags + "-I/opt/xpmem/include"
return cflags
def setup_build_environment(self, env):
# mvapich2 configure fails when F90 and F90FLAGS are set
env.unset("F90")
env.unset("F90FLAGS")
def setup_run_environment(self, env):
if "pmi_version=pmi1" in self.spec:
env.set("SLURM_MPI_TYPE", "pmi1")
@@ -213,31 +209,11 @@ def setup_run_environment(self, env):
# Because MPI functions as a compiler, we need to treat it as one and
# add its compiler paths to the run environment.
self.setup_compiler_environment(env)
self.setup_mpi_wrapper_variables(env)
def setup_dependent_build_environment(self, env, dependent_spec):
self.setup_compiler_environment(env)
# use the Spack compiler wrappers under MPI
dependent_module = dependent_spec.package.module
env.set("MPICH_CC", dependent_module.spack_cc)
env.set("MPICH_CXX", dependent_module.spack_cxx)
env.set("MPICH_F77", dependent_module.spack_f77)
env.set("MPICH_F90", dependent_module.spack_fc)
env.set("MPICH_FC", dependent_module.spack_fc)
def setup_compiler_environment(self, env):
# For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
# Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
env.set("MPICC", join_path(self.prefix.bin, "mpicc"))
env.set("MPICXX", join_path(self.prefix.bin, "mpicxx"))
env.set("MPIF77", join_path(self.prefix.bin, "mpif77"))
env.set("MPIF90", join_path(self.prefix.bin, "mpif90"))
def setup_dependent_package(self, module, dependent_spec):
self.spec.mpicc = join_path(self.prefix.bin, "mpicc")
self.spec.mpicxx = join_path(self.prefix.bin, "mpicxx")
self.spec.mpifc = join_path(self.prefix.bin, "mpif90")
self.spec.mpif77 = join_path(self.prefix.bin, "mpif77")
self.setup_mpi_wrapper_variables(env)
MpichEnvironmentModifications.setup_dependent_build_environment(self, env, dependent_spec)
def configure_args(self):
spec = self.spec