zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update exawind packages (#40793)

Browse Source
This commit is contained in:
psakievich 2024-04-04 12:54:20 -06:00 committed by GitHub
parent 18f0ac0f94
commit 6942c7f35b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 197 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
var/spack/repos/builtin/packages/amr-wind/package.py
View File

@ -20,6 +20,7 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
license("BSD-3-Clause")
version("main", branch="main", submodules=True)
version("0.9.0", tag="v0.9.0", submodules=True)
variant("hypre", default=True, description="Enable Hypre integration")
variant("ascent", default=False, description="Enable Ascent integration")
@ -31,9 +32,22 @@ class AmrWind(CMakePackage, CudaPackage, ROCmPackage):
variant("shared", default=True, description="Build shared libraries")
variant("tests", default=True, description="Activate regression tests")
variant("tiny_profile", default=False, description="Activate tiny profile")
variant("hdf5", default=False, description="Enable HDF5 plots with ZFP compression")
variant("umpire", default=False, description="Enable Umpire")
variant("sycl", default=False, description="Enable SYCL backend")
variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
depends_on("hypre~int64@2.20.0:", when="+hypre")
depends_on("hypre+mpi", when="+hypre+mpi")
depends_on("hdf5~mpi", when="+hdf5~mpi")
depends_on("hdf5+mpi", when="+hdf5+mpi")
depends_on("h5z-zfp", when="+hdf5")
depends_on("zfp", when="+hdf5")
depends_on("hypre+umpire", when="+umpire")
depends_on("hypre+sycl", when="+sycl")
depends_on("hypre+gpu-aware-mpi", when="+gpu-aware-mpi")
depends_on("hypre@2.29.0:", when="@0.9.0:+hypre")
for arch in CudaPackage.cuda_arch_values:
depends_on("hypre+cuda cuda_arch=%s" % arch, when="+cuda+hypre cuda_arch=%s" % arch)
for arch in ROCmPackage.amdgpu_targets:
@ -88,6 +102,13 @@ def cmake_args(self):
if "+mpi" in self.spec:
args.append(define("MPI_HOME", self.spec["mpi"].prefix))
if "+hdf5" in self.spec:
cmake_options.append(self.define("AMR_WIND_ENABLE_HDF5", True))
cmake_options.append(self.define("AMR_WIND_ENABLE_HDF5_ZFP", True))
# Help AMReX understand if HDF5 is parallel or not.
# Building HDF5 with CMake as Spack does, causes this inspection to break.
cmake_options.append(self.define("HDF5_IS_PARALLEL", spec.satisfies("+mpi")))
if "+cuda" in self.spec:
amrex_arch = [
"{0:.1f}".format(float(i) / 10.0) for i in self.spec.variants["cuda_arch"].value
@ -100,4 +121,16 @@ def cmake_args(self):
targets = self.spec.variants["amdgpu_target"].value
args.append("-DAMReX_AMD_ARCH=" + ";".join(str(x) for x in targets))
if "+sycl" in self.spec:
cmake_options.append(self.define("AMR_WIND_ENABLE_SYCL", True))
requires(
"%dpcpp",
"%oneapi",
policy="one_of",
msg=(
"AMReX's SYCL GPU Backend requires DPC++ (dpcpp) "
"or the oneAPI CXX (icpx) compiler."
),
)
return args

88
var/spack/repos/builtin/packages/exawind/package.py
View File

@ -6,37 +6,93 @@
from spack.package import *
class Exawind(CMakePackage):
class Exawind(CMakePackage, CudaPackage, ROCmPackage):
"""Multi-application driver for Exawind project."""
homepage = "https://github.com/Exawind/exawind-driver"
git = "https://github.com/Exawind/exawind-driver.git"
maintainers("jrood-nrel", "psakievich")
maintainers("jrood-nrel")
tags = ["ecp", "ecp-apps"]
# Testing is currently always enabled, but should be optional in the future
# to avoid cloning the mesh submodule
version("master", branch="main", submodules=True)
version("1.0.0", tag="v1.0.0", submodules=True)
license("Apache-2.0")
version("master", branch="main")
variant("openfast", default=False, description="Enable OpenFAST integration")
variant("hypre", default=True, description="Enable hypre solver")
variant("stk_simd", default=False, description="Enable SIMD in STK")
variant("umpire", default=False, description="Enable Umpire")
variant("tiny_profile", default=False, description="Turn on AMR-wind with tiny profile")
variant("sycl", default=False, description="Enable SYCL backend for AMR-Wind")
variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
depends_on("trilinos+stk")
depends_on("tioga+shared~nodegid")
depends_on("nalu-wind+hypre+openfast+tioga+wind-utils")
depends_on("amr-wind+hypre+mpi+netcdf+openfast")
depends_on("openfast+cxx+shared@2.6.0:")
conflicts("amr-wind+hypre", when="+sycl")
for arch in CudaPackage.cuda_arch_values:
depends_on("amr-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
depends_on("nalu-wind+cuda cuda_arch=%s" % arch, when="+cuda cuda_arch=%s" % arch)
for arch in ROCmPackage.amdgpu_targets:
depends_on("amr-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
depends_on("nalu-wind+rocm amdgpu_target=%s" % arch, when="+rocm amdgpu_target=%s" % arch)
depends_on("nalu-wind+tioga")
depends_on("amr-wind+netcdf+mpi")
depends_on("tioga~nodegid")
depends_on("yaml-cpp@0.6:")
depends_on("nalu-wind+openfast", when="+openfast")
depends_on("amr-wind+hypre", when="+hypre~sycl")
depends_on("amr-wind~hypre", when="~hypre")
depends_on("nalu-wind+hypre", when="+hypre")
depends_on("nalu-wind~hypre", when="~hypre")
depends_on("amr-wind+sycl", when="+sycl")
depends_on("nalu-wind+umpire", when="+umpire")
depends_on("amr-wind+umpire", when="+umpire")
depends_on("amr-wind+tiny_profile", when="+tiny_profile")
depends_on("nalu-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
depends_on("amr-wind+gpu-aware-mpi", when="+gpu-aware-mpi")
depends_on("nalu-wind@2.0.0:", when="@1.0.0:")
depends_on("amr-wind@0.9.0:", when="@1.0.0:")
depends_on("tioga@1.0.0:", when="@1.0.0:")
def cmake_args(self):
spec = self.spec
args = [
self.define("Trilinos_DIR", spec["trilinos"].prefix),
self.define("TIOGA_DIR", spec["tioga"].prefix),
self.define("Nalu-Wind_DIR", spec["nalu-wind"].prefix),
self.define("AMR-Wind_DIR", spec["amr-wind"].prefix),
self.define("OpenFAST_DIR", spec["openfast"].prefix),
self.define("YAML-CPP_DIR", spec["yaml-cpp"].prefix),
]
args = [self.define("MPI_HOME", spec["mpi"].prefix)]
if "+umpire" in self.spec:
args.append(self.define_from_variant("EXAWIND_ENABLE_UMPIRE", "umpire"))
args.append(self.define("UMPIRE_DIR", self.spec["umpire"].prefix))
if spec.satisfies("+cuda"):
args.append(self.define("EXAWIND_ENABLE_CUDA", True))
args.append(self.define("CUDAToolkit_ROOT", self.spec["cuda"].prefix))
args.append(self.define("EXAWIND_CUDA_ARCH", self.spec.variants["cuda_arch"].value))
if spec.satisfies("+rocm"):
targets = self.spec.variants["amdgpu_target"].value
args.append(self.define("EXAWIND_ENABLE_ROCM", True))
args.append(self.define("CMAKE_CXX_COMPILER", self.spec["hip"].hipcc))
args.append(self.define("CMAKE_HIP_ARCHITECTURES", ";".join(str(x) for x in targets)))
args.append(self.define("AMDGPU_TARGETS", ";".join(str(x) for x in targets)))
args.append(self.define("GPU_TARGETS", ";".join(str(x) for x in targets)))
if spec.satisfies("^amr-wind+hdf5"):
args.append(self.define("H5Z_ZFP_USE_STATIC_LIBS", True))
if spec.satisfies("^amr-wind+ascent"):
args.append(self.define("CMAKE_EXE_LINKER_FLAGS", self.compiler.openmp_flag))
return args
def setup_build_environment(self, env):
if "~stk_simd" in self.spec:
env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
if "+rocm" in self.spec:
env.set("OMPI_CXX", self.spec["hip"].hipcc)
env.set("MPICH_CXX", self.spec["hip"].hipcc)
env.set("MPICXX_CXX", self.spec["hip"].hipcc)

4
var/spack/repos/builtin/packages/hypre/package.py
View File

@ -72,6 +72,7 @@ class Hypre(AutotoolsPackage, CudaPackage, ROCmPackage):
variant("int64", default=False, description="Use 64bit integers")
variant("mixedint", default=False, description="Use 64bit integers while reducing memory use")
variant("complex", default=False, description="Use complex values")
variant("gpu-aware-mpi", default=False, description="Use gpu-aware mpi")
variant("mpi", default=True, description="Enable MPI support")
variant("openmp", default=False, description="Enable OpenMP support")
variant("debug", default=False, description="Build debug instead of optimized version")
@ -300,6 +301,9 @@ def configure_args(self):
configure_args.append("--with-magma-lib=%s" % spec["magma"].libs)
configure_args.append("--with-magma")
if "+gpu-aware-mpi" in spec:
options.append("--enable-gpu-aware-mpi")
configure_args.extend(self.enable_or_disable("fortran"))
return configure_args

76
var/spack/repos/builtin/packages/nalu-wind/package.py
View File

@ -15,7 +15,7 @@ def _parse_float(val):
return False
class NaluWind(CMakePackage, CudaPackage):
class NaluWind(CMakePackage, CudaPackage, ROCmPackage):
"""Nalu-Wind: Wind energy focused variant of Nalu."""
homepage = "https://nalu-wind.readthedocs.io"
@ -26,6 +26,7 @@ class NaluWind(CMakePackage, CudaPackage):
tags = ["ecp", "ecp-apps"]
version("master", branch="master")
version("2.0.0", tag="v2.0.0")
variant("pic", default=True, description="Position independent code")
variant(
@ -45,12 +46,31 @@ class NaluWind(CMakePackage, CudaPackage):
variant("hypre", default=True, description="Compile with Hypre support")
variant("trilinos-solvers", default=True, description="Compile with Trilinos Solvers support")
variant("catalyst", default=False, description="Compile with Catalyst support")
variant("shared", default=True, description="Build shared libraries")
variant("fftw", default=False, description="Compile with FFTW support")
variant("fsi", default=False, description="Enable fluid-structure-interaction models")
variant("boost", default=False, description="Enable Boost integration")
variant("gpu-aware-mpi", default=False, description="gpu-aware-mpi")
variant("wind-utils", default=False, description="Build wind-utils")
variant("umpire", default=False, description="Enable Umpire")
conflicts(
"+shared",
when="+cuda",
msg="invalid device functions are generated with shared libs and cuda",
)
conflicts(
"+shared",
when="+rocm",
msg="invalid device functions are generated with shared libs and rocm",
)
conflicts("+cuda", when="+rocm")
conflicts("+rocm", when="+cuda")
depends_on("mpi")
depends_on("yaml-cpp@0.5.3:")
depends_on("openfast@4.0.0:+cxx+netcdf", when="+fsi")
depends_on("trilinos@13.4.1+exodus+zoltan+stk", when="@=2.0.0")
depends_on("hypre@2.29.0:", when="@2.0.0:+hypre")
depends_on(
"trilinos@13:+exodus+tpetra+zoltan+stk~superlu-dist~superlu+hdf5+shards~hypre+gtest"
)
@ -76,12 +96,42 @@ class NaluWind(CMakePackage, CudaPackage):
"hypre@develop +mpi+cuda~int64~superlu-dist cuda_arch={0}".format(_arch),
when="+hypre+cuda cuda_arch={0}".format(_arch),
)
for _arch in ROCmPackage.amdgpu_targets:
depends_on(
"trilinos@13.4.0.2022.10.27: "
"~shared+exodus+tpetra+zoltan+stk~superlu-dist~superlu"
"+hdf5+shards~hypre+gtest+rocm amdgpu_target={0}".format(_arch),
when="+rocm amdgpu_target={0}".format(_arch),
)
depends_on(
"hypre+rocm amdgpu_target={0}".format(_arch),
when="+hypre+rocm amdgpu_target={0}".format(_arch),
)
depends_on("trilinos-catalyst-ioss-adapter", when="+catalyst")
depends_on("fftw+mpi", when="+fftw")
depends_on("nccmp")
# indirect dependency needed to make original concretizer work
depends_on("netcdf-c+parallel-netcdf")
depends_on("boost +filesystem +iostreams cxxstd=14", when="+boost")
supported_cxxstd = ["17"]
variant(
"cxxstd", default="17", values=supported_cxxstd, multi=False, description="cxx standard"
)
for std in supported_cxxstd:
depends_on("trilinos cxxstd=%s" % std, when="cxxstd=%s" % std)
def setup_build_environment(self, env):
if "~stk_simd" in self.spec:
env.append_flags("CXXFLAGS", "-DUSE_STK_SIMD_NONE")
if "+cuda" in self.spec:
env.set("CUDA_LAUNCH_BLOCKING", "1")
env.set("CUDA_MANAGED_FORCE_DEVICE_ALLOC", "1")
env.set("OMPI_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
env.set("MPICH_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
env.set("MPICXX_CXX", self.spec["kokkos-nvcc-wrapper"].kokkos_cxx)
if "+rocm" in self.spec:
env.append_flags("CXXFLAGS", "-fgpu-rdc")
def cmake_args(self):
spec = self.spec
@ -95,22 +145,26 @@ def cmake_args(self):
self.define_from_variant("ENABLE_CUDA", "cuda"),
self.define_from_variant("ENABLE_WIND_UTILS", "wind-utils"),
self.define_from_variant("ENABLE_BOOST", "boost"),
self.define_from_variant("CMAKE_CXX_STANDARD", "cxxstd"),
self.define_from_variant("BUILD_SHARED_LIBS", "shared"),
self.define_from_variant("ENABLE_OPENFAST", "openfast"),
self.define_from_variant("ENABLE_TIOGA", "tioga"),
self.define_from_variant("ENABLE_HYPRE", "hypre"),
self.define_from_variant("ENABLE_TRILINOS_SOLVERS", "trilinos-solvers"),
self.define_from_variant("ENABLE_PARAVIEW_CATALYST", "catalyst"),
self.define_from_variant("ENABLE_FFTW", "fftw"),
self.define_from_variant("ENABLE_UMPIRE", "umpire"),
]
args.append(self.define_from_variant("ENABLE_OPENFAST", "openfast"))
if "+openfast" in spec:
args.append(self.define("OpenFAST_DIR", spec["openfast"].prefix))
args.append(self.define_from_variant("ENABLE_TIOGA", "tioga"))
if "+tioga" in spec:
args.append(self.define("TIOGA_DIR", spec["tioga"].prefix))
args.append(self.define_from_variant("ENABLE_HYPRE", "hypre"))
if "+hypre" in spec:
args.append(self.define("HYPRE_DIR", spec["hypre"].prefix))
args.append(self.define_from_variant("ENABLE_TRILINOS_SOLVERS", "trilinos-solvers"))
args.append(self.define_from_variant("ENABLE_PARAVIEW_CATALYST", "catalyst"))
if "+catalyst" in spec:
args.append(
self.define(
@ -118,7 +172,6 @@ def cmake_args(self):
)
)
args.append(self.define_from_variant("ENABLE_FFTW", "fftw"))
if "+fftw" in spec:
args.append(self.define("FFTW_DIR", spec["fftw"].prefix))
@ -131,6 +184,15 @@ def cmake_args(self):
]
)
if "+umpire" in spec:
args.append(self.define("UMPIRE_DIR", spec["umpire"].prefix))
if "+rocm" in spec:
args.append(self.define("CMAKE_CXX_COMPILER", spec["hip"].hipcc))
args.append(self.define("ENABLE_ROCM", True))
targets = spec.variants["amdgpu_target"].value
args.append(self.define("GPU_TARGETS", ";".join(str(x) for x in targets)))
if "darwin" in spec.architecture:
args.append(self.define("CMAKE_MACOSX_RPATH", "ON"))

17
var/spack/repos/builtin/packages/openfast/hub_seg_fault.patch Normal file
View File

@ -0,0 +1,17 @@
diff --git a/modules/openfast-library/src/FAST_Solver.f90 b/modules/openfast-library/src/FAST_Solver.f90
index 364d0b78..10056965 100644
--- a/modules/openfast-library/src/FAST_Solver.f90
+++ b/modules/openfast-library/src/FAST_Solver.f90
@@ -607,9 +607,9 @@ SUBROUTINE AD_InputSolve_IfW( p_FAST, u_AD, y_IfW, y_OpFM, ErrStat, ErrMsg )
end if
if (u_AD%rotors(1)%NacelleMotion%NNodes > 0) then
- u_AD%rotors(1)%InflowOnNacelle(1) = y_OpFM%u(node)
- u_AD%rotors(1)%InflowOnNacelle(2) = y_OpFM%v(node)
- u_AD%rotors(1)%InflowOnNacelle(3) = y_OpFM%w(node)
+ u_AD%rotors(1)%InflowOnNacelle(1) = y_OpFM%u(1)
+ u_AD%rotors(1)%InflowOnNacelle(2) = y_OpFM%v(1)
+ u_AD%rotors(1)%InflowOnNacelle(3) = y_OpFM%w(1)
node = node + 1
else
u_AD%rotors(1)%InflowOnNacelle = 0.0_ReKi

1
var/spack/repos/builtin/packages/openfast/package.py
View File

@ -13,6 +13,7 @@ class Openfast(CMakePackage):
git = "https://github.com/OpenFAST/openfast.git"
maintainers("jrood-nrel")
patch("hub_seg_fault.patch", when="@2.7:3.2")
license("Apache-2.0")

1
var/spack/repos/builtin/packages/tioga/package.py
View File

@ -20,6 +20,7 @@ class Tioga(CMakePackage):
# The original TIOGA repo has possibly been abandoned,
# so work on TIOGA has continued in the Exawind project
version("develop", git="https://github.com/Exawind/tioga.git", branch="exawind")
version("1.0.0", git="https://github.com/Exawind/tioga.git", tag="v1.0.0")
version("master", branch="master")
variant("shared", default=sys.platform != "darwin", description="Build shared libraries")