zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ExaGO: add v1.4.0 and v1.4.1 (#29712)

Browse Source 
* ExaGO: Handling of cuda architectures and amdgpu targets changed
  to effectively handle multiple targets. See #28441.
* Add ROCm support to ExaGO and update ROCm support in HiOp
* ExaGO+rocm requires HiOp+rocm
* Newer versions of CMake may set HIP_CLANG_INCLUDE_PATH incorrectly:
  add comments to the ExaGO/HiOp packages explaining how to address
  this problem if it occurs.
This commit is contained in:
Asher Mancinelli 2022-03-29 11:01:30 -06:00 committed by GitHub
parent 9516fa9447
commit 078ee48c4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 84 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
var/spack/repos/builtin/packages/exago/package.py
View File

@ -6,7 +6,7 @@
from spack import *
class Exago(CMakePackage, CudaPackage):
class Exago(CMakePackage, CudaPackage, ROCmPackage):
"""ExaGO is a package for solving large-scale power grid optimization
problems on parallel and distributed architectures, particularly targeted
for exascale machines."""
@ -15,7 +15,9 @@ class Exago(CMakePackage, CudaPackage):
git = 'https://gitlab.pnnl.gov/exasgd/frameworks/exago.git'
maintainers = ['ashermancinelli', 'CameronRutherford']
version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True, preferred=True)
version('1.4.1', commit='ea607c685444b5f345bfdc9a59c345f0f30adde2', submodules=True, preferred=True)
version('1.4.0', commit='4f4c3fdb40b52ace2d6ba000e7f24b340ec8e886', submodules=True)
version('1.3.0', commit='58b039d746a6eac8e84b0afc01354cd58caec485', submodules=True)
version('1.2.0', commit='255a214e', submodules=True)
version('1.1.2', commit='db3bb16e', submodules=True)
version('1.1.1', commit='0e0a3f27', submodules=True)
@ -30,6 +32,7 @@ class Exago(CMakePackage, CudaPackage):
variant('mpi', default=True, description='Enable/Disable MPI')
variant('raja', default=False, description='Enable/Disable RAJA')
variant('python', default=True, description='Enable/Disable Python bindings')
conflicts('+python', when='@:1.3.0', msg='Python bindings require ExaGO 1.4')
# Solver options
variant('hiop', default=False, description='Enable/Disable HiOp')
@ -44,6 +47,7 @@ class Exago(CMakePackage, CudaPackage):
depends_on('raja', when='+raja')
depends_on('raja+cuda', when='+raja+cuda')
depends_on('raja+rocm', when='+raja+rocm')
depends_on('raja@0.14.0:', when='@1.1.0: +raja')
depends_on('umpire', when='+raja')
depends_on('umpire@6.0.0:', when='@1.1.0: +raja')
@ -65,37 +69,61 @@ class Exago(CMakePackage, CudaPackage):
depends_on('hiop@0.5.3:', when='@1.3.0:+hiop')
depends_on('hiop+cuda', when='+hiop+cuda')
depends_on('hiop+rocm', when='+hiop+rocm')
depends_on('hiop~mpi', when='+hiop~mpi')
depends_on('hiop+mpi', when='+hiop+mpi')
depends_on('petsc@3.13:3.14', when='@:1.2.99')
depends_on('petsc@3.16.0', when='@1.3.0:')
depends_on('petsc@3.16.0:', when='@1.3.0:')
depends_on('petsc~mpi', when='~mpi')
depends_on('ipopt', when='+ipopt')
depends_on('py-mpi4py', when='@1.3.0:+mpi+python')
flag_handler = build_system_flags
def cmake_args(self):
args = []
spec = self.spec
args.append("-DEXAGO_RUN_TESTS=ON")
# NOTE: If building with spack develop on a cluster, you may want to
# change the ctest launch command to use your job scheduler like so:
#
# args.append(
# self.define('EXAGO_CTEST_LAUNCH_COMMAND', 'srun -t 10:00'))
args.append(self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi'))
args.append(self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja'))
args.append(self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop'))
args.append(self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt'))
args.append(self.define_from_variant('EXAGO_ENABLE_GPU', 'cuda'))
args.append(self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda'))
args.append(self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python'))
args.append("-DPETSC_DIR='{0}'".format(spec['petsc'].prefix))
args.extend([
self.define('EXAGO_ENABLE_GPU', '+cuda' in spec or '+rocm' in spec),
self.define_from_variant('EXAGO_ENABLE_CUDA', 'cuda'),
self.define_from_variant('EXAGO_ENABLE_HIP', 'rocm'),
self.define('PETSC_DIR', spec['petsc'].prefix),
self.define('EXAGO_RUN_TESTS', True),
self.define_from_variant('EXAGO_ENABLE_MPI', 'mpi'),
self.define_from_variant('EXAGO_ENABLE_RAJA', 'raja'),
self.define_from_variant('EXAGO_ENABLE_HIOP', 'hiop'),
self.define_from_variant('EXAGO_ENABLE_IPOPT', 'ipopt'),
self.define_from_variant('EXAGO_ENABLE_PYTHON', 'python'),
])
if '+cuda' in spec:
cuda_arch_list = spec.variants['cuda_arch'].value
cuda_arch = cuda_arch_list[0]
if cuda_arch != 'none':
if cuda_arch_list[0] != 'none':
args.append(
"-DCMAKE_CUDA_ARCHITECTURES={0}".format(cuda_arch))
self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch_list))
# NOTE: if +rocm, some HIP CMake variables may not be set correctly.
# Namely, HIP_CLANG_INCLUDE_PATH. If the configure phase fails due to
# this variable being undefined, adding the following line typically
# resolves this issue:
#
# args.append(
# self.define('HIP_CLANG_INCLUDE_PATH',
# '/opt/rocm-X.Y.Z/llvm/lib/clang/14.0.0/include/'))
if '+rocm' in spec:
rocm_arch_list = spec.variants['amdgpu_target'].value
if rocm_arch_list[0] != 'none':
args.append(self.define('GPU_TARGETS', rocm_arch_list))
args.append(self.define('AMDGPU_TARGETS', rocm_arch_list))
return args

54
var/spack/repos/builtin/packages/hiop/package.py
View File

@ -76,7 +76,8 @@ class Hiop(CMakePackage, CudaPackage, ROCmPackage):
depends_on('magma@{0}:'.format(magma_v), when='@{0}:+cuda'.format(hiop_v))
depends_on('magma@{0}:'.format(magma_v), when='@{0}:+rocm'.format(hiop_v))
depends_on('raja+openmp', when='+raja')
depends_on('raja', when='+raja')
depends_on('raja+openmp', when='+raja~cuda~rocm')
depends_on('raja@0.14.0:', when='@0.5.0:+raja')
depends_on('raja+cuda', when='+raja+cuda')
depends_on('raja+rocm', when='+raja+rocm')
@ -105,15 +106,19 @@ def cmake_args(self):
args = []
spec = self.spec
if spec.satisfies('+rocm') or spec.satisfies('+cuda'):
args.append('-DHIOP_USE_GPU=ON')
args.append('-DHIOP_USE_MAGMA=ON')
use_gpu = '+cuda' in spec or '+rocm' in spec
if use_gpu:
args.extend([
self.define('HIOP_USE_GPU', True),
self.define('HIOP_USE_MAGMA', True),
self.define('HIOP_MAGMA_DIR', spec['magma'].prefix),
])
args.extend([
self.define('HIOP_BUILD_STATIC', True),
self.define('LAPACK_FOUND', True),
self.define('LAPACK_LIBRARIES', spec['lapack'].libs + spec['blas'].libs),
self.define('HIOP_USE_HIP', False),
self.define_from_variant('HIOP_BUILD_SHARED', 'shared'),
self.define_from_variant('HIOP_USE_MPI', 'mpi'),
self.define_from_variant('HIOP_DEEPCHECKS', 'deepchecking'),
@ -127,6 +132,12 @@ def cmake_args(self):
self.define_from_variant('HIOP_TEST_WITH_BSUB', 'jsrun'),
])
# NOTE: If building with spack develop on a cluster, you may want to
# change the ctest launch command to use your job scheduler like so:
#
# args.append(
# self.define('HIOP_CTEST_LAUNCH_COMMAND', 'srun -t 10:00'))
if '+mpi' in spec:
args.extend([
self.define('MPI_HOME', spec['mpi'].prefix),
@ -134,17 +145,34 @@ def cmake_args(self):
self.define('MPI_CXX_COMPILER', spec['mpi'].mpicxx),
self.define('MPI_Fortran_COMPILER', spec['mpi'].mpifc),
])
# NOTE: On Cray platforms, libfabric is occasionally not picked up
# by Spack, causing HiOp's CMake code to fail to find MPI Fortran
# libraries. If this is the case, adding the following lines may
# resolve the issue. Searching <builddir>/CMakeFiles/CMakeError.log
# for MPI Fortran errors is the fastest way to check for this error.
#
# args.append(
# self.define('MPI_Fortran_LINK_FLAGS',
# '-L/path/to/libfabric/lib64/ -lfabric'))
if '+cuda' in spec:
cuda_arch_list = spec.variants['cuda_arch'].value
cuda_arch = cuda_arch_list[0]
if cuda_arch != 'none':
args.extend([
self.define('HIOP_NVCC_ARCH', 'sm_{0}'.format(cuda_arch)),
self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch),
])
if '+magma' in spec:
args.append(self.define('HIOP_MAGMA_DIR', spec['magma'].prefix))
if cuda_arch_list[0] != 'none':
args.append(self.define('CMAKE_CUDA_ARCHITECTURES', cuda_arch_list))
# NOTE: if +rocm, some HIP CMake variables may not be set correctly.
# Namely, HIP_CLANG_INCLUDE_PATH. If the configure phase fails due to
# this variable being undefined, adding the following line typically
# resolves this issue:
#
# args.append(
# self.define('HIP_CLANG_INCLUDE_PATH',
# '/opt/rocm-X.Y.Z/llvm/lib/clang/14.0.0/include/'))
if '+rocm' in spec:
rocm_arch_list = spec.variants['amdgpu_target'].value
if rocm_arch_list[0] != 'none':
args.append(self.define('GPU_TARGETS', rocm_arch_list))
args.append(self.define('AMDGPU_TARGETS', rocm_arch_list))
if '+kron' in spec:
args.append(self.define('HIOP_UMFPACK_DIR', spec['suite-sparse'].prefix))