zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mfem: patch @4.3.0 to support hypre up to v2.23.0 (#26640)

Browse Source
This commit is contained in:
Veselin Dobrev 2021-10-12 09:29:56 -07:00 committed by GitHub
parent ff66c237c0
commit e2a64bb483
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 577 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

575
var/spack/repos/builtin/packages/mfem/mfem-4.3-hypre-2.23.0.patch Normal file
View File

@ -0,0 +1,575 @@
diff --git a/fem/pfespace.cpp b/fem/pfespace.cpp
index 1e2a47a5b..53964328e 100644
--- a/fem/pfespace.cpp
+++ b/fem/pfespace.cpp
@@ -1278,12 +1278,17 @@ const FiniteElement *ParFiniteElementSpace::GetFaceNbrFaceFE(int i) const
void ParFiniteElementSpace::Lose_Dof_TrueDof_Matrix()
{
+ P -> StealData();
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrix *csrP = (hypre_ParCSRMatrix*)(*P);
hypre_ParCSRMatrixOwnsRowStarts(csrP) = 1;
hypre_ParCSRMatrixOwnsColStarts(csrP) = 1;
- P -> StealData();
dof_offsets.LoseData();
tdof_offsets.LoseData();
+#else
+ dof_offsets.DeleteAll();
+ tdof_offsets.DeleteAll();
+#endif
}
void ParFiniteElementSpace::ConstructTrueDofs()
diff --git a/linalg/complex_operator.cpp b/linalg/complex_operator.cpp
index 1f42ab425..8a2f8e580 100644
--- a/linalg/complex_operator.cpp
+++ b/linalg/complex_operator.cpp
@@ -787,10 +787,16 @@ HypreParMatrix * ComplexHypreParMatrix::GetSystemMatrix() const
2 * num_cols_offd, cmap,
true);
+#if MFEM_HYPRE_VERSION <= 22200
// Give the new matrix ownership of row_starts and col_starts
hypre_ParCSRMatrix *hA = (hypre_ParCSRMatrix*)(*A);
+
hypre_ParCSRMatrixSetRowStartsOwner(hA,1);
hypre_ParCSRMatrixSetColStartsOwner(hA,1);
+#else
+ mfem_hypre_TFree_host(row_starts);
+ mfem_hypre_TFree_host(col_starts);
+#endif
return A;
}
diff --git a/linalg/hypre.cpp b/linalg/hypre.cpp
index 3f393e39a..6ca80e987 100644
--- a/linalg/hypre.cpp
+++ b/linalg/hypre.cpp
@@ -88,7 +88,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
{
x = hypre_ParVectorCreate(comm,glob_size,col);
hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
// The data will be destroyed by hypre (this is the default)
hypre_ParVectorSetDataOwner(x,1);
hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -105,7 +107,9 @@ HypreParVector::HypreParVector(MPI_Comm comm, HYPRE_BigInt glob_size,
hypre_ParVectorSetDataOwner(x,1); // owns the seq vector
hypre_Vector *x_loc = hypre_ParVectorLocalVector(x);
hypre_SeqVectorSetDataOwner(x_loc,0);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
double tmp = 0.0;
hypre_VectorData(x_loc) = &tmp;
#ifdef HYPRE_USING_CUDA
@@ -128,7 +132,9 @@ HypreParVector::HypreParVector(const HypreParVector &y) : Vector()
x = hypre_ParVectorCreate(y.x -> comm, y.x -> global_size,
y.x -> partitioning);
hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
hypre_ParVectorSetDataOwner(x,1);
hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
_SetDataAndSize_();
@@ -162,7 +168,9 @@ HypreParVector::HypreParVector(ParFiniteElementSpace *pfes)
x = hypre_ParVectorCreate(pfes->GetComm(), pfes->GlobalTrueVSize(),
pfes->GetTrueDofOffsets());
hypre_ParVectorInitialize(x);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParVectorSetPartitioningOwner(x,0);
+#endif
// The data will be destroyed by hypre (this is the default)
hypre_ParVectorSetDataOwner(x,1);
hypre_SeqVectorSetDataOwner(hypre_ParVectorLocalVector(x),1);
@@ -683,8 +691,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, HYPRE_BigInt glob_size,
A = hypre_ParCSRMatrixCreate(comm, glob_size, glob_size, row_starts,
row_starts, 0, diag->NumNonZeroElems(), 0);
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
hypre_CSRMatrixSetDataOwner(A->diag,0);
diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -726,8 +736,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
row_starts, col_starts,
0, diag->NumNonZeroElems(), 0);
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
hypre_CSRMatrixSetDataOwner(A->diag,0);
diagOwner = CopyCSR(diag, mem_diag, A->diag, false);
@@ -770,8 +782,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
offd->Width(), diag->NumNonZeroElems(),
offd->NumNonZeroElems());
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
hypre_CSRMatrixSetDataOwner(A->diag,0);
diagOwner = CopyCSR(diag, mem_diag, A->diag, own_diag_offd);
@@ -817,8 +831,10 @@ HypreParMatrix::HypreParMatrix(
A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
row_starts, col_starts, offd_num_cols, 0, 0);
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
HYPRE_Int local_num_rows = hypre_CSRMatrixNumRows(A->diag);
@@ -931,8 +947,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm,
A = hypre_ParCSRMatrixCreate(comm, global_num_rows, global_num_cols,
row_starts, col_starts, 0, nnz, 0);
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
hypre_CSRMatrixSetDataOwner(A->diag,0);
diagOwner = CopyBoolCSR(diag, mem_diag, A->diag);
@@ -989,8 +1007,10 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int id, int np,
}
hypre_ParCSRMatrixSetDataOwner(A,1);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(A,0);
hypre_ParCSRMatrixSetColStartsOwner(A,0);
+#endif
mem_diag.data.New(diag_nnz);
for (HYPRE_Int i = 0; i < diag_nnz; i++)
@@ -1177,6 +1197,13 @@ HypreParMatrix::HypreParMatrix(MPI_Comm comm, int nrows,
{
hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
}
+#if MFEM_HYPRE_VERSION > 22200
+ mfem_hypre_TFree_host(row_starts);
+ if (rows != cols)
+ {
+ mfem_hypre_TFree_host(col_starts);
+ }
+#endif
hypre_MatvecCommPkgCreate(A);
height = GetNumRows();
@@ -1263,6 +1290,7 @@ void HypreParMatrix::SetOwnerFlags(signed char diag, signed char offd,
void HypreParMatrix::CopyRowStarts()
{
+#if MFEM_HYPRE_VERSION <= 22200
if (!A || hypre_ParCSRMatrixOwnsRowStarts(A) ||
(hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
hypre_ParCSRMatrixOwnsColStarts(A)))
@@ -1297,10 +1325,12 @@ void HypreParMatrix::CopyRowStarts()
hypre_ParCSRMatrixColStarts(A) = new_row_starts;
hypre_ParCSRMatrixOwnsColStarts(A) = 0;
}
+#endif
}
void HypreParMatrix::CopyColStarts()
{
+#if MFEM_HYPRE_VERSION <= 22200
if (!A || hypre_ParCSRMatrixOwnsColStarts(A) ||
(hypre_ParCSRMatrixRowStarts(A) == hypre_ParCSRMatrixColStarts(A) &&
hypre_ParCSRMatrixOwnsRowStarts(A)))
@@ -1339,6 +1369,7 @@ void HypreParMatrix::CopyColStarts()
{
hypre_ParCSRMatrixOwnsColStarts(A) = 1;
}
+#endif
}
void HypreParMatrix::GetDiag(Vector &diag) const
@@ -1490,6 +1521,10 @@ HypreParMatrix * HypreParMatrix::Transpose() const
HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
double threshold) const
{
+ // hypre_ParCSRMatrixExtractSubmatrixFC works on host only, so we move this
+ // matrix to host, temporarily:
+ HostRead();
+
if (!(A->comm))
{
hypre_MatvecCommPkgCreate(A);
@@ -1501,15 +1536,28 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
int local_num_vars = hypre_CSRMatrixNumRows(hypre_ParCSRMatrixDiag(A));
// Form hypre CF-splitting array designating submatrix as F-points (-1)
+#ifdef hypre_IntArrayData
+ // hypre_BoomerAMGCoarseParms needs CF_marker to be hypre_IntArray *
+ hypre_IntArray *CF_marker;
+
+ CF_marker = hypre_IntArrayCreate(local_num_vars);
+ hypre_IntArrayInitialize_v2(CF_marker, HYPRE_MEMORY_HOST);
+ hypre_IntArraySetConstantValues(CF_marker, 1);
+#else
Array<HYPRE_Int> CF_marker(local_num_vars);
CF_marker = 1;
+#endif
for (int j=0; j<indices.Size(); j++)
{
if (indices[j] > local_num_vars)
{
MFEM_WARNING("WARNING : " << indices[j] << " > " << local_num_vars);
}
+#ifdef hypre_IntArrayData
+ hypre_IntArrayData(CF_marker)[indices[j]] = -1;
+#else
CF_marker[indices[j]] = -1;
+#endif
}
// Construct cpts_global array on hypre matrix structure
@@ -1518,10 +1566,22 @@ HypreParMatrix *HypreParMatrix::ExtractSubmatrix(const Array<int> &indices,
CF_marker, NULL, &cpts_global);
// Extract submatrix into *submat
+#ifdef hypre_IntArrayData
+ hypre_ParCSRMatrixExtractSubmatrixFC(A, hypre_IntArrayData(CF_marker),
+ cpts_global, "FF", &submat,
+ threshold);
+#else
hypre_ParCSRMatrixExtractSubmatrixFC(A, CF_marker, cpts_global,
"FF", &submat, threshold);
+#endif
mfem_hypre_TFree(cpts_global);
+#ifdef hypre_IntArrayData
+ hypre_IntArrayDestroy(CF_marker);
+#endif
+
+ HypreRead(); // restore the matrix location to the default hypre location
+
return new HypreParMatrix(submat);
}
#endif
@@ -1791,9 +1851,14 @@ HypreParMatrix* HypreParMatrix::LeftDiagMult(const SparseMatrix &D,
DA_diag, DA_offd, new_col_map_offd,
own_diag_offd);
+#if MFEM_HYPRE_VERSION <= 22200
// Give ownership of row_starts, col_starts, and col_map_offd to DA
hypre_ParCSRMatrixSetRowStartsOwner(DA->A, 1);
hypre_ParCSRMatrixSetColStartsOwner(DA->A, 1);
+#else
+ mfem_hypre_TFree_host(new_row_starts);
+ mfem_hypre_TFree_host(new_col_starts);
+#endif
DA->colMapOwner = 1;
return DA;
@@ -1948,18 +2013,22 @@ void HypreParMatrix::Threshold(double threshold)
row_starts = hypre_ParCSRMatrixRowStarts(A);
col_starts = hypre_ParCSRMatrixColStarts(A);
+#if MFEM_HYPRE_VERSION <= 22200
bool old_owns_row = hypre_ParCSRMatrixOwnsRowStarts(A);
bool old_owns_col = hypre_ParCSRMatrixOwnsColStarts(A);
+#endif
HYPRE_BigInt global_num_rows = hypre_ParCSRMatrixGlobalNumRows(A);
HYPRE_BigInt global_num_cols = hypre_ParCSRMatrixGlobalNumCols(A);
parcsr_A_ptr = hypre_ParCSRMatrixCreate(comm, global_num_rows,
global_num_cols,
row_starts, col_starts,
0, 0, 0);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixOwnsRowStarts(parcsr_A_ptr) = old_owns_row;
hypre_ParCSRMatrixOwnsColStarts(parcsr_A_ptr) = old_owns_col;
hypre_ParCSRMatrixOwnsRowStarts(A) = 0;
hypre_ParCSRMatrixOwnsColStarts(A) = 0;
+#endif
csr_A = hypre_MergeDiagAndOffd(A);
@@ -1994,7 +2063,12 @@ void HypreParMatrix::Threshold(double threshold)
hypre_ParCSRMatrixSetNumNonzeros(A);
/* Make sure that the first entry in each row is the diagonal one. */
+#if MFEM_HYPRE_VERSION <= 22200
if (row_starts == col_starts)
+#else
+ if ((row_starts[0] == col_starts[0]) &&
+ (row_starts[1] == col_starts[1]))
+#endif
{
hypre_CSRMatrixReorder(hypre_ParCSRMatrixDiag(A));
}
@@ -2310,7 +2384,7 @@ inline void delete_hypre_ParCSRMatrixColMapOffd(hypre_ParCSRMatrix *A)
{
HYPRE_BigInt *A_col_map_offd = hypre_ParCSRMatrixColMapOffd(A);
int size = hypre_CSRMatrixNumCols(hypre_ParCSRMatrixOffd(A));
- Memory<HYPRE_Int>(A_col_map_offd, size, true).Delete();
+ Memory<HYPRE_BigInt>(A_col_map_offd, size, true).Delete();
}
void HypreParMatrix::Destroy()
@@ -2503,11 +2577,14 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
// hypre_ParCSRMatrixRAPKT
}
#else
+#if MFEM_HYPRE_VERSION <= 22200
HYPRE_Int P_owns_its_col_starts =
hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
+#endif
hypre_BoomerAMGBuildCoarseOperator(*P,*A,*P,&rap);
+#if MFEM_HYPRE_VERSION <= 22200
/* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
from P (even if it does not own them)! */
hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2516,6 +2593,7 @@ HypreParMatrix * RAP(const HypreParMatrix *A, const HypreParMatrix *P)
{
hypre_ParCSRMatrixSetColStartsOwner(*P, 1);
}
+#endif
#endif
hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -2536,13 +2614,16 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
hypre_ParCSRMatrixDestroy(Q);
}
#else
+#if MFEM_HYPRE_VERSION <= 22200
HYPRE_Int P_owns_its_col_starts =
hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*P));
HYPRE_Int Rt_owns_its_col_starts =
hypre_ParCSRMatrixOwnsColStarts((hypre_ParCSRMatrix*)(*Rt));
+#endif
hypre_BoomerAMGBuildCoarseOperator(*Rt,*A,*P,&rap);
+#if MFEM_HYPRE_VERSION <= 22200
/* Warning: hypre_BoomerAMGBuildCoarseOperator steals the col_starts
from Rt and P (even if they do not own them)! */
hypre_ParCSRMatrixSetRowStartsOwner(rap,0);
@@ -2555,6 +2636,7 @@ HypreParMatrix * RAP(const HypreParMatrix * Rt, const HypreParMatrix *A,
{
hypre_ParCSRMatrixSetColStartsOwner(*Rt, 1);
}
+#endif
#endif
hypre_ParCSRMatrixSetNumNonzeros(rap);
@@ -3144,8 +3226,12 @@ void HypreSmoother::SetOperator(const Operator &op)
}
else
{
+#if MFEM_HYPRE_VERSION <= 22200
min_eig_est = 0;
hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
+#else
+ hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
+#endif
}
Z = new HypreParVector(*A);
}
@@ -3159,8 +3245,12 @@ void HypreSmoother::SetOperator(const Operator &op)
}
else
{
+#if MFEM_HYPRE_VERSION <= 22200
min_eig_est = 0;
hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est);
+#else
+ hypre_ParCSRMaxEigEstimate(*A, poly_scale, &max_eig_est, &min_eig_est);
+#endif
}
// The Taubin and FIR polynomials are defined on [0, 2]
@@ -4472,9 +4562,8 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes)
// HYPRE_BoomerAMGSetDofFunc as in the following code.
if (order_bynodes)
{
- // hypre actually deletes the following pointer in HYPRE_BoomerAMGDestroy,
- // so we don't need to track it
- HYPRE_Int *mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height);
+ // Generate DofFunc mapping on the host
+ HYPRE_Int *h_mapping = mfem_hypre_CTAlloc_host(HYPRE_Int, height);
int h_nnodes = height / dim; // nodes owned in linear algebra (not fem)
MFEM_VERIFY(height % dim == 0, "Ordering does not work as claimed!");
int k = 0;
@@ -4482,9 +4571,24 @@ void HypreBoomerAMG::SetSystemsOptions(int dim, bool order_bynodes)
{
for (int j = 0; j < h_nnodes; ++j)
{
- mapping[k++] = i;
+ h_mapping[k++] = i;
}
}
+
+ // After the addition of hypre_IntArray, mapping is assumed
+ // to be a device pointer. Previously, it was assumed to be
+ // a host pointer.
+#if defined(hypre_IntArrayData) && defined(HYPRE_USING_GPU)
+ HYPRE_Int *mapping = mfem_hypre_CTAlloc(HYPRE_Int, height);
+ hypre_TMemcpy(mapping, h_mapping, HYPRE_Int, height,
+ HYPRE_MEMORY_DEVICE, HYPRE_MEMORY_HOST);
+ mfem_hypre_TFree_host(h_mapping);
+#else
+ HYPRE_Int *mapping = h_mapping;
+#endif
+
+ // hypre actually deletes the mapping pointer in HYPRE_BoomerAMGDestroy,
+ // so we don't need to track it
HYPRE_BoomerAMGSetDofFunc(amg_precond, mapping);
}
@@ -4529,7 +4633,8 @@ void HypreBoomerAMG::RecomputeRBMs()
rbms.SetSize(nrbms);
gf_rbms.SetSize(nrbms);
- gf_rbms[0] = rbms_rxy.ParallelAverage();
+ gf_rbms[0] = fespace->NewTrueDofVector();
+ rbms_rxy.GetTrueDofs(*gf_rbms[0]);
}
else if (dim == 3)
{
@@ -4548,9 +4653,12 @@ void HypreBoomerAMG::RecomputeRBMs()
rbms.SetSize(nrbms);
gf_rbms.SetSize(nrbms);
- gf_rbms[0] = rbms_rxy.ParallelAverage();
- gf_rbms[1] = rbms_ryz.ParallelAverage();
- gf_rbms[2] = rbms_rzx.ParallelAverage();
+ gf_rbms[0] = fespace->NewTrueDofVector();
+ gf_rbms[1] = fespace->NewTrueDofVector();
+ gf_rbms[2] = fespace->NewTrueDofVector();
+ rbms_rxy.GetTrueDofs(*gf_rbms[0]);
+ rbms_ryz.GetTrueDofs(*gf_rbms[1]);
+ rbms_rzx.GetTrueDofs(*gf_rbms[2]);
}
else
{
@@ -4997,11 +5105,11 @@ HypreADS::HypreADS(const HypreParMatrix &A, ParFiniteElementSpace *face_fespace)
void HypreADS::Init(ParFiniteElementSpace *face_fespace)
{
int cycle_type = 11;
- int rlx_type = 2;
int rlx_sweeps = 1;
double rlx_weight = 1.0;
double rlx_omega = 1.0;
#ifndef HYPRE_USING_CUDA
+ int rlx_type = 2;
int amg_coarsen_type = 10;
int amg_agg_levels = 1;
int amg_rlx_type = 8;
@@ -5009,6 +5117,7 @@ void HypreADS::Init(ParFiniteElementSpace *face_fespace)
int amg_interp_type = 6;
int amg_Pmax = 4;
#else
+ int rlx_type = 1;
int amg_coarsen_type = 8;
int amg_agg_levels = 0;
int amg_rlx_type = 18;
@@ -5592,16 +5701,18 @@ HypreLOBPCG::OperatorMatvec( void *matvec_data,
Operator *Aop = (Operator*)A;
- int width = Aop->Width();
-
hypre_ParVector * xPar = (hypre_ParVector *)x;
hypre_ParVector * yPar = (hypre_ParVector *)y;
- Vector xVec(xPar->local_vector->data, width);
- Vector yVec(yPar->local_vector->data, width);
+ HypreParVector xVec(xPar);
+ HypreParVector yVec(yPar);
Aop->Mult( xVec, yVec );
+ // Move data back to hypre's device memory location in case the above Mult
+ // operation moved it to host.
+ yVec.HypreReadWrite();
+
return 0;
}
@@ -5617,19 +5728,20 @@ HypreLOBPCG::PrecondSolve(void *solver,
void *b,
void *x)
{
- Solver *PC = (Solver*)solver;
- Operator *OP = (Operator*)A;
-
- int width = OP->Width();
+ Solver *PC = (Solver*)solver;
hypre_ParVector * bPar = (hypre_ParVector *)b;
hypre_ParVector * xPar = (hypre_ParVector *)x;
- Vector bVec(bPar->local_vector->data, width);
- Vector xVec(xPar->local_vector->data, width);
+ HypreParVector bVec(bPar);
+ HypreParVector xVec(xPar);
PC->Mult( bVec, xVec );
+ // Move data back to hypre's device memory location in case the above Mult
+ // operation moved it to host.
+ xVec.HypreReadWrite();
+
return 0;
}
diff --git a/linalg/hypre_parcsr.cpp b/linalg/hypre_parcsr.cpp
index 5f1f50f3f..1c1ba1640 100644
--- a/linalg/hypre_parcsr.cpp
+++ b/linalg/hypre_parcsr.cpp
@@ -508,8 +508,10 @@ void hypre_ParCSRMatrixEliminateAAe(hypre_ParCSRMatrix *A,
hypre_ParCSRMatrixColStarts(A),
0, 0, 0);
+#if MFEM_HYPRE_VERSION <= 22200
hypre_ParCSRMatrixSetRowStartsOwner(*Ae, 0);
hypre_ParCSRMatrixSetColStartsOwner(*Ae, 0);
+#endif
hypre_CSRMatrix *Ae_diag = hypre_ParCSRMatrixDiag(*Ae);
hypre_CSRMatrix *Ae_offd = hypre_ParCSRMatrixOffd(*Ae);
@@ -1002,10 +1004,17 @@ void hypre_ParCSRMatrixSplit(hypre_ParCSRMatrix *A,
hypre_ParCSRMatrixOwnsData(blocks[i]) = 1;
+#if MFEM_HYPRE_VERSION <= 22200
/* only the first block will own the row/col_starts */
hypre_ParCSRMatrixOwnsRowStarts(blocks[i]) = !i;
hypre_ParCSRMatrixOwnsColStarts(blocks[i]) = !i;
+#endif
}
+
+#if MFEM_HYPRE_VERSION > 22200
+ mfem_hypre_TFree_host(row_starts);
+ mfem_hypre_TFree_host(col_starts);
+#endif
}
/* Based on hypre_CSRMatrixMatvec in hypre's csr_matvec.c */
@@ -1916,9 +1925,12 @@ hypre_ParCSRMatrixAdd(hypre_ParCSRMatrix *A,
/* C owns diag, offd, and cmap. */
hypre_ParCSRMatrixSetDataOwner(C, 1);
+
+#if MFEM_HYPRE_VERSION <= 22200
/* C does not own row and column starts. */
hypre_ParCSRMatrixSetRowStartsOwner(C, 0);
hypre_ParCSRMatrixSetColStartsOwner(C, 0);
+#endif
return C;
}

3
var/spack/repos/builtin/packages/mfem/package.py
View File

@ -200,7 +200,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
depends_on('mpi', when='+mpi')
depends_on('hypre@2.10.0:2.13', when='@:3.3+mpi')
depends_on('hypre@:2.20.0', when='@3.4:4.2+mpi')
depends_on('hypre@:2.22.0', when='@4.3.0+mpi')
depends_on('hypre@:2.23.0', when='@4.3.0+mpi')
depends_on('hypre', when='+mpi')
depends_on('metis', when='+metis')
@ -300,6 +300,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
patch('mfem-4.2-umpire.patch', when='@4.2.0+umpire')
patch('mfem-4.2-slepc.patch', when='@4.2.0+slepc')
patch('mfem-4.2-petsc-3.15.0.patch', when='@4.2.0+petsc ^petsc@3.15.0:')
patch('mfem-4.3-hypre-2.23.0.patch', when='@4.3.0')
# Patch to fix MFEM makefile syntax error. See
# https://github.com/mfem/mfem/issues/1042 for the bug report and