From 1c0c73a404aa09c70dd403867bfaf71c0283a963 Mon Sep 17 00:00:00 2001
From: Veselin Dobrev <dobrev@llnl.gov>
Date: Sun, 4 May 2025 03:04:10 -0700
Subject: [PATCH] [mfem] Updates for v4.8

[magma] Add new versions for the rocm-core dependency

[spectrum-mpi] Do not query languages that are not used by a dependent package

[sundials] Fix llvm-amdgpu's C compiler: clang -> amdclang
---
 .../repos/builtin/packages/magma/package.py   |   5 +
 .../packages/mfem/mfem-4.8-nvcc-c++17.patch   | 129 ++++++++++++++++++
 .../repos/builtin/packages/mfem/package.py    |  38 ++++--
 .../builtin/packages/spectrum-mpi/package.py  |  11 +-
 .../builtin/packages/sundials/package.py      |   2 +-
 5 files changed, 172 insertions(+), 13 deletions(-)
 create mode 100644 var/spack/repos/builtin/packages/mfem/mfem-4.8-nvcc-c++17.patch

diff --git a/var/spack/repos/builtin/packages/magma/package.py b/var/spack/repos/builtin/packages/magma/package.py
index a937c710ad9..6a684a0e40c 100644
--- a/var/spack/repos/builtin/packages/magma/package.py
+++ b/var/spack/repos/builtin/packages/magma/package.py
@@ -69,6 +69,11 @@ class Magma(CMakePackage, CudaPackage, ROCmPackage):
         "6.1.2",
         "6.2.0",
         "6.2.1",
+        "6.2.4",
+        "6.3.0",
+        "6.3.1",
+        "6.3.2",
+        "6.3.3",
     ]:
         depends_on(f"rocm-core@{ver}", when=f"@2.8.0: +rocm ^hip@{ver}")
     depends_on("python", when="@master", type="build")
diff --git a/var/spack/repos/builtin/packages/mfem/mfem-4.8-nvcc-c++17.patch b/var/spack/repos/builtin/packages/mfem/mfem-4.8-nvcc-c++17.patch
new file mode 100644
index 00000000000..db6e104aa54
--- /dev/null
+++ b/var/spack/repos/builtin/packages/mfem/mfem-4.8-nvcc-c++17.patch
@@ -0,0 +1,129 @@
+diff --git a/miniapps/dpg/convection-diffusion.cpp b/miniapps/dpg/convection-diffusion.cpp
+index 7659e52745..170f8f1c5e 100644
+--- a/miniapps/dpg/convection-diffusion.cpp
++++ b/miniapps/dpg/convection-diffusion.cpp
+@@ -79,7 +79,7 @@ enum prob_type
+ };
+ 
+ prob_type prob;
+-Vector beta;
++Vector beta_glob;
+ real_t epsilon;
+ 
+ real_t exact_u(const Vector & X);
+@@ -120,7 +120,7 @@ int main(int argc, char *argv[])
+                   "Theta parameter for AMR");
+    args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                   " 0: manufactured, 1: Erickson-Johnson ");
+-   args.AddOption(&beta, "-beta", "--beta",
++   args.AddOption(&beta_glob, "-beta", "--beta",
+                   "Vector Coefficient beta");
+    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                   "--no-static-condensation", "Enable static condensation.");
+@@ -147,11 +147,11 @@ int main(int argc, char *argv[])
+    int dim = mesh.Dimension();
+    MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+ 
+-   if (beta.Size() == 0)
++   if (beta_glob.Size() == 0)
+    {
+-      beta.SetSize(dim);
+-      beta = 0.0;
+-      beta[0] = 1.;
++      beta_glob.SetSize(dim);
++      beta_glob = 0.0;
++      beta_glob[0] = 1.;
+    }
+ 
+    args.PrintOptions(std::cout);
+@@ -199,10 +199,10 @@ int main(int argc, char *argv[])
+    ConstantCoefficient eps2(1/(epsilon*epsilon));
+ 
+    ConstantCoefficient negeps(-epsilon);
+-   VectorConstantCoefficient betacoeff(beta);
+-   Vector negbeta = beta; negbeta.Neg();
+-   DenseMatrix bbt(beta.Size());
+-   MultVVt(beta, bbt);
++   VectorConstantCoefficient betacoeff(beta_glob);
++   Vector negbeta = beta_glob; negbeta.Neg();
++   DenseMatrix bbt(beta_glob.Size());
++   MultVVt(beta_glob, bbt);
+    MatrixConstantCoefficient bbtcoeff(bbt);
+    VectorConstantCoefficient negbetacoeff(negbeta);
+ 
+@@ -598,7 +598,7 @@ void exact_hatf(const Vector & X, Vector & hatf)
+    hatf.SetSize(X.Size());
+    for (int i = 0; i<hatf.Size(); i++)
+    {
+-      hatf[i] = beta[i] * u - sigma[i];
++      hatf[i] = beta_glob[i] * u - sigma[i];
+    }
+ }
+ 
+@@ -612,7 +612,7 @@ real_t f_exact(const Vector & X)
+    real_t s = 0;
+    for (int i = 0; i<du.Size(); i++)
+    {
+-      s += beta[i] * du[i];
++      s += beta_glob[i] * du[i];
+    }
+    return -epsilon * d2u + s;
+ }
+diff --git a/miniapps/dpg/pconvection-diffusion.cpp b/miniapps/dpg/pconvection-diffusion.cpp
+index 64d75d8d10..062f07d0f1 100644
+--- a/miniapps/dpg/pconvection-diffusion.cpp
++++ b/miniapps/dpg/pconvection-diffusion.cpp
+@@ -91,7 +91,7 @@ static const char *enum_str[] =
+ };
+ 
+ prob_type prob;
+-Vector beta;
++Vector beta_glob;
+ real_t epsilon;
+ 
+ real_t exact_u(const Vector & X);
+@@ -141,7 +141,7 @@ int main(int argc, char *argv[])
+                   "Theta parameter for AMR");
+    args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                   " 0: lshape, 1: General");
+-   args.AddOption(&beta, "-beta", "--beta",
++   args.AddOption(&beta_glob, "-beta", "--beta",
+                   "Vector Coefficient beta");
+    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                   "--no-static-condensation", "Enable static condensation.");
+@@ -181,19 +181,19 @@ int main(int argc, char *argv[])
+       case sinusoidal:
+       case EJ:
+       {
+-         if (beta.Size() == 0)
++         if (beta_glob.Size() == 0)
+          {
+-            beta.SetSize(dim);
+-            beta = 0.0;
+-            beta[0] = 1.;
++            beta_glob.SetSize(dim);
++            beta_glob = 0.0;
++            beta_glob[0] = 1.;
+          }
+          break;
+       }
+       case bdr_layer:
+       {
+-         beta.SetSize(dim);
+-         beta[0] = 1.;
+-         beta[1] = 2.;
++         beta_glob.SetSize(dim);
++         beta_glob[0] = 1.;
++         beta_glob[1] = 2.;
+          exact_known = false;
+       }
+       break;
+@@ -846,7 +846,7 @@ void beta_function(const Vector & X, Vector & beta_val)
+    }
+    else
+    {
+-      beta_val = beta;
++      beta_val = beta_glob;
+    }
+ }
+ 
diff --git a/var/spack/repos/builtin/packages/mfem/package.py b/var/spack/repos/builtin/packages/mfem/package.py
index 92ef28f5438..877de433dfa 100644
--- a/var/spack/repos/builtin/packages/mfem/package.py
+++ b/var/spack/repos/builtin/packages/mfem/package.py
@@ -339,6 +339,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("gslib~mpi~mpiio", when="+gslib~mpi")
     depends_on("gslib@1.0.5:1.0.6", when="@:4.2+gslib")
     depends_on("gslib@1.0.7:", when="@4.3.0:+gslib")
+    depends_on("gslib@1.0.9:", when="@4.8.0:+gslib")
     depends_on("suite-sparse", when="+suite-sparse")
     depends_on("superlu-dist", when="+superlu-dist")
     # If superlu-dist is built with +cuda, propagate cuda_arch
@@ -393,8 +394,10 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("conduit@0.3.1:,master:", when="+conduit")
     depends_on("conduit+mpi", when="+conduit+mpi")
     depends_on("libfms@0.2.0:", when="+fms")
-    depends_on("ginkgo@1.4.0:", when="+ginkgo")
+    depends_on("ginkgo@1.4.0:1.8", when="@:4.7+ginkgo")
+    depends_on("ginkgo@1.9.0:", when="@4.8:+ginkgo")
     conflicts("cxxstd=11", when="^ginkgo")
+    conflicts("cxxstd=14", when="^ginkgo@1.9:")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "ginkgo+cuda cuda_arch={0}".format(sm_), when="+ginkgo+cuda cuda_arch={0}".format(sm_)
@@ -520,6 +523,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     )
     patch("mfem-4.7.patch", when="@4.7.0")
     patch("mfem-4.7-sundials-7.patch", when="@4.7.0+sundials ^sundials@7:")
+    patch("mfem-4.8-nvcc-c++17.patch", when="@4.8.0+cuda")
 
     phases = ["configure", "build", "install"]
 
@@ -576,7 +580,7 @@ def find_optional_library(name, prefix):
                 mfem_mpiexec = "jsrun"
                 mfem_mpiexec_np = "-p"
         elif "FLUX_EXEC_PATH" in os.environ:
-            mfem_mpiexec = "flux run"
+            mfem_mpiexec = "flux run -x -N 1"
             mfem_mpiexec_np = "-n"
         elif "PBS_JOBID" in os.environ:
             mfem_mpiexec = "mpiexec"
@@ -643,11 +647,13 @@ def find_optional_library(name, prefix):
             cxxstd = "14"
         if self.spec.satisfies("^sundials@6.4.0:"):
             cxxstd = "14"
-        if self.spec.satisfies("^ginkgo"):
-            cxxstd = "14"
         # When rocPRIM is used (e.g. by PETSc + ROCm) we need C++14:
         if self.spec.satisfies("^rocprim@5.5.0:"):
             cxxstd = "14"
+        if self.spec.satisfies("^ginkgo@1.4.0:1.8"):
+            cxxstd = "14"
+        if self.spec.satisfies("^ginkgo@1.9.0:"):
+            cxxstd = "17"
         cxxstd_req = spec.variants["cxxstd"].value
         if cxxstd_req != "auto":
             # Constraints for valid standard level should be imposed during
@@ -716,7 +722,7 @@ def find_optional_library(name, prefix):
 
             hypre_gpu_libs = ""
             if "+cuda" in hypre:
-                hypre_gpu_libs = " -lcusparse -lcurand -lcublas"
+                hypre_gpu_libs = " -lcusolver -lcusparse -lcurand -lcublas"
             elif "+rocm" in hypre:
                 hypre_rocm_libs = LibraryList([])
                 if "^rocsparse" in hypre:
@@ -856,7 +862,7 @@ def find_optional_library(name, prefix):
             ]
 
         if "+pumi" in spec:
-            pumi_libs = [
+            pumi_libs_names = [
                 "pumi",
                 "crv",
                 "ma",
@@ -870,6 +876,14 @@ def find_optional_library(name, prefix):
                 "apf_zoltan",
                 "spr",
             ]
+            pumi_libs_names = ["lib" + name for name in pumi_libs_names]
+            pumi = spec["pumi"]
+            pumi_libs = find_libraries(
+                pumi_libs_names,
+                pumi.prefix,
+                shared=("+shared" in pumi),
+                recursive=True,
+            )
             pumi_dep_zoltan = ""
             pumi_dep_parmetis = ""
             if "+zoltan" in spec["pumi"]:
@@ -882,7 +896,7 @@ def find_optional_library(name, prefix):
                 "PUMI_OPT=-I%s" % spec["pumi"].prefix.include,
                 "PUMI_LIB=%s %s %s"
                 % (
-                    ld_flags_from_dirs([spec["pumi"].prefix.lib], pumi_libs),
+                    ld_flags_from_library_list(pumi_libs),
                     pumi_dep_zoltan,
                     pumi_dep_parmetis,
                 ),
@@ -1026,8 +1040,16 @@ def find_optional_library(name, prefix):
                 # to libflang.so (also needed for libpgmath.so and others).
                 rocmcc_bin_dir = os.path.dirname(env["SPACK_CXX"])
                 rocmcc_prefix = os.path.dirname(rocmcc_bin_dir)
-                rocmcc_libflang = find_libraries("libflang", rocmcc_prefix, recursive=True)
+                rocmcc_libflang = find_libraries(
+                    "libflang",
+                    join_path(rocmcc_prefix, "lib/llvm/lib"),
+                    recursive=False,
+                )
                 hip_libs += rocmcc_libflang
+                # The AMD version of cray-mpich, libmpi_amd.so, needs the rpath
+                # to libpmi.so.0
+                libpmi_lib = find_libraries("libpmi", "/opt/cray/pe/lib64")
+                hip_libs += libpmi_lib
 
             if hip_headers:
                 options += ["HIP_OPT=%s" % hip_headers.cpp_flags]
diff --git a/var/spack/repos/builtin/packages/spectrum-mpi/package.py b/var/spack/repos/builtin/packages/spectrum-mpi/package.py
index a5b0871684f..0ef32d9d713 100644
--- a/var/spack/repos/builtin/packages/spectrum-mpi/package.py
+++ b/var/spack/repos/builtin/packages/spectrum-mpi/package.py
@@ -128,10 +128,13 @@ def setup_dependent_build_environment(self, env, dependent_spec):
             env.set("MPIF90", os.path.join(self.prefix.bin, "mpif90"))
 
         dependent_module = dependent_spec.package.module
-        env.set("OMPI_CC", dependent_module.spack_cc)
-        env.set("OMPI_CXX", dependent_module.spack_cxx)
-        env.set("OMPI_FC", dependent_module.spack_fc)
-        env.set("OMPI_F77", dependent_module.spack_f77)
+        if dependent_spec.satisfies("^c"):
+            env.set("OMPI_CC", dependent_module.spack_cc)
+        if dependent_spec.satisfies("^cxx"):
+            env.set("OMPI_CXX", dependent_module.spack_cxx)
+        if dependent_spec.satisfies("^fortran"):
+            env.set("OMPI_FC", dependent_module.spack_fc)
+            env.set("OMPI_F77", dependent_module.spack_f77)
         env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib)
 
     def setup_run_environment(self, env):
diff --git a/var/spack/repos/builtin/packages/sundials/package.py b/var/spack/repos/builtin/packages/sundials/package.py
index 6d3885d4ccf..d4d0705b9ef 100644
--- a/var/spack/repos/builtin/packages/sundials/package.py
+++ b/var/spack/repos/builtin/packages/sundials/package.py
@@ -404,7 +404,7 @@ def cmake_args(self):
         if "+rocm" in spec:
             args.extend(
                 [
-                    define("CMAKE_C_COMPILER", spec["llvm-amdgpu"].prefix.bin.clang),
+                    define("CMAKE_C_COMPILER", spec["llvm-amdgpu"].prefix.bin.amdclang),
                     define("CMAKE_CXX_COMPILER", spec["hip"].hipcc),
                     define("HIP_PATH", spec["hip"].prefix),
                     define("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include),