Merge branch 'develop' into packages/mfem-4.8

for building mfem

share/spack/gitlab/cloud_pipelines/stacks/radiuss/spack.yaml

@@ -5,11 +5,17 @@ spack:
       target: [ "x86_64_v3" ]
       require:
       - target=x86_64_v3
-      - '%gcc@7.5.0'
+      # prefer %gcc@7.5.0 but also allow %gcc@12
+      - one_of: ['%gcc@7.5.0', '%gcc@12']
 
       providers:
         mpi: [mvapich2]
 
+    mfem:
+      require:
+      # gcc@7.5.0 crashes when building mfem@4.8.0, so use gcc@12
+      - '%gcc@12'
+
   specs:
   - ascent  # ^conduit@0.6.0
   - axom

var/spack/repos/spack_repo/builtin/packages/mfem/mfem-4.8-nvcc-c++17.patch

@@ -0,0 +1,129 @@
+diff --git a/miniapps/dpg/convection-diffusion.cpp b/miniapps/dpg/convection-diffusion.cpp
+index 7659e52745..170f8f1c5e 100644
+--- a/miniapps/dpg/convection-diffusion.cpp
++++ b/miniapps/dpg/convection-diffusion.cpp
+@@ -79,7 +79,7 @@ enum prob_type
+ };
+ 
+ prob_type prob;
+-Vector beta;
++Vector beta_glob;
+ real_t epsilon;
+ 
+ real_t exact_u(const Vector & X);
+@@ -120,7 +120,7 @@ int main(int argc, char *argv[])
+                   "Theta parameter for AMR");
+    args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                   " 0: manufactured, 1: Erickson-Johnson ");
+-   args.AddOption(&beta, "-beta", "--beta",
++   args.AddOption(&beta_glob, "-beta", "--beta",
+                   "Vector Coefficient beta");
+    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                   "--no-static-condensation", "Enable static condensation.");
+@@ -147,11 +147,11 @@ int main(int argc, char *argv[])
+    int dim = mesh.Dimension();
+    MFEM_VERIFY(dim > 1, "Dimension = 1 is not supported in this example");
+ 
+-   if (beta.Size() == 0)
++   if (beta_glob.Size() == 0)
+    {
+-      beta.SetSize(dim);
+-      beta = 0.0;
+-      beta[0] = 1.;
++      beta_glob.SetSize(dim);
++      beta_glob = 0.0;
++      beta_glob[0] = 1.;
+    }
+ 
+    args.PrintOptions(std::cout);
+@@ -199,10 +199,10 @@ int main(int argc, char *argv[])
+    ConstantCoefficient eps2(1/(epsilon*epsilon));
+ 
+    ConstantCoefficient negeps(-epsilon);
+-   VectorConstantCoefficient betacoeff(beta);
+-   Vector negbeta = beta; negbeta.Neg();
+-   DenseMatrix bbt(beta.Size());
+-   MultVVt(beta, bbt);
++   VectorConstantCoefficient betacoeff(beta_glob);
++   Vector negbeta = beta_glob; negbeta.Neg();
++   DenseMatrix bbt(beta_glob.Size());
++   MultVVt(beta_glob, bbt);
+    MatrixConstantCoefficient bbtcoeff(bbt);
+    VectorConstantCoefficient negbetacoeff(negbeta);
+ 
+@@ -598,7 +598,7 @@ void exact_hatf(const Vector & X, Vector & hatf)
+    hatf.SetSize(X.Size());
+    for (int i = 0; i<hatf.Size(); i++)
+    {
+-      hatf[i] = beta[i] * u - sigma[i];
++      hatf[i] = beta_glob[i] * u - sigma[i];
+    }
+ }
+ 
+@@ -612,7 +612,7 @@ real_t f_exact(const Vector & X)
+    real_t s = 0;
+    for (int i = 0; i<du.Size(); i++)
+    {
+-      s += beta[i] * du[i];
++      s += beta_glob[i] * du[i];
+    }
+    return -epsilon * d2u + s;
+ }
+diff --git a/miniapps/dpg/pconvection-diffusion.cpp b/miniapps/dpg/pconvection-diffusion.cpp
+index 64d75d8d10..062f07d0f1 100644
+--- a/miniapps/dpg/pconvection-diffusion.cpp
++++ b/miniapps/dpg/pconvection-diffusion.cpp
+@@ -91,7 +91,7 @@ static const char *enum_str[] =
+ };
+ 
+ prob_type prob;
+-Vector beta;
++Vector beta_glob;
+ real_t epsilon;
+ 
+ real_t exact_u(const Vector & X);
+@@ -141,7 +141,7 @@ int main(int argc, char *argv[])
+                   "Theta parameter for AMR");
+    args.AddOption(&iprob, "-prob", "--problem", "Problem case"
+                   " 0: lshape, 1: General");
+-   args.AddOption(&beta, "-beta", "--beta",
++   args.AddOption(&beta_glob, "-beta", "--beta",
+                   "Vector Coefficient beta");
+    args.AddOption(&static_cond, "-sc", "--static-condensation", "-no-sc",
+                   "--no-static-condensation", "Enable static condensation.");
+@@ -181,19 +181,19 @@ int main(int argc, char *argv[])
+       case sinusoidal:
+       case EJ:
+       {
+-         if (beta.Size() == 0)
++         if (beta_glob.Size() == 0)
+          {
+-            beta.SetSize(dim);
+-            beta = 0.0;
+-            beta[0] = 1.;
++            beta_glob.SetSize(dim);
++            beta_glob = 0.0;
++            beta_glob[0] = 1.;
+          }
+          break;
+       }
+       case bdr_layer:
+       {
+-         beta.SetSize(dim);
+-         beta[0] = 1.;
+-         beta[1] = 2.;
++         beta_glob.SetSize(dim);
++         beta_glob[0] = 1.;
++         beta_glob[1] = 2.;
+          exact_known = false;
+       }
+       break;
+@@ -846,7 +846,7 @@ void beta_function(const Vector & X, Vector & beta_val)
+    }
+    else
+    {
+-      beta_val = beta;
++      beta_val = beta_glob;
+    }
+ }
+ 

var/spack/repos/spack_repo/builtin/packages/mfem/package.py

@@ -49,6 +49,13 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     # other version.
     version("develop", branch="master")
 
+    version(
+        "4.8.0",
+        sha256="49bd2a076b0d87863092cb55f8524b5292d9afb2e48c19f80222ada367819016",
+        url="https://bit.ly/mfem-4-8",
+        extension="tar.gz",
+    )
+
     version(
         "4.7.0",
         sha256="5e889493f5f79848f7b2d16afaae307c59880ac2a7ff2315551c60ca54717751",
@@ -282,7 +289,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
 
     depends_on("mpi", when="+mpi")
     depends_on("hipsparse", when="@4.4.0:+rocm")
-    depends_on("hipblas", when="@4.4.0:+rocm")
+    depends_on("hipblas", when="@4.8.0:+rocm")
 
     with when("+mpi"):
         depends_on("hypre")
@@ -332,6 +339,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("gslib~mpi~mpiio", when="+gslib~mpi")
     depends_on("gslib@1.0.5:1.0.6", when="@:4.2+gslib")
     depends_on("gslib@1.0.7:", when="@4.3.0:+gslib")
+    depends_on("gslib@1.0.9:", when="@4.8.0:+gslib")
     depends_on("suite-sparse", when="+suite-sparse")
     depends_on("superlu-dist", when="+superlu-dist")
     # If superlu-dist is built with +cuda, propagate cuda_arch
@@ -386,8 +394,10 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     depends_on("conduit@0.3.1:,master:", when="+conduit")
     depends_on("conduit+mpi", when="+conduit+mpi")
     depends_on("libfms@0.2.0:", when="+fms")
-    depends_on("ginkgo@1.4.0:", when="+ginkgo")
+    depends_on("ginkgo@1.4.0:1.8", when="@:4.7+ginkgo")
+    depends_on("ginkgo@1.9.0:", when="@4.8:+ginkgo")
     conflicts("cxxstd=11", when="^ginkgo")
+    conflicts("cxxstd=14", when="^ginkgo@1.9:")
     for sm_ in CudaPackage.cuda_arch_values:
         depends_on(
             "ginkgo+cuda cuda_arch={0}".format(sm_), when="+ginkgo+cuda cuda_arch={0}".format(sm_)
@@ -513,6 +523,7 @@ class Mfem(Package, CudaPackage, ROCmPackage):
     )
     patch("mfem-4.7.patch", when="@4.7.0")
     patch("mfem-4.7-sundials-7.patch", when="@4.7.0+sundials ^sundials@7:")
+    patch("mfem-4.8-nvcc-c++17.patch", when="@4.8.0+cuda")
 
     phases = ["configure", "build", "install"]
 
@@ -569,7 +580,7 @@ def find_optional_library(name, prefix):
                 mfem_mpiexec = "jsrun"
                 mfem_mpiexec_np = "-p"
         elif "FLUX_EXEC_PATH" in os.environ:
-            mfem_mpiexec = "flux run"
+            mfem_mpiexec = "flux run -x -N 1"
             mfem_mpiexec_np = "-n"
         elif "PBS_JOBID" in os.environ:
             mfem_mpiexec = "mpiexec"
@@ -636,11 +647,13 @@ def find_optional_library(name, prefix):
             cxxstd = "14"
         if self.spec.satisfies("^sundials@6.4.0:"):
             cxxstd = "14"
-        if self.spec.satisfies("^ginkgo"):
-            cxxstd = "14"
         # When rocPRIM is used (e.g. by PETSc + ROCm) we need C++14:
         if self.spec.satisfies("^rocprim@5.5.0:"):
             cxxstd = "14"
+        if self.spec.satisfies("^ginkgo@1.4.0:1.8"):
+            cxxstd = "14"
+        if self.spec.satisfies("^ginkgo@1.9.0:"):
+            cxxstd = "17"
         cxxstd_req = spec.variants["cxxstd"].value
         if cxxstd_req != "auto":
             # Constraints for valid standard level should be imposed during
@@ -709,7 +722,7 @@ def find_optional_library(name, prefix):
 
             hypre_gpu_libs = ""
             if "+cuda" in hypre:
-                hypre_gpu_libs = " -lcusparse -lcurand -lcublas"
+                hypre_gpu_libs = " -lcusolver -lcusparse -lcurand -lcublas"
             elif "+rocm" in hypre:
                 hypre_rocm_libs = LibraryList([])
                 if "^rocsparse" in hypre:
@@ -849,7 +862,7 @@ def find_optional_library(name, prefix):
             ]
 
         if "+pumi" in spec:
-            pumi_libs = [
+            pumi_libs_names = [
                 "pumi",
                 "crv",
                 "ma",
@@ -863,6 +876,11 @@ def find_optional_library(name, prefix):
                 "apf_zoltan",
                 "spr",
             ]
+            pumi_libs_names = ["lib" + name for name in pumi_libs_names]
+            pumi = spec["pumi"]
+            pumi_libs = find_libraries(
+                pumi_libs_names, pumi.prefix, shared=("+shared" in pumi), recursive=True
+            )
             pumi_dep_zoltan = ""
             pumi_dep_parmetis = ""
             if "+zoltan" in spec["pumi"]:
@@ -874,11 +892,7 @@ def find_optional_library(name, prefix):
             options += [
                 "PUMI_OPT=-I%s" % spec["pumi"].prefix.include,
                 "PUMI_LIB=%s %s %s"
-                % (
+                % (ld_flags_from_library_list(pumi_libs), pumi_dep_zoltan, pumi_dep_parmetis),
-                    ld_flags_from_dirs([spec["pumi"].prefix.lib], pumi_libs),
-                    pumi_dep_zoltan,
-                    pumi_dep_parmetis,
-                ),
             ]
 
         if "+gslib" in spec:
@@ -987,9 +1001,10 @@ def find_optional_library(name, prefix):
             if "^rocprim" in spec and not spec["hip"].external:
                 # rocthrust [via petsc+rocm] has a dependency on rocprim
                 hip_headers += spec["rocprim"].headers
-            if "^hipblas" in spec:
+            if "^hipblas" in spec:  # hipblas is needed @4.8.0:+rocm
+                # note: superlu-dist+rocm needs the hipblas header path too
                 hipblas = spec["hipblas"]
-                hip_headers += hipblas.headers
+                hip_headers += self.all_headers(hipblas)
                 hip_libs += hipblas.libs
             if "%cce" in spec:
                 # We assume the proper Cray CCE module (cce) is loaded:
@@ -1013,6 +1028,22 @@ def find_optional_library(name, prefix):
                 hip_libs += find_libraries(craylibs, craylibs_path)
                 craylibs_path2 = join_path(craylibs_path, "../../../cce-clang", proc, "lib")
                 hip_libs += find_libraries("libunwind", craylibs_path2)
+            elif spec.satisfies("%rocmcc ^cray-mpich"):
+                # The AMD version of cray-mpich, libmpi_amd.so, needs the rpath
+                # to libflang.so (also needed for libpgmath.so and others).
+                rocmcc_bin_dir = os.path.dirname(env["SPACK_CXX"])
+                rocmcc_prefix = os.path.dirname(rocmcc_bin_dir)
+                rocmcc_libflang = find_libraries(
+                    "libflang", join_path(rocmcc_prefix, "lib/llvm/lib"), recursive=False
+                )
+                hip_libs += rocmcc_libflang
+            if spec.satisfies("^cray-mpich"):
+                # The cray-mpich library, libmpi_*.so, needs the rpath to
+                # libpmi.so.0 and libpmi2.so.0 if that path is not configured
+                # properly on system level.
+                libpmi_lib = find_libraries("libpmi", "/opt/cray/pe/lib64")
+                if libpmi_lib:
+                    hip_libs += libpmi_lib
 
             if hip_headers:
                 options += ["HIP_OPT=%s" % hip_headers.cpp_flags]
@@ -1377,3 +1408,12 @@ def ld_flags_from_dirs(self, pkg_dirs_list, pkg_libs_list):
         flags += ["-L%s" % dir for dir in pkg_dirs_list if not self.is_sys_lib_path(dir)]
         flags += ["-l%s" % lib for lib in pkg_libs_list]
         return " ".join(flags)
+
+    def all_headers(self, root_spec):
+        all_hdrs = HeaderList([])
+        for dep in root_spec.traverse(deptype="link"):
+            try:
+                all_hdrs += root_spec[dep.name].headers
+            except NoHeadersError:
+                pass
+        return all_hdrs

var/spack/repos/spack_repo/builtin/packages/petsc/package.py

@@ -264,6 +264,9 @@ class Petsc(Package, CudaPackage, ROCmPackage):
     variant("hwloc", default=False, description="Activates support for hwloc")
     variant("kokkos", default=False, description="Activates support for kokkos and kokkos-kernels")
     variant("fortran", default=True, description="Activates fortran support")
+    variant(
+        "fortran-bindings", default=True, when="+fortran", description="Activates fortran bindings"
+    )
 
     with when("+rocm"):
         # https://github.com/spack/spack/issues/37416
@@ -499,6 +502,8 @@ def mpi_dependent_options(self):
             ]
             if "+fortran" in self.spec:
                 compiler_opts.append("--with-fc=%s" % os.environ["FC"])
+                fb = "1" if self.spec.satisfies("+fortran-bindings") else "0"
+                compiler_opts.append(f"--with-fortran-bindings={fb}")
             else:
                 compiler_opts.append("--with-fc=0")
         else:
@@ -508,6 +513,8 @@ def mpi_dependent_options(self):
             ]
             if "+fortran" in self.spec:
                 compiler_opts.append("--with-fc=%s" % self.spec["mpi"].mpifc)
+                fb = "1" if self.spec.satisfies("+fortran-bindings") else "0"
+                compiler_opts.append(f"--with-fortran-bindings={fb}")
             else:
                 compiler_opts.append("--with-fc=0")
             if self.spec.satisfies("%intel"):

var/spack/repos/spack_repo/builtin/packages/spectrum_mpi/package.py

@@ -132,10 +132,13 @@ def setup_dependent_build_environment(
             env.set("MPIF90", os.path.join(self.prefix.bin, "mpif90"))
 
         dependent_module = dependent_spec.package.module
-        env.set("OMPI_CC", dependent_module.spack_cc)
+        if dependent_spec.satisfies("^c"):
-        env.set("OMPI_CXX", dependent_module.spack_cxx)
+            env.set("OMPI_CC", dependent_module.spack_cc)
-        env.set("OMPI_FC", dependent_module.spack_fc)
+        if dependent_spec.satisfies("^cxx"):
-        env.set("OMPI_F77", dependent_module.spack_f77)
+            env.set("OMPI_CXX", dependent_module.spack_cxx)
+        if dependent_spec.satisfies("^fortran"):
+            env.set("OMPI_FC", dependent_module.spack_fc)
+            env.set("OMPI_F77", dependent_module.spack_f77)
         env.prepend_path("LD_LIBRARY_PATH", self.prefix.lib)
 
     def setup_run_environment(self, env: EnvironmentModifications) -> None:

var/spack/repos/spack_repo/builtin/packages/sundials/package.py

@@ -405,7 +405,7 @@ def cmake_args(self):
         if "+rocm" in spec:
             args.extend(
                 [
-                    define("CMAKE_C_COMPILER", spec["llvm-amdgpu"].prefix.bin.clang),
+                    define("CMAKE_C_COMPILER", spec["llvm-amdgpu"].prefix.bin.amdclang),
                     define("CMAKE_CXX_COMPILER", spec["hip"].hipcc),
                     define("HIP_PATH", spec["hip"].prefix),
                     define("HIP_CLANG_INCLUDE_PATH", spec["llvm-amdgpu"].prefix.include),