Cray support: use linux platform for newer craype versions (#29392)

Newer versions of the CrayPE for EX systems have standalone compiler executables for CCE and compiler wrappers for Cray MPICH. With those, we can treat the cray systems as part of the linux platform rather than having a separate cray platform.

This PR:
- [x] Changes cray platform detection to ignore EX systems with Craype version 21.10 or later
- [x] Changes the cce compiler to be detectable via paths
- [x] Changes the spack compiler wrapper to understand the executable names for the standalone cce compiler (`craycc`, `crayCC`, `crayftn`).

lib/spack/env/cc

@@ -241,28 +241,28 @@ case "$command" in
         mode=cpp
         debug_flags="-g"
         ;;
-    cc|c89|c99|gcc|clang|armclang|icc|icx|pgcc|nvc|xlc|xlc_r|fcc|amdclang|cl.exe)
+    cc|c89|c99|gcc|clang|armclang|icc|icx|pgcc|nvc|xlc|xlc_r|fcc|amdclang|cl.exe|craycc)
         command="$SPACK_CC"
         language="C"
         comp="CC"
         lang_flags=C
         debug_flags="-g"
         ;;
-    c++|CC|g++|clang++|armclang++|icpc|icpx|dpcpp|pgc++|nvc++|xlc++|xlc++_r|FCC|amdclang++)
+    c++|CC|g++|clang++|armclang++|icpc|icpx|dpcpp|pgc++|nvc++|xlc++|xlc++_r|FCC|amdclang++|crayCC)
         command="$SPACK_CXX"
         language="C++"
         comp="CXX"
         lang_flags=CXX
         debug_flags="-g"
         ;;
-    ftn|f90|fc|f95|gfortran|flang|armflang|ifort|ifx|pgfortran|nvfortran|xlf90|xlf90_r|nagfor|frt|amdflang)
+    ftn|f90|fc|f95|gfortran|flang|armflang|ifort|ifx|pgfortran|nvfortran|xlf90|xlf90_r|nagfor|frt|amdflang|crayftn)
         command="$SPACK_FC"
         language="Fortran 90"
         comp="FC"
         lang_flags=F
         debug_flags="-g"
         ;;
-    f77|xlf|xlf_r|pgf77|amdflang)
+    f77|xlf|xlf_r|pgf77)
         command="$SPACK_F77"
         language="Fortran 77"
         comp="F77"

lib/spack/env/cce/case-insensitive/crayCC

@@ -0,0 +1 @@
+../../cc

lib/spack/env/cce/craycc

@@ -0,0 +1 @@
+../cc

lib/spack/env/cce/crayftn

@@ -0,0 +1 @@
+../cc

lib/spack/spack/compilers/cce.py

@@ -2,7 +2,6 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
 import os
 
 from spack.compiler import Compiler, UnsupportedCompilerFlag
@@ -12,17 +11,25 @@
 class Cce(Compiler):
     """Cray compiler environment compiler."""
 
+    def __init__(self, *args, **kwargs):
+        super(Cce, self).__init__(*args, **kwargs)
+        # For old cray compilers on module based systems we replace
+        # ``version_argument`` with the old value. Cannot be a property
+        # as the new value is used in classmethods for path-based detection
+        if not self.is_clang_based:
+            self.version_argument = "-V"
+
     # Subclasses use possible names of C compiler
-    cc_names = ["cc"]
+    cc_names = ["craycc", "cc"]
 
     # Subclasses use possible names of C++ compiler
-    cxx_names = ["CC"]
+    cxx_names = ["crayCC", "CC"]
 
     # Subclasses use possible names of Fortran 77 compiler
-    f77_names = ["ftn"]
+    f77_names = ["crayftn", "ftn"]
 
     # Subclasses use possible names of Fortran 90 compiler
-    fc_names = ["ftn"]
+    fc_names = ["crayftn", "ftn"]
 
     # MacPorts builds gcc versions with prefixes and -mp-X.Y suffixes.
     suffixes = [r"-mp-\d\.\d"]
@@ -30,24 +37,30 @@ class Cce(Compiler):
     PrgEnv = "PrgEnv-cray"
     PrgEnv_compiler = "cce"
 
-    link_paths = {
+    @property
-        "cc": os.path.join("cce", "cc"),
+    def link_paths(self):
-        "cxx": os.path.join("cce", "case-insensitive", "CC"),
+        if self.PrgEnv in self.modules:
-        "f77": os.path.join("cce", "ftn"),
+            # Old module-based interface to cray compilers
-        "fc": os.path.join("cce", "ftn"),
+            return {
-    }
+                "cc": os.path.join("cce", "cc"),
+                "cxx": os.path.join("case-insensitive", "CC"),
+                "f77": os.path.join("cce", "ftn"),
+                "fc": os.path.join("cce", "ftn"),
+            }
+
+        return {
+            "cc": os.path.join("cce", "craycc"),
+            "cxx": os.path.join("cce", "case-insensitive", "crayCC"),
+            "f77": os.path.join("cce", "crayftn"),
+            "fc": os.path.join("cce", "crayftn"),
+        }
 
     @property
     def is_clang_based(self):
         version = self._real_version or self.version
         return version >= ver("9.0") and "classic" not in str(version)
 
-    @property
+    version_argument = "--version"
-    def version_argument(self):
-        if self.is_clang_based:
-            return "--version"
-        return "-V"
-
     version_regex = r"[Vv]ersion.*?(\d+(\.\d+)+)"
 
     @property

lib/spack/spack/platforms/cray.py

@@ -12,6 +12,7 @@
 import llnl.util.tty as tty
 
 import spack.target
+import spack.version
 from spack.operating_systems.cray_backend import CrayBackend
 from spack.operating_systems.cray_frontend import CrayFrontend
 from spack.paths import build_env_path
@@ -31,6 +32,9 @@
     "abudhabi": "piledriver",
 }
 
+_ex_craype_dir = "/opt/cray/pe/cpe"
+_xc_craype_dir = "/opt/cray/pe/cdt"
+
 
 def _target_name_from_craype_target_name(name):
     return _craype_name_to_target_name.get(name, name)
@@ -109,19 +113,48 @@ def setup_platform_environment(self, pkg, env):
         if os.environ.get("CRAY_LD_LIBRARY_PATH"):
             env.prepend_path("LD_LIBRARY_PATH", os.environ["CRAY_LD_LIBRARY_PATH"])
 
+    @classmethod
+    def craype_type_and_version(cls):
+        if os.path.isdir(_ex_craype_dir):
+            craype_dir = _ex_craype_dir
+            craype_type = "EX"
+        elif os.path.isdir(_xc_craype_dir):
+            craype_dir = _xc_craype_dir
+            craype_type = "XC"
+        else:
+            return (None, None)
+
+        # Take the default version from known symlink path
+        default_path = os.path.join(craype_dir, "default")
+        if os.path.islink(default_path):
+            version = spack.version.Version(os.readlink(default_path))
+            return (craype_type, version)
+
+        # If no default version, sort available versions and return latest
+        versions_available = [spack.version.Version(v) for v in os.listdir(craype_dir)]
+        versions_available.sort(reverse=True)
+        return (craype_type, versions_available[0])
+
     @classmethod
     def detect(cls):
         """
-        Detect whether this system is a Cray machine.
+        Detect whether this system requires CrayPE module support.
 
-        We detect the Cray platform based on the availability through `module`
+        Systems with newer CrayPE (21.10 for EX systems, future work for CS and
-        of the Cray programming environment. If this environment is available,
+        XC systems) have compilers and MPI wrappers that can be used directly
-        we can use it to find compilers, target modules, etc. If the Cray
+        by path. These systems are considered ``linux`` platforms.
-        programming environment is not available via modules, then we will
+
-        treat it as a standard linux system, as the Cray compiler wrappers
+        For systems running an older CrayPE, we detect the Cray platform based
-        and other components of the Cray programming environment are
+        on the availability through `module` of the Cray programming
-        irrelevant without module support.
+        environment. If this environment is available, we can use it to find
+        compilers, target modules, etc. If the Cray programming environment is
+        not available via modules, then we will treat it as a standard linux
+        system, as the Cray compiler wrappers and other components of the Cray
+        programming environment are irrelevant without module support.
         """
+        craype_type, craype_version = cls.craype_type_and_version()
+        if craype_type == "EX" and craype_version >= spack.version.Version("21.10"):
+            return False
         return "opt/cray" in os.environ.get("MODULEPATH", "")
 
     def _default_target_from_env(self):

lib/spack/spack/test/architecture.py

@@ -8,6 +8,8 @@
 
 import pytest
 
+import llnl.util.filesystem as fs
+
 import spack.concretize
 import spack.operating_systems
 import spack.platforms
@@ -214,3 +216,27 @@ def test_concretize_target_ranges(root_target_range, dep_target_range, result, m
         spec.concretize()
 
     assert str(spec).count("arch=test-debian6-%s" % result) == 2
+
+
+@pytest.mark.parametrize(
+    "versions,default,expected",
+    [
+        (["21.11", "21.9"], "21.11", False),
+        (["21.11", "21.9"], "21.9", True),
+        (["21.11", "21.9"], None, False),
+    ],
+)
+def test_cray_platform_detection(versions, default, expected, tmpdir, monkeypatch, working_env):
+    ex_path = str(tmpdir.join("fake_craype_dir"))
+    fs.mkdirp(ex_path)
+
+    with fs.working_dir(ex_path):
+        for version in versions:
+            fs.touch(version)
+        if default:
+            os.symlink(default, "default")
+
+    monkeypatch.setattr(spack.platforms.cray, "_ex_craype_dir", ex_path)
+    os.environ["MODULEPATH"] = "/opt/cray/pe"
+
+    assert spack.platforms.cray.Cray.detect() == expected

var/spack/repos/builtin/packages/aluminum/package.py

@@ -54,13 +54,13 @@ class Aluminum(CMakePackage, CudaPackage, ROCmPackage):
     variant(
         "ofi_libfabric_plugin",
         default=True,
-        when="+rccl platform=cray",
+        when="+rccl",
         description="Builds with support for OFI libfabric enhanced RCCL/NCCL communication lib",
     )
     variant(
         "ofi_libfabric_plugin",
         default=True,
-        when="+nccl platform=cray",
+        when="+nccl",
         description="Builds with support for OFI libfabric enhanced RCCL/NCCL communication lib",
     )
 
@@ -75,8 +75,8 @@ class Aluminum(CMakePackage, CudaPackage, ROCmPackage):
     depends_on("hipcub", when="@:0.1,0.6.0: +rocm")
 
     depends_on("rccl", when="+rccl")
-    depends_on("aws-ofi-rccl", when="+rccl +ofi_libfabric_plugin platform=cray")
+    depends_on("aws-ofi-rccl", when="+rccl +ofi_libfabric_plugin")
-    depends_on("aws-ofi-nccl", when="+nccl +ofi_libfabric_plugin platform=cray")
+    depends_on("aws-ofi-nccl", when="+nccl +ofi_libfabric_plugin")
 
     conflicts("~cuda", when="+cuda_rma", msg="CUDA RMA support requires CUDA")
     conflicts("+cuda", when="+rocm", msg="CUDA and ROCm support are mutually exclusive")

var/spack/repos/builtin/packages/esmf/package.py

@@ -255,7 +255,7 @@ def edit(self, spec, prefix):
         # ESMF_COMM must be set to indicate which MPI implementation
         # is used to build the ESMF library.
         if "+mpi" in spec:
-            if "platform=cray" in self.spec:
+            if "^cray-mpich" in self.spec:
                 os.environ["ESMF_COMM"] = "mpi"
             elif "^mvapich2" in spec:
                 os.environ["ESMF_COMM"] = "mvapich2"

var/spack/repos/builtin/packages/mpich/package.py

@@ -390,8 +390,7 @@ def setup_run_environment(self, env):
         # their run environments the code to make the compilers available.
         # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
         # Cray MPIs always have cray in the module name, e.g. "cray-mpich"
-        external_modules = self.spec.external_modules
+        if self.spec.satisfies("platform=cray"):
-        if external_modules and "cray" in external_modules[0]:
             # This is intended to support external MPICH instances registered
             # by Spack on Cray machines prior to a879c87; users defining an
             # external MPICH entry for Cray should generally refer to the
@@ -420,8 +419,7 @@ def setup_dependent_package(self, module, dependent_spec):
 
         # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
         # Cray MPIs always have cray in the module name, e.g. "cray-mpich"
-        external_modules = spec.external_modules
+        if self.spec.satisfies("platform=cray"):
-        if external_modules and "cray" in external_modules[0]:
             spec.mpicc = spack_cc
             spec.mpicxx = spack_cxx
             spec.mpifc = spack_fc

var/spack/repos/builtin/packages/mvapich2/package.py

@@ -358,8 +358,7 @@ def setup_dependent_build_environment(self, env, dependent_spec):
     def setup_compiler_environment(self, env):
         # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
         # Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
-        external_modules = self.spec.external_modules
+        if self.spec.satisfies("platform=cray"):
-        if external_modules and "cray" in external_modules[0]:
             env.set("MPICC", spack_cc)
             env.set("MPICXX", spack_cxx)
             env.set("MPIF77", spack_fc)
@@ -373,8 +372,7 @@ def setup_compiler_environment(self, env):
     def setup_dependent_package(self, module, dependent_spec):
         # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
         # Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
-        external_modules = self.spec.external_modules
+        if self.spec.satisfies("platform=cray"):
-        if external_modules and "cray" in external_modules[0]:
             self.spec.mpicc = spack_cc
             self.spec.mpicxx = spack_cxx
             self.spec.mpifc = spack_fc

var/spack/repos/builtin/packages/opencoarrays/package.py

@@ -35,7 +35,7 @@ class Opencoarrays(CMakePackage):
     depends_on("mpi")
     # This patch removes a bunch of checks for the version of MPI available on
     # the system. They make the Crays hang.
-    patch("CMakeLists.patch", when="platform=cray")
+    patch("CMakeLists.patch", when="^cray-mpich")
 
     def cmake_args(self):
         args = []

var/spack/repos/builtin/packages/scorep/package.py

@@ -124,7 +124,11 @@ def configure_args(self):
 
         if spec.satisfies("^intel-mpi"):
             config_args.append("--with-mpi=intel3")
-        elif spec.satisfies("^mpich") or spec.satisfies("^mvapich2"):
+        elif (
+            spec.satisfies("^mpich")
+            or spec.satisfies("^mvapich2")
+            or spec.satisfies("^cray-mpich")
+        ):
             config_args.append("--with-mpi=mpich3")
         elif spec.satisfies("^openmpi"):
             config_args.append("--with-mpi=openmpi")

var/spack/repos/builtin/packages/xsdk/package.py

@@ -190,7 +190,7 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("datatransferkit@3.1-rc2", when="@0.6.0 +trilinos +datatransferkit")
 
     xsdk_depends_on("petsc +trilinos", when="+trilinos @:0.6.0")
-    xsdk_depends_on("petsc +batch", when="platform=cray @0.5.0:")
+    xsdk_depends_on("petsc +batch", when="@0.5.0: ^cray-mpich")
     xsdk_depends_on(
         "petsc@main+mpi+hypre+superlu-dist+metis+hdf5~mumps+double~int64",
         when="@develop",
@@ -398,7 +398,7 @@ class Xsdk(BundlePackage, CudaPackage, ROCmPackage):
     xsdk_depends_on("py-libensemble@0.5.2+petsc4py", when="@0.5.0 +libensemble")
     xsdk_depends_on("py-petsc4py@3.12.0", when="@0.5.0 +libensemble")
 
-    xsdk_depends_on("precice ~petsc", when="platform=cray +precice")
+    xsdk_depends_on("precice ~petsc", when="+precice ^cray-mpich")
     xsdk_depends_on("precice@develop", when="@develop +precice")
     xsdk_depends_on("precice@2.3.0", when="@0.7.0 +precice")
     xsdk_depends_on("precice@2.1.1", when="@0.6.0 +precice")