Cray: fix Blue Waters support and user-built MPIs on Cray (#16593)

* Cray: fix Blue Waters support

* pkg-config env vars needed on Blue Waters

* cray platform: fix support for user-build MPI on cray machines

* reintroduce cray environment cleaning behind cnl version guard

* cray platform: fix support for user-build MPI on cray machines

Co-authored-by: Gregory <becker33@llnl.gov>

lib/spack/spack/build_environment.py

@@ -53,6 +53,7 @@
 import spack.paths
 import spack.schema.environment
 import spack.store
+import spack.architecture as arch
 from spack.util.string import plural
 from spack.util.environment import (
     env_flag, filter_system_paths, get_path, is_system_path,
@@ -141,14 +142,18 @@ def clean_environment():
     # can affect how some packages find libraries.  We want to make
     # sure that builds never pull in unintended external dependencies.
     env.unset('LD_LIBRARY_PATH')
-    env.unset('CRAY_LD_LIBRARY_PATH')
     env.unset('LIBRARY_PATH')
     env.unset('CPATH')
     env.unset('LD_RUN_PATH')
     env.unset('DYLD_LIBRARY_PATH')
     env.unset('DYLD_FALLBACK_LIBRARY_PATH')
 
-    # Remove all pkgconfig stuff from craype
+    # On Cray systems newer than CNL5, unset CRAY_LD_LIBRARY_PATH to avoid
+    # interference with Spack dependencies. CNL5 (e.g. Blue Waters) requires
+    # these variables to be set.
+    hostarch = arch.Arch(arch.platform(), 'default_os', 'default_target')
+    if str(hostarch.platform) == 'cray' and str(hostarch.os) != 'cnl5':
+        env.unset('CRAY_LD_LIBRARY_PATH')
         for varname in os.environ.keys():
             if 'PKGCONF' in varname:
                 env.unset(varname)
@@ -355,10 +360,6 @@ def set_build_environment_variables(pkg, env, dirty):
         extra_rpaths = ':'.join(compiler.extra_rpaths)
         env.set('SPACK_COMPILER_EXTRA_RPATHS', extra_rpaths)
 
-    implicit_rpaths = compiler.implicit_rpaths()
-    if implicit_rpaths:
-        env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths))
-
     # Add bin directories from dependencies to the PATH for the build.
     for prefix in build_prefixes:
         for dirname in ['bin', 'bin64']:
@@ -733,6 +734,11 @@ def setup_package(pkg, dirty):
 
         load_external_modules(pkg)
 
+    implicit_rpaths = pkg.compiler.implicit_rpaths()
+    if implicit_rpaths:
+        build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS',
+                      ':'.join(implicit_rpaths))
+
     # Make sure nothing's strange about the Spack environment.
     validate(build_env, tty.warn)
     build_env.apply_modifications()

lib/spack/spack/platforms/cray.py

@@ -22,7 +22,9 @@
     'x86-naples': 'zen',
     'x86-rome': 'zen',  # Cheating because we have the wrong modules on rzcrayz
     'x86-skylake': 'skylake_avx512',
-    'mic-knl': 'mic_knl'
+    'mic-knl': 'mic_knl',
+    'interlagos': 'bulldozer',
+    'abudhabi': 'piledriver',
 }
 
 
@@ -176,7 +178,7 @@ def modules_from_listdir():
             craype_default_path = '/opt/cray/pe/craype/default/modulefiles'
             if os.path.isdir(craype_default_path):
                 return os.listdir(craype_default_path)
-            return None
+            return []
 
         if getattr(self, '_craype_targets', None) is None:
             strategies = [

var/spack/repos/builtin/packages/mpich/package.py

@@ -161,8 +161,9 @@ def setup_build_environment(self, env):
             env.set('FFLAGS', '-fallow-argument-mismatch')
 
     def setup_dependent_build_environment(self, env, dependent_spec):
-        # On Cray, the regular compiler wrappers *are* the MPI wrappers.
+        # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
-        if 'platform=cray' in self.spec:
+        # Cray MPIs always have cray in the module name, e.g. "cray-mpich"
+        if self.spec.external_module and 'cray' in self.spec.external_module:
             env.set('MPICC', spack_cc)
             env.set('MPICXX', spack_cxx)
             env.set('MPIF77', spack_fc)
@@ -180,7 +181,9 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         env.set('MPICH_FC', spack_fc)
 
     def setup_dependent_package(self, module, dependent_spec):
-        if 'platform=cray' in self.spec:
+        # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
+        # Cray MPIs always have cray in the module name, e.g. "cray-mpich"
+        if self.spec.external_module and 'cray' in self.spec.external_module:
             self.spec.mpicc = spack_cc
             self.spec.mpicxx = spack_cxx
             self.spec.mpifc = spack_fc

var/spack/repos/builtin/packages/mvapich2/package.py

@@ -209,8 +209,9 @@ def setup_run_environment(self, env):
             env.set('SLURM_MPI_TYPE', 'pmi2')
 
     def setup_dependent_build_environment(self, env, dependent_spec):
-        # On Cray, the regular compiler wrappers *are* the MPI wrappers.
+        # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
-        if 'platform=cray' in self.spec:
+        # Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
+        if self.spec.external_module and 'cray' in self.spec.external_module:
             env.set('MPICC',  spack_cc)
             env.set('MPICXX', spack_cxx)
             env.set('MPIF77', spack_fc)
@@ -228,7 +229,9 @@ def setup_dependent_build_environment(self, env, dependent_spec):
         env.set('MPICH_FC', spack_fc)
 
     def setup_dependent_package(self, module, dependent_spec):
-        if 'platform=cray' in self.spec:
+        # For Cray MPIs, the regular compiler wrappers *are* the MPI wrappers.
+        # Cray MPIs always have cray in the module name, e.g. "cray-mvapich"
+        if self.spec.external_module and 'cray' in self.spec.external_module:
             self.spec.mpicc = spack_cc
             self.spec.mpicxx = spack_cxx
             self.spec.mpifc = spack_fc