Intel mpi: allow use of external libfabric (#27292)

Intel mpi comes with an installation of libfabric (which it needs as a
dependency). It can use other implementations of libfabric at runtime
though, so if you install a package that depends on `mpi` and
`libfabric`, you can specify `intel-mpi+external-libfabric` and ensure
that the Spack-built instance is used (both by `intel-mpi` and the
root).

Apply analogous change to intel-oneapi-mpi.

lib/spack/spack/build_systems/intel.py

@@ -994,6 +994,16 @@ def libs(self):
                 libnames,
                 root=self.component_lib_dir('mpi'),
                 shared=True, recursive=True) + result
+            # Intel MPI since 2019 depends on libfabric which is not in the
+            # lib directory but in a directory of its own which should be
+            # included in the rpath
+            if self.version_yearlike >= ver('2019'):
+                d = ancestor(self.component_lib_dir('mpi'))
+                if '+external-libfabric' in self.spec:
+                    result += self.spec['libfabric'].libs
+                else:
+                    result += find_libraries(['libfabric'],
+                                             os.path.join(d, 'libfabric', 'lib'))
 
         if '^mpi' in self.spec.root and ('+mkl' in self.spec or
                                          self.provides('scalapack')):
@@ -1091,15 +1101,6 @@ def _setup_dependent_env_callback(
                 # which performs dizzyingly similar but necessarily different
                 # actions, and (b) function code leaves a bit more breathing
                 # room within the suffocating corset of flake8 line length.
-
-                # Intel MPI since 2019 depends on libfabric which is not in the
-                # lib directory but in a directory of its own which should be
-                # included in the rpath
-                if self.version_yearlike >= ver('2019'):
-                    d = ancestor(self.component_lib_dir('mpi'))
-                    libfabrics_path = os.path.join(d, 'libfabric', 'lib')
-                    env.append_path('SPACK_COMPILER_EXTRA_RPATHS',
-                                    libfabrics_path)
             else:
                 raise InstallError('compilers_of_client arg required for MPI')
 

var/spack/repos/builtin/packages/intel-mpi/package.py

@@ -59,6 +59,9 @@ class IntelMpi(IntelPackage):
 
     provides('mpi')
 
+    variant('external-libfabric', default=False, description='Enable external libfabric dependency')
+    depends_on('libfabric', when='+external-libfabric', type=('build', 'link', 'run'))
+
     def setup_dependent_build_environment(self, *args):
         # Handle in callback, conveying client's compilers in additional arg.
         # CAUTION - DUP code in:

var/spack/repos/builtin/packages/intel-oneapi-mpi/package.py

@@ -4,9 +4,7 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 
-import glob
 import platform
-import subprocess
 
 from spack import *
 
@@ -38,11 +36,11 @@ class IntelOneapiMpi(IntelOneApiLibraryPackage):
 
     variant('ilp64', default=False,
             description='Build with ILP64 support')
+    variant('external-libfabric', default=False, description='Enable external libfabric dependency')
+    depends_on('libfabric', when='+external-libfabric', type=('link', 'run'))
 
     provides('mpi@:3.1')
 
-    depends_on('patchelf', type='build')
-
     @property
     def component_dir(self):
         return 'mpi'
@@ -87,17 +85,19 @@ def libs(self):
         libs += find_libraries(['libmpicxx', 'libmpifort'], lib_dir)
         libs += find_libraries('libmpi', release_lib_dir)
         libs += find_system_libraries(['libdl', 'librt', 'libpthread'])
+
+        # Find libfabric for libmpi.so
+        if '+external-libfabric' in self.spec:
+            libs += self.spec['libfabric'].libs
+        else:
+            libs += find_libraries(['libfabric'],
+                                   join_path(self.component_path, 'libfabric', 'lib'))
+
         return libs
 
     def install(self, spec, prefix):
         super(IntelOneapiMpi, self).install(spec, prefix)
 
-        # Patch libmpi.so rpath so it can find libfabric
-        libfabric_rpath = join_path(self.component_path, 'libfabric', 'lib')
-        for libmpi in glob.glob(join_path(self.component_path,
-                                          'lib', '**', 'libmpi*.so')):
-            subprocess.call(['patchelf', '--set-rpath', libfabric_rpath, libmpi])
-
         # When spack builds from source
         # fix I_MPI_SUBSTITUTE_INSTALLDIR and
         #   __EXEC_PREFIX_TO_BE_FILLED_AT_INSTALL_TIME__