find_libraries: use targeted, iterative deepening with default max depth

Use some form of iterative deepening with early return in
`find_libraries`:

1. Search `<root>`, `<root>/{lib, lib64,...}` dirs at max depth 0 and
   return early if found.
2. Search `<root>/{lib, lib64,...}` dirs at max depth 1 and return early
   if found (covers e.g. `<root>/lib/x86_64-linux-gnu/` subdirs)
3. Fall back to exhaustive search up to `max_depth - 1` of
   `<root>/{lib, lib64, ...}`.

Set the default library search handler of `max_depth` to `4` to still
cover `<root>/lib/pythonX.Y/site-packages/<name>/`.

lib/spack/llnl/util/filesystem.py

@@ -2323,9 +2323,14 @@ def find_system_libraries(libraries, shared=True):
 
 
 def find_libraries(
-    libraries, root, shared=True, recursive=False, runtime=True, max_depth: Optional[int] = None
+    libraries: Union[List[str], str],
-):
+    root: str,
-    """Returns an iterable of full paths to libraries found in a root dir.
+    shared: bool = True,
+    recursive: bool = False,
+    runtime: bool = True,
+    max_depth: Optional[int] = None,
+) -> LibraryList:
+    """Find libraries in the specified root directory.
 
     Accepts any glob characters accepted by fnmatch:
 
@@ -2339,18 +2344,15 @@ def find_libraries(
     =======  ====================================
 
     Parameters:
-        libraries (str or list): Library name(s) to search for
+        libraries: library name(s) to search for
-        root (str): The root directory to start searching from
+        root: the root directory to start searching from
-        shared (bool): if True searches for shared libraries,
+        shared: if True searches for shared libraries, otherwise for static. Defaults to True.
-            otherwise for static. Defaults to True.
+        recursive: if False (default) search only root folder, if True recurse from the root. Note
-        recursive (bool): if False search only root folder,
+            that recursive search does not imply exhaustive search. The function returns early if
-            if True descends top-down from the root. Defaults to False.
+            libraries are found in typical, low-depth library directories.
-        max_depth (int): if set, don't search below this depth. Cannot be set
+        max_depth: if set, don't search below this depth. Cannot be set if recursive is False
-            if recursive is False
+        runtime: Windows only option, no-op elsewhere. If True (default), search for runtime shared
-        runtime (bool): Windows only option, no-op elsewhere. If true,
+            libs (.DLL), otherwise, search for .Lib files. If shared is False, this has no meaning.
-            search for runtime shared libs (.DLL), otherwise, search
-            for .Lib files. If shared is false, this has no meaning.
-            Defaults to True.
 
     Returns:
         LibraryList: The libraries that have been found
@@ -2359,10 +2361,13 @@ def find_libraries(
     if isinstance(libraries, str):
         libraries = [libraries]
     elif not isinstance(libraries, collections.abc.Sequence):
-        message = "{0} expects a string or sequence of strings as the "
+        raise TypeError(
-        message += "first argument [got {1} instead]"
+            f"{find_libraries.__name__} expects a string or sequence of strings as the "
-        message = message.format(find_libraries.__name__, type(libraries))
+            f"first argument [got {type(libraries)} instead]"
-        raise TypeError(message)
+        )
+
+    if not recursive and max_depth is not None:
+        raise ValueError(f"max_depth ({max_depth}) cannot be set if recursive is False")
 
     if sys.platform == "win32":
         static_ext = "lib"
@@ -2385,33 +2390,47 @@ def find_libraries(
         suffixes = [static_ext]
 
     # List of libraries we are searching with suffixes
-    libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
+    libraries = [f"{lib}.{suffix}" for lib in libraries for suffix in suffixes]
 
     if not recursive:
-        if max_depth:
-            raise ValueError(f"max_depth ({max_depth}) cannot be set if recursive is False")
-        # If not recursive, look for the libraries directly in root
         return LibraryList(find(root, libraries, recursive=False))
 
-    # To speedup the search for external packages configured e.g. in /usr,
+    # Even if recursive is True, we will do some form of targeted, iterative deepening, in order
-    # perform first non-recursive search in root/lib then in root/lib64 and
+    # to return early if libraries are found in common, low-depth library directories.
-    # finally search all of root recursively. The search stops when the first
-    # match is found.
-    common_lib_dirs = ["lib", "lib64"]
     if sys.platform == "win32":
-        common_lib_dirs.extend(["bin", "Lib"])
+        common_lib_dirs = ("lib", "lib64", "bin", "Lib")
-
-    for subdir in common_lib_dirs:
-        dirname = join_path(root, subdir)
-        if not os.path.isdir(dirname):
-            continue
-        found_libs = find(dirname, libraries, False)
-        if found_libs:
-            break
     else:
-        found_libs = find(root, libraries, recursive=True, max_depth=max_depth)
+        common_lib_dirs = ("lib", "lib64")
 
-    return LibraryList(found_libs)
+    if os.path.basename(root) not in common_lib_dirs:
+        # search root and its direct library subdirectories non-recursively
+        non_recursive = [root, *(os.path.join(root, libdir) for libdir in common_lib_dirs)]
+        # avoid the expensive recursive search of the root directory
+        fallback_recursive = [os.path.join(root, libdir) for libdir in common_lib_dirs]
+        # reduce max_depth by 1 as we already joined the common library directories
+        if max_depth is not None:
+            max_depth -= 1
+    else:
+        # the call site already has a common library dir as root
+        non_recursive = [root]
+        fallback_recursive = [root]
+
+    found_libs = find(non_recursive, libraries, recursive=False)
+
+    if found_libs:
+        return LibraryList(found_libs)
+
+    # Do one more (manual) step of iterative deepening, to early exit on typical
+    # <root>/lib/<triplet>/ sub-directories before exhaustive, max_depth search. Slightly better
+    # would be to add lib/<triplet> itself to common_lib_dirs, but we are lacking information to
+    # determine the triplet.
+    if max_depth is None or max_depth > 1:
+        found_libs = find(fallback_recursive, libraries, max_depth=1)
+        if found_libs:
+            return LibraryList(found_libs)
+
+    # Finally fall back to exhaustive, recursive search
+    return LibraryList(find(fallback_recursive, libraries, recursive=True, max_depth=max_depth))
 
 
 def find_all_shared_libraries(root, recursive=False, runtime=True):

lib/spack/spack/spec.py

@@ -1158,9 +1158,12 @@ def _libs_default_handler(spec: "Spec"):
     )
 
     for shared in search_shared:
-        # Since we are searching for link libraries, on Windows search only for
+        # Since we are searching for link libraries, on Windows search only for .Lib extensions by
-        # ".Lib" extensions by default as those represent import libraries for implicit links.
+        # default as those represent import libraries for implicit links.
-        libs = fs.find_libraries(name, home, shared=shared, recursive=True, runtime=False)
+        # Set max_depth=4 to allow searching in <home>/lib/pythonX.Y/site-packages/<name>/
+        libs = fs.find_libraries(
+            name, home, shared=shared, recursive=True, runtime=False, max_depth=4
+        )
         if libs:
             return libs