External finding: update default paths; treat .bat as executable on Windows (#39850)

.bat or .exe files can be considered executable on Windows. This PR
expands the regex for detectable packages to allow for the detection
of packages that vendor .bat wrappers (intel mpi for example).

Additional changes:

* Outside of Windows, when searching for executables `path_hints=None`
  was used to indicate that default path hints should be provided,
  and `[]` was taken to mean that no defaults should be chosen
  (in that case, nothing is searched); behavior on Windows has
  now been updated to match.
* Above logic for handling of `path_hints=[]`  has also been extended
  to library search (for both Linux and Windows).
* All exceptions for external packages were documented as timeout
  errors: this commit adds a distinction for other types of errors
  in warning messages to the user.

lib/spack/spack/detection/path.py

@@ -39,12 +39,21 @@
     DETECTION_TIMEOUT = 120
 
 
-def common_windows_package_paths() -> List[str]:
+def common_windows_package_paths(pkg_cls=None) -> List[str]:
+    """Get the paths for common package installation location on Windows
+    that are outside the PATH
+    Returns [] on unix
+    """
+    if sys.platform != "win32":
+        return []
     paths = WindowsCompilerExternalPaths.find_windows_compiler_bundled_packages()
     paths.extend(find_win32_additional_install_paths())
     paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
     paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
     paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
+    if pkg_cls:
+        paths.extend(compute_windows_user_path_for_package(pkg_cls))
+        paths.extend(compute_windows_program_path_for_package(pkg_cls))
     return paths
 
 
@@ -62,8 +71,6 @@ def executables_in_path(path_hints: List[str]) -> Dict[str, str]:
         path_hints: list of paths to be searched. If None the list will be
             constructed based on the PATH environment variable.
     """
-    if sys.platform == "win32":
-        path_hints.extend(common_windows_package_paths())
     search_paths = llnl.util.filesystem.search_paths_for_executables(*path_hints)
     return path_to_dict(search_paths)
 
@@ -88,27 +95,39 @@ def libraries_in_ld_and_system_library_path(
             DYLD_LIBRARY_PATH, and DYLD_FALLBACK_LIBRARY_PATH environment
             variables as well as the standard system library paths.
     """
-    path_hints = (
+    default_lib_search_paths = (
-        path_hints
+        spack.util.environment.get_path("LD_LIBRARY_PATH")
-        or spack.util.environment.get_path("LD_LIBRARY_PATH")
         + spack.util.environment.get_path("DYLD_LIBRARY_PATH")
         + spack.util.environment.get_path("DYLD_FALLBACK_LIBRARY_PATH")
         + spack.util.ld_so_conf.host_dynamic_linker_search_paths()
     )
+    path_hints = path_hints if path_hints is not None else default_lib_search_paths
+
     search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
     return path_to_dict(search_paths)
 
 
-def libraries_in_windows_paths(path_hints: List[str]) -> Dict[str, str]:
+def libraries_in_windows_paths(path_hints: Optional[List[str]] = None) -> Dict[str, str]:
-    path_hints.extend(spack.util.environment.get_path("PATH"))
+    """Get the paths of all libraries available from the system PATH paths.
-    search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
+
+    For more details, see `libraries_in_ld_and_system_library_path` regarding
+    return type and contents.
+
+    Args:
+        path_hints: list of paths to be searched. If None the list will be
+            constructed based on the set of PATH environment
+            variables as well as the standard system library paths.
+    """
+    search_hints = (
+        path_hints if path_hints is not None else spack.util.environment.get_path("PATH")
+    )
+    search_paths = llnl.util.filesystem.search_paths_for_libraries(*search_hints)
     # on Windows, some libraries (.dlls) are found in the bin directory or sometimes
     # at the search root. Add both of those options to the search scheme
-    search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*path_hints))
+    search_paths.extend(llnl.util.filesystem.search_paths_for_executables(*search_hints))
+    if path_hints is None:
+        # if no user provided path was given, add defaults to the search
         search_paths.extend(WindowsKitExternalPaths.find_windows_kit_lib_paths())
-    search_paths.extend(WindowsKitExternalPaths.find_windows_kit_bin_paths())
-    search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_installed_roots_paths())
-    search_paths.extend(WindowsKitExternalPaths.find_windows_kit_reg_sdk_paths())
         # SDK and WGL should be handled by above, however on occasion the WDK is in an atypical
         # location, so we handle that case specifically.
         search_paths.extend(WindowsKitExternalPaths.find_windows_driver_development_kit_paths())
@@ -125,19 +144,8 @@ def _group_by_prefix(paths: Set[str]) -> Dict[str, Set[str]]:
 class Finder:
     """Inspects the file-system looking for packages. Guesses places where to look using PATH."""
 
-    def path_hints(
+    def default_path_hints(self) -> List[str]:
-        self, *, pkg: "spack.package_base.PackageBase", initial_guess: Optional[List[str]] = None
+        return []
-    ) -> List[str]:
-        """Returns the list of paths to be searched.
-
-        Args:
-            pkg: package being detected
-            initial_guess: initial list of paths from caller
-        """
-        result = initial_guess or []
-        result.extend(compute_windows_user_path_for_package(pkg))
-        result.extend(compute_windows_program_path_for_package(pkg))
-        return result
 
     def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]:
         """Returns the list of patterns used to match candidate files.
@@ -245,6 +253,8 @@ def find(
         Args:
             pkg_name: package being detected
             initial_guess: initial list of paths to search from the caller
+                           if None, default paths are searched. If this
+                           is an empty list, nothing will be searched.
         """
         import spack.repo
 
@@ -252,13 +262,18 @@ def find(
         patterns = self.search_patterns(pkg=pkg_cls)
         if not patterns:
             return []
-        path_hints = self.path_hints(pkg=pkg_cls, initial_guess=initial_guess)
+        if initial_guess is None:
-        candidates = self.candidate_files(patterns=patterns, paths=path_hints)
+            initial_guess = self.default_path_hints()
+            initial_guess.extend(common_windows_package_paths(pkg_cls))
+        candidates = self.candidate_files(patterns=patterns, paths=initial_guess)
         result = self.detect_specs(pkg=pkg_cls, paths=candidates)
         return result
 
 
 class ExecutablesFinder(Finder):
+    def default_path_hints(self) -> List[str]:
+        return spack.util.environment.get_path("PATH")
+
     def search_patterns(self, *, pkg: "spack.package_base.PackageBase") -> List[str]:
         result = []
         if hasattr(pkg, "executables") and hasattr(pkg, "platform_executables"):
@@ -298,7 +313,7 @@ def candidate_files(self, *, patterns: List[str], paths: List[str]) -> List[str]
         libraries_by_path = (
             libraries_in_ld_and_system_library_path(path_hints=paths)
             if sys.platform != "win32"
-            else libraries_in_windows_paths(paths)
+            else libraries_in_windows_paths(path_hints=paths)
         )
         patterns = [re.compile(x) for x in patterns]
         result = []
@@ -334,21 +349,16 @@ def by_path(
     # TODO: Packages should be able to define both .libraries and .executables in the future
     # TODO: determine_spec_details should get all relevant libraries and executables in one call
     executables_finder, libraries_finder = ExecutablesFinder(), LibrariesFinder()
-
-    executables_path_guess = (
-        spack.util.environment.get_path("PATH") if path_hints is None else path_hints
-    )
-    libraries_path_guess = [] if path_hints is None else path_hints
     detected_specs_by_package: Dict[str, Tuple[concurrent.futures.Future, ...]] = {}
 
     result = collections.defaultdict(list)
     with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor:
         for pkg in packages_to_search:
             executable_future = executor.submit(
-                executables_finder.find, pkg_name=pkg, initial_guess=executables_path_guess
+                executables_finder.find, pkg_name=pkg, initial_guess=path_hints
             )
             library_future = executor.submit(
-                libraries_finder.find, pkg_name=pkg, initial_guess=libraries_path_guess
+                libraries_finder.find, pkg_name=pkg, initial_guess=path_hints
             )
             detected_specs_by_package[pkg] = executable_future, library_future
 
@@ -359,9 +369,13 @@ def by_path(
                     if detected:
                         _, unqualified_name = spack.repo.partition_package_name(pkg_name)
                         result[unqualified_name].extend(detected)
-                except Exception:
+                except concurrent.futures.TimeoutError:
                     llnl.util.tty.debug(
                         f"[EXTERNAL DETECTION] Skipping {pkg_name}: timeout reached"
                     )
+                except Exception as e:
+                    llnl.util.tty.debug(
+                        f"[EXTERNAL DETECTION] Skipping {pkg_name}: exception occured {e}"
+                    )
 
     return result

lib/spack/spack/test/cmd/external.py

@@ -28,21 +28,12 @@ def _mock_search(path_hints=None):
     return _factory
 
 
-@pytest.fixture
-def _platform_executables(monkeypatch):
-    def _win_exe_ext():
-        return ".bat"
-
-    monkeypatch.setattr(spack.util.path, "win_exe_ext", _win_exe_ext)
-
-
 def define_plat_exe(exe):
     if sys.platform == "win32":
         exe += ".bat"
     return exe
 
 
-@pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/spack/spack/pull/39850")
 def test_find_external_single_package(mock_executable):
     cmake_path = mock_executable("cmake", output="echo cmake version 1.foo")
     search_dir = cmake_path.parent.parent
@@ -54,7 +45,7 @@ def test_find_external_single_package(mock_executable):
     assert len(detected_spec) == 1 and detected_spec[0].spec == Spec("cmake@1.foo")
 
 
-def test_find_external_two_instances_same_package(mock_executable, _platform_executables):
+def test_find_external_two_instances_same_package(mock_executable):
     # Each of these cmake instances is created in a different prefix
     # In Windows, quoted strings are echo'd with quotes includes
     # we need to avoid that for proper regex.
@@ -236,32 +227,7 @@ def test_list_detectable_packages(mutable_config, mutable_mock_repo):
     assert external.returncode == 0
 
 
-@pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/spack/spack/pull/39850")
+def test_overriding_prefix(mock_executable, mutable_config, monkeypatch):
-def test_packages_yaml_format(mock_executable, mutable_config, monkeypatch, _platform_executables):
-    # Prepare an environment to detect a fake gcc
-    gcc_exe = mock_executable("gcc", output="echo 4.2.1")
-    prefix = os.path.dirname(gcc_exe)
-    monkeypatch.setenv("PATH", prefix)
-
-    # Find the external spec
-    external("find", "gcc")
-
-    # Check entries in 'packages.yaml'
-    packages_yaml = spack.config.get("packages")
-    assert "gcc" in packages_yaml
-    assert "externals" in packages_yaml["gcc"]
-    externals = packages_yaml["gcc"]["externals"]
-    assert len(externals) == 1
-    external_gcc = externals[0]
-    assert external_gcc["spec"] == "gcc@4.2.1 languages=c"
-    assert external_gcc["prefix"] == os.path.dirname(prefix)
-    assert "extra_attributes" in external_gcc
-    extra_attributes = external_gcc["extra_attributes"]
-    assert "prefix" not in extra_attributes
-    assert extra_attributes["compilers"]["c"] == str(gcc_exe)
-
-
-def test_overriding_prefix(mock_executable, mutable_config, monkeypatch, _platform_executables):
     gcc_exe = mock_executable("gcc", output="echo 4.2.1")
     search_dir = gcc_exe.parent
 
@@ -282,10 +248,7 @@ def _determine_variants(cls, exes, version_str):
     assert gcc.external_path == os.path.sep + os.path.join("opt", "gcc", "bin")
 
 
-@pytest.mark.xfail(sys.platform == "win32", reason="https://github.com/spack/spack/pull/39850")
+def test_new_entries_are_reported_correctly(mock_executable, mutable_config, monkeypatch):
-def test_new_entries_are_reported_correctly(
-    mock_executable, mutable_config, monkeypatch, _platform_executables
-):
     # Prepare an environment to detect a fake gcc
     gcc_exe = mock_executable("gcc", output="echo 4.2.1")
     prefix = os.path.dirname(gcc_exe)

lib/spack/spack/test/detection.py

@@ -0,0 +1,30 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import collections
+
+import spack.detection
+import spack.spec
+
+
+def test_detection_update_config(mutable_config):
+    # mock detected package
+    detected_packages = collections.defaultdict(list)
+    detected_packages["cmake"] = [
+        spack.detection.common.DetectedPackage(
+            spec=spack.spec.Spec("cmake@3.27.5"), prefix="/usr/bin"
+        )
+    ]
+
+    # update config for new package
+    spack.detection.common.update_configuration(detected_packages)
+    # Check entries in 'packages.yaml'
+    packages_yaml = spack.config.get("packages")
+    assert "cmake" in packages_yaml
+    assert "externals" in packages_yaml["cmake"]
+    externals = packages_yaml["cmake"]["externals"]
+    assert len(externals) == 1
+    external_gcc = externals[0]
+    assert external_gcc["spec"] == "cmake@3.27.5"
+    assert external_gcc["prefix"] == "/usr/bin"

lib/spack/spack/util/path.py

@@ -98,7 +98,7 @@ def replacements():
 
 
 def win_exe_ext():
-    return ".exe"
+    return r"(?:\.bat|\.exe)"
 
 
 def sanitize_filename(filename: str) -> str: