Windows: fix library loading and enable Clingo bootstrapping (#33400)

Changes to improve locating shared libraries on Windows, which in
turn enables the use of Clingo. This PR attempts to establish a
proper distinction between linking on Windows vs. Linux/Mac: on
Windows, linking is always done with .lib files (never .dll files).
This somewhat complicates the model since the Spec.lib method could
return libraries that were used for both linking and loading, but
since these are not always the same on Windows, it was decided to
treat Spec.libs as being for link-time libraries. Additional functions
are added to help dependents locate run-time libraries.

* Clingo is now the default concretizer on Windows
* Clingo is now the concretizer used for unit tests on Windows
* Fix a permissions issue that can occur while moving Git files during
  fetching/staging
* Packages can now implement "win_add_library_dependent" to register
  files/directories that include libraries that would need to link
  to dependency dlls
* Packages can now implement "win_add_rpath" to register the locations
  of dlls that dependents would want to load
* "Spec.libs" on Windows is updated to return link-time libraries
  (i.e. .lib files, rather than .dll files)
* PackageBase.rpath on Windows is now updated to return the most-likely
  locations where .dlls will be found (which is generally in the bin/
  directory)

lib/spack/spack/fetch_strategy.py

@@ -865,7 +865,12 @@ def clone(self, dest=None, commit=None, branch=None, tag=None, bare=False):
                 repo_name = get_single_file(".")
                 if self.stage:
                     self.stage.srcdir = repo_name
-                shutil.move(repo_name, dest)
+                shutil.copytree(repo_name, dest, symlinks=True)
+                shutil.rmtree(
+                    repo_name,
+                    ignore_errors=False,
+                    onerror=fs.readonly_file_handler(ignore_errors=True),
+                )
 
             with working_dir(dest):
                 checkout_args = ["checkout", commit]

lib/spack/spack/package_base.py

@@ -140,11 +140,30 @@ class WindowsRPathMeta(object):
     they would a genuine RPATH, i.e. adding directories that contain
     runtime library dependencies"""
 
-    def add_search_paths(self, *path):
-        """Add additional rpaths that are not implicitly included in the search
-        scheme
+    def win_add_library_dependent(self):
+        """Return extra set of directories that require linking for package
+
+        This method should be overridden by packages that produce
+        binaries/libraries/python extension modules/etc that are installed into
+        directories outside a package's `bin`, `lib`, and `lib64` directories,
+        but still require linking against one of the packages dependencies, or
+        other components of the package itself. No-op otherwise.
+
+        Returns:
+            List of additional directories that require linking
         """
-        self.win_rpath.include_additional_link_paths(*path)
+        return []
+
+    def win_add_rpath(self):
+        """Return extra set of rpaths for package
+
+        This method should be overridden by packages needing to
+        include additional paths to be searched by rpath. No-op otherwise
+
+        Returns:
+            List of additional rpaths
+        """
+        return []
 
     def windows_establish_runtime_linkage(self):
         """Establish RPATH on Windows
@@ -152,6 +171,8 @@ def windows_establish_runtime_linkage(self):
         Performs symlinking to incorporate rpath dependencies to Windows runtime search paths
         """
         if is_windows:
+            self.win_rpath.add_library_dependent(*self.win_add_library_dependent())
+            self.win_rpath.add_rpath(*self.win_add_rpath())
             self.win_rpath.establish_link()
 
 
@@ -2571,12 +2592,17 @@ def fetch_remote_versions(self, concurrency=128):
     @property
     def rpath(self):
         """Get the rpath this package links with, as a list of paths."""
-        rpaths = [self.prefix.lib, self.prefix.lib64]
         deps = self.spec.dependencies(deptype="link")
-        rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
-        rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
+
+        # on Windows, libraries of runtime interest are typically
+        # stored in the bin directory
         if is_windows:
+            rpaths = [self.prefix.bin]
             rpaths.extend(d.prefix.bin for d in deps if os.path.isdir(d.prefix.bin))
+        else:
+            rpaths = [self.prefix.lib, self.prefix.lib64]
+            rpaths.extend(d.prefix.lib for d in deps if os.path.isdir(d.prefix.lib))
+            rpaths.extend(d.prefix.lib64 for d in deps if os.path.isdir(d.prefix.lib64))
         return rpaths
 
     @property

lib/spack/spack/spec.py

@@ -1030,7 +1030,9 @@ def _libs_default_handler(descriptor, spec, cls):
     )
 
     for shared in search_shared:
-        libs = fs.find_libraries(name, home, shared=shared, recursive=True)
+        # Since we are searching for link libraries, on Windows search only for
+        # ".Lib" extensions by default as those represent import libraries for implict links.
+        libs = fs.find_libraries(name, home, shared=shared, recursive=True, runtime=False)
         if libs:
             return libs
 

lib/spack/spack/test/concretize.py

@@ -2,7 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
-import os
+import posixpath
 import sys
 
 import jinja2
@@ -459,7 +459,7 @@ def test_compiler_inheritance(self, compiler_str):
     def test_external_package(self):
         spec = Spec("externaltool%gcc")
         spec.concretize()
-        assert spec["externaltool"].external_path == os.path.sep + os.path.join(
+        assert spec["externaltool"].external_path == posixpath.sep + posixpath.join(
             "path", "to", "external_tool"
         )
         assert "externalprereq" not in spec
@@ -490,10 +490,10 @@ def test_nobuild_package(self):
     def test_external_and_virtual(self):
         spec = Spec("externaltest")
         spec.concretize()
-        assert spec["externaltool"].external_path == os.path.sep + os.path.join(
+        assert spec["externaltool"].external_path == posixpath.sep + posixpath.join(
             "path", "to", "external_tool"
         )
-        assert spec["stuff"].external_path == os.path.sep + os.path.join(
+        assert spec["stuff"].external_path == posixpath.sep + posixpath.join(
             "path", "to", "external_virtual_gcc"
         )
         assert spec["externaltool"].compiler.satisfies("gcc")