Windows rpath support (#31930)
Add a post-install step which runs (only) on Windows to modify an install prefix, adding symlinks to all dependency libraries. Windows does not have the same concept of RPATHs as Linux, but when resolving symbols will check the local directory for dependency libraries; by placing a symlink to each dependency library in the directory with the library that needs it, the package can then use all Spack-built dependencies. Note: * This collects dependency libraries based on Package.rpath, which includes only direct link dependencies * There is no examination of libraries to check what dependencies they require, so all libraries of dependencies are symlinked into any directory of the package which contains libraries
This commit is contained in:
John W. Parent
@@ -22,7 +22,7 @@
|
||||
from llnl.util import tty
|
||||
from llnl.util.compat import Sequence
|
||||
from llnl.util.lang import dedupe, memoized
|
||||
from llnl.util.symlink import symlink
|
||||
from llnl.util.symlink import islink, symlink
|
||||
|
||||
from spack.util.executable import Executable
|
||||
from spack.util.path import path_to_os_path, system_path_filter
|
||||
@@ -637,7 +637,11 @@ def copy_tree(src, dest, symlinks=True, ignore=None, _permissions=False):
|
||||
if symlinks:
|
||||
target = os.readlink(s)
|
||||
if os.path.isabs(target):
|
||||
new_target = re.sub(abs_src, abs_dest, target)
|
||||
|
||||
def escaped_path(path):
|
||||
return path.replace("\\", r"\\")
|
||||
|
||||
new_target = re.sub(escaped_path(abs_src), escaped_path(abs_dest), target)
|
||||
if new_target != target:
|
||||
tty.debug("Redirecting link {0} to {1}".format(target, new_target))
|
||||
target = new_target
|
||||
@@ -1903,7 +1907,11 @@ def names(self):
|
||||
name = x[3:]
|
||||
|
||||
# Valid extensions include: ['.dylib', '.so', '.a']
|
||||
for ext in [".dylib", ".so", ".a"]:
|
||||
# on non Windows platform
|
||||
# Windows valid library extensions are:
|
||||
# ['.dll', '.lib']
|
||||
valid_exts = [".dll", ".lib"] if is_windows else [".dylib", ".so", ".a"]
|
||||
for ext in valid_exts:
|
||||
i = name.rfind(ext)
|
||||
if i != -1:
|
||||
names.append(name[:i])
|
||||
@@ -2046,15 +2054,23 @@ def find_libraries(libraries, root, shared=True, recursive=False):
|
||||
message = message.format(find_libraries.__name__, type(libraries))
|
||||
raise TypeError(message)
|
||||
|
||||
if is_windows:
|
||||
static = "lib"
|
||||
shared = "dll"
|
||||
else:
|
||||
# Used on both Linux and macOS
|
||||
static = "a"
|
||||
shared = "so"
|
||||
|
||||
# Construct the right suffix for the library
|
||||
if shared:
|
||||
# Used on both Linux and macOS
|
||||
suffixes = ["so"]
|
||||
suffixes = [shared]
|
||||
if sys.platform == "darwin":
|
||||
# Only used on macOS
|
||||
suffixes.append("dylib")
|
||||
else:
|
||||
suffixes = ["a"]
|
||||
suffixes = [static]
|
||||
|
||||
# List of libraries we are searching with suffixes
|
||||
libraries = ["{0}.{1}".format(lib, suffix) for lib in libraries for suffix in suffixes]
|
||||
@@ -2067,7 +2083,11 @@ def find_libraries(libraries, root, shared=True, recursive=False):
|
||||
# perform first non-recursive search in root/lib then in root/lib64 and
|
||||
# finally search all of root recursively. The search stops when the first
|
||||
# match is found.
|
||||
for subdir in ("lib", "lib64"):
|
||||
common_lib_dirs = ["lib", "lib64"]
|
||||
if is_windows:
|
||||
common_lib_dirs.extend(["bin", "Lib"])
|
||||
|
||||
for subdir in common_lib_dirs:
|
||||
dirname = join_path(root, subdir)
|
||||
if not os.path.isdir(dirname):
|
||||
continue
|
||||
@@ -2080,6 +2100,155 @@ def find_libraries(libraries, root, shared=True, recursive=False):
|
||||
return LibraryList(found_libs)
|
||||
|
||||
|
||||
def find_all_shared_libraries(root, recursive=False):
|
||||
"""Convenience function that returns the list of all shared libraries found
|
||||
in the directory passed as argument.
|
||||
|
||||
See documentation for `llnl.util.filesystem.find_libraries` for more information
|
||||
"""
|
||||
return find_libraries("*", root=root, shared=True, recursive=recursive)
|
||||
|
||||
|
||||
def find_all_static_libraries(root, recursive=False):
|
||||
"""Convenience function that returns the list of all static libraries found
|
||||
in the directory passed as argument.
|
||||
|
||||
See documentation for `llnl.util.filesystem.find_libraries` for more information
|
||||
"""
|
||||
return find_libraries("*", root=root, shared=False, recursive=recursive)
|
||||
|
||||
|
||||
def find_all_libraries(root, recursive=False):
|
||||
"""Convenience function that returns the list of all libraries found
|
||||
in the directory passed as argument.
|
||||
|
||||
See documentation for `llnl.util.filesystem.find_libraries` for more information
|
||||
"""
|
||||
|
||||
return find_all_shared_libraries(root, recursive=recursive) + find_all_static_libraries(
|
||||
root, recursive=recursive
|
||||
)
|
||||
|
||||
|
||||
class WindowsSimulatedRPath(object):
|
||||
"""Class representing Windows filesystem rpath analog
|
||||
|
||||
One instance of this class is associated with a package (only on Windows)
|
||||
For each lib/binary directory in an associated package, this class introduces
|
||||
a symlink to any/all dependent libraries/binaries. This includes the packages
|
||||
own bin/lib directories, meaning the libraries are linked to the bianry directory
|
||||
and vis versa.
|
||||
"""
|
||||
|
||||
def __init__(self, package, link_install_prefix=True):
|
||||
"""
|
||||
Args:
|
||||
package (spack.package_base.PackageBase): Package requiring links
|
||||
link_install_prefix (bool): Link against package's own install or stage root.
|
||||
Packages that run their own executables during build and require rpaths to
|
||||
the build directory during build time require this option. Default: install
|
||||
root
|
||||
"""
|
||||
self.pkg = package
|
||||
self._addl_rpaths = set()
|
||||
self.link_install_prefix = link_install_prefix
|
||||
self._internal_links = set()
|
||||
|
||||
@property
|
||||
def link_dest(self):
|
||||
"""
|
||||
Set of directories where package binaries/libraries are located.
|
||||
"""
|
||||
if hasattr(self.pkg, "libs") and self.pkg.libs:
|
||||
pkg_libs = set(self.pkg.libs.directories)
|
||||
else:
|
||||
pkg_libs = set((self.pkg.prefix.lib, self.pkg.prefix.lib64))
|
||||
|
||||
return pkg_libs | set([self.pkg.prefix.bin]) | self.internal_links
|
||||
|
||||
@property
|
||||
def internal_links(self):
|
||||
"""
|
||||
linking that would need to be established within the package itself. Useful for links
|
||||
against extension modules/build time executables/internal linkage
|
||||
"""
|
||||
return self._internal_links
|
||||
|
||||
def add_internal_links(self, *dest):
|
||||
"""
|
||||
Incorporate additional paths into the rpath (sym)linking scheme.
|
||||
|
||||
Paths provided to this method are linked against by a package's libraries
|
||||
and libraries found at these paths are linked against a package's binaries.
|
||||
(i.e. /site-packages -> /bin and /bin -> /site-packages)
|
||||
|
||||
Specified paths should be outside of a package's lib, lib64, and bin
|
||||
directories.
|
||||
"""
|
||||
self._internal_links = self._internal_links | set(*dest)
|
||||
|
||||
@property
|
||||
def link_targets(self):
|
||||
"""
|
||||
Set of libraries this package needs to link against during runtime
|
||||
These packages will each be symlinked into the packages lib and binary dir
|
||||
"""
|
||||
|
||||
dependent_libs = []
|
||||
for path in self.pkg.rpath:
|
||||
dependent_libs.extend(list(find_all_shared_libraries(path, recursive=True)))
|
||||
for extra_path in self._addl_rpaths:
|
||||
dependent_libs.extend(list(find_all_shared_libraries(extra_path, recursive=True)))
|
||||
return set(dependent_libs)
|
||||
|
||||
def include_additional_link_paths(self, *paths):
|
||||
"""
|
||||
Add libraries found at the root of provided paths to runtime linking
|
||||
|
||||
These are libraries found outside of the typical scope of rpath linking
|
||||
that require manual inclusion in a runtime linking scheme
|
||||
|
||||
Args:
|
||||
*paths (str): arbitrary number of paths to be added to runtime linking
|
||||
"""
|
||||
self._addl_rpaths = self._addl_rpaths | set(paths)
|
||||
|
||||
def establish_link(self):
|
||||
"""
|
||||
(sym)link packages to runtime dependencies based on RPath configuration for
|
||||
Windows heuristics
|
||||
"""
|
||||
# from build_environment.py:463
|
||||
# The top-level package is always RPATHed. It hasn't been installed yet
|
||||
# so the RPATHs are added unconditionally
|
||||
|
||||
# for each binary install dir in self.pkg (i.e. pkg.prefix.bin, pkg.prefix.lib)
|
||||
# install a symlink to each dependent library
|
||||
for library, lib_dir in itertools.product(self.link_targets, self.link_dest):
|
||||
if not path_contains_subdirectory(library, lib_dir):
|
||||
file_name = os.path.basename(library)
|
||||
dest_file = os.path.join(lib_dir, file_name)
|
||||
if os.path.exists(lib_dir):
|
||||
try:
|
||||
symlink(library, dest_file)
|
||||
# For py2 compatibility, we have to catch the specific Windows error code
|
||||
# associate with trying to create a file that already exists (winerror 183)
|
||||
except OSError as e:
|
||||
if e.winerror == 183:
|
||||
# We have either already symlinked or we are encoutering a naming clash
|
||||
# either way, we don't want to overwrite existing libraries
|
||||
already_linked = islink(dest_file)
|
||||
tty.debug(
|
||||
"Linking library %s to %s failed, " % (library, dest_file)
|
||||
+ "already linked."
|
||||
if already_linked
|
||||
else "library with name %s already exists." % file_name
|
||||
)
|
||||
pass
|
||||
else:
|
||||
raise e
|
||||
|
||||
|
||||
@system_path_filter
|
||||
@memoized
|
||||
def can_access_dir(path):
|
||||
|
||||
Reference in New Issue
Block a user