"spack external find": also find library-only packages (#28005)
Update "spack external find --all" to also find library-only packages. A Package can add a ".libraries" attribute, which is a list of regular expressions to use to find libraries associated with the Package. "spack external find --all" will search LD_LIBRARY_PATH for potential libraries. This PR adds examples for NCCL, RCCL, and hipblas packages. These examples specify the suffix ".so" for the regular expressions used to find libraries, so generally are only useful for detecting library packages on Linux.
This commit is contained in:
Brian Van Essen
GitHub
parent
a58fa289b9
commit
29da99427e
@ -1940,6 +1940,11 @@ def files_in(*search_paths):
|
||||
return files
|
||||
|
||||
|
||||
def is_readable_file(file_path):
|
||||
"""Return True if the path passed as argument is readable"""
|
||||
return os.path.isfile(file_path) and os.access(file_path, os.R_OK)
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def search_paths_for_executables(*path_hints):
|
||||
"""Given a list of path hints returns a list of paths where
|
||||
@ -1968,6 +1973,38 @@ def search_paths_for_executables(*path_hints):
|
||||
return executable_paths
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def search_paths_for_libraries(*path_hints):
|
||||
"""Given a list of path hints returns a list of paths where
|
||||
to search for a shared library.
|
||||
|
||||
Args:
|
||||
*path_hints (list of paths): list of paths taken into
|
||||
consideration for a search
|
||||
|
||||
Returns:
|
||||
A list containing the real path of every existing directory
|
||||
in `path_hints` and its `lib` and `lib64` subdirectory if it exists.
|
||||
"""
|
||||
library_paths = []
|
||||
for path in path_hints:
|
||||
if not os.path.isdir(path):
|
||||
continue
|
||||
|
||||
path = os.path.abspath(path)
|
||||
library_paths.append(path)
|
||||
|
||||
lib_dir = os.path.join(path, 'lib')
|
||||
if os.path.isdir(lib_dir):
|
||||
library_paths.append(lib_dir)
|
||||
|
||||
lib64_dir = os.path.join(path, 'lib64')
|
||||
if os.path.isdir(lib64_dir):
|
||||
library_paths.append(lib64_dir)
|
||||
|
||||
return library_paths
|
||||
|
||||
|
||||
@system_path_filter
|
||||
def partition_path(path, entry=None):
|
||||
"""
|
||||
|
||||
@ -91,6 +91,8 @@ def external_find(args):
|
||||
packages_to_check = spack.repo.path.all_packages()
|
||||
|
||||
detected_packages = spack.detection.by_executable(packages_to_check)
|
||||
detected_packages.update(spack.detection.by_library(packages_to_check))
|
||||
|
||||
new_entries = spack.detection.update_configuration(
|
||||
detected_packages, scope=args.scope, buildable=not args.not_buildable
|
||||
)
|
||||
|
||||
@ -3,10 +3,11 @@
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
from .common import DetectedPackage, executable_prefix, update_configuration
|
||||
from .path import by_executable, executables_in_path
|
||||
from .path import by_executable, by_library, executables_in_path
|
||||
|
||||
__all__ = [
|
||||
'DetectedPackage',
|
||||
'by_library',
|
||||
'by_executable',
|
||||
'executables_in_path',
|
||||
'executable_prefix',
|
||||
|
||||
@ -150,6 +150,29 @@ def executable_prefix(executable_dir):
|
||||
return os.sep.join(components[:idx])
|
||||
|
||||
|
||||
def library_prefix(library_dir):
|
||||
"""Given a directory where an library is found, guess the prefix
|
||||
(i.e. the "root" directory of that installation) and return it.
|
||||
|
||||
Args:
|
||||
library_dir: directory where an library is found
|
||||
"""
|
||||
# Given a prefix where an library is found, assuming that prefix
|
||||
# contains /lib/ or /lib64/, strip off the 'lib' or 'lib64' directory
|
||||
# to get a Spack-compatible prefix
|
||||
assert os.path.isdir(library_dir)
|
||||
|
||||
components = library_dir.split(os.sep)
|
||||
if 'lib64' in components:
|
||||
idx = components.index('lib64')
|
||||
return os.sep.join(components[:idx])
|
||||
elif 'lib' in components:
|
||||
idx = components.index('lib')
|
||||
return os.sep.join(components[:idx])
|
||||
else:
|
||||
return library_dir
|
||||
|
||||
|
||||
def update_configuration(detected_packages, scope=None, buildable=True):
|
||||
"""Add the packages passed as arguments to packages.yaml
|
||||
|
||||
|
||||
@ -25,6 +25,7 @@
|
||||
executable_prefix,
|
||||
find_win32_additional_install_paths,
|
||||
is_executable,
|
||||
library_prefix,
|
||||
)
|
||||
|
||||
|
||||
@ -72,6 +73,34 @@ def executables_in_path(path_hints=None):
|
||||
return path_to_exe
|
||||
|
||||
|
||||
def libraries_in_ld_library_path(path_hints=None):
|
||||
"""Get the paths of all libraries available from LD_LIBRARY_PATH.
|
||||
|
||||
For convenience, this is constructed as a dictionary where the keys are
|
||||
the library paths and the values are the names of the libraries
|
||||
(i.e. the basename of the library path).
|
||||
|
||||
There may be multiple paths with the same basename. In this case it is
|
||||
assumed there are two different instances of the library.
|
||||
|
||||
Args:
|
||||
path_hints (list): list of paths to be searched. If None the list will be
|
||||
constructed based on the LD_LIBRARY_PATH environment variable.
|
||||
"""
|
||||
path_hints = path_hints or spack.util.environment.get_path('LD_LIBRARY_PATH')
|
||||
search_paths = llnl.util.filesystem.search_paths_for_libraries(*path_hints)
|
||||
|
||||
path_to_lib = {}
|
||||
# Reverse order of search directories so that a lib in the first
|
||||
# LD_LIBRARY_PATH entry overrides later entries
|
||||
for search_path in reversed(search_paths):
|
||||
for lib in os.listdir(search_path):
|
||||
lib_path = os.path.join(search_path, lib)
|
||||
if llnl.util.filesystem.is_readable_file(lib_path):
|
||||
path_to_lib[lib_path] = lib
|
||||
return path_to_lib
|
||||
|
||||
|
||||
def _group_by_prefix(paths):
|
||||
groups = collections.defaultdict(set)
|
||||
for p in paths:
|
||||
@ -79,6 +108,106 @@ def _group_by_prefix(paths):
|
||||
return groups.items()
|
||||
|
||||
|
||||
# TODO consolidate this with by_executable
|
||||
# Packages should be able to define both .libraries and .executables in the future
|
||||
# determine_spec_details should get all relevant libraries and executables in one call
|
||||
def by_library(packages_to_check, path_hints=None):
|
||||
# Techniques for finding libraries is determined on a per recipe basis in
|
||||
# the determine_version class method. Some packages will extract the
|
||||
# version number from a shared libraries filename.
|
||||
# Other libraries could use the strings function to extract it as described
|
||||
# in https://unix.stackexchange.com/questions/58846/viewing-linux-library-executable-version-info
|
||||
"""Return the list of packages that have been detected on the system,
|
||||
searching by LD_LIBRARY_PATH.
|
||||
|
||||
Args:
|
||||
packages_to_check (list): list of packages to be detected
|
||||
path_hints (list): list of paths to be searched. If None the list will be
|
||||
constructed based on the LD_LIBRARY_PATH environment variable.
|
||||
"""
|
||||
path_to_lib_name = libraries_in_ld_library_path(path_hints=path_hints)
|
||||
lib_pattern_to_pkgs = collections.defaultdict(list)
|
||||
for pkg in packages_to_check:
|
||||
if hasattr(pkg, 'libraries'):
|
||||
for lib in pkg.libraries:
|
||||
lib_pattern_to_pkgs[lib].append(pkg)
|
||||
|
||||
pkg_to_found_libs = collections.defaultdict(set)
|
||||
for lib_pattern, pkgs in lib_pattern_to_pkgs.items():
|
||||
compiled_re = re.compile(lib_pattern)
|
||||
for path, lib in path_to_lib_name.items():
|
||||
if compiled_re.search(lib):
|
||||
for pkg in pkgs:
|
||||
pkg_to_found_libs[pkg].add(path)
|
||||
|
||||
pkg_to_entries = collections.defaultdict(list)
|
||||
resolved_specs = {} # spec -> lib found for the spec
|
||||
|
||||
for pkg, libs in pkg_to_found_libs.items():
|
||||
if not hasattr(pkg, 'determine_spec_details'):
|
||||
llnl.util.tty.warn(
|
||||
"{0} must define 'determine_spec_details' in order"
|
||||
" for Spack to detect externally-provided instances"
|
||||
" of the package.".format(pkg.name))
|
||||
continue
|
||||
|
||||
for prefix, libs_in_prefix in sorted(_group_by_prefix(libs)):
|
||||
try:
|
||||
specs = _convert_to_iterable(
|
||||
pkg.determine_spec_details(prefix, libs_in_prefix)
|
||||
)
|
||||
except Exception as e:
|
||||
specs = []
|
||||
msg = 'error detecting "{0}" from prefix {1} [{2}]'
|
||||
warnings.warn(msg.format(pkg.name, prefix, str(e)))
|
||||
|
||||
if not specs:
|
||||
llnl.util.tty.debug(
|
||||
'The following libraries in {0} were decidedly not '
|
||||
'part of the package {1}: {2}'
|
||||
.format(prefix, pkg.name, ', '.join(
|
||||
_convert_to_iterable(libs_in_prefix)))
|
||||
)
|
||||
|
||||
for spec in specs:
|
||||
pkg_prefix = library_prefix(prefix)
|
||||
|
||||
if not pkg_prefix:
|
||||
msg = "no lib/ or lib64/ dir found in {0}. Cannot "
|
||||
"add it as a Spack package"
|
||||
llnl.util.tty.debug(msg.format(prefix))
|
||||
continue
|
||||
|
||||
if spec in resolved_specs:
|
||||
prior_prefix = ', '.join(
|
||||
_convert_to_iterable(resolved_specs[spec]))
|
||||
|
||||
llnl.util.tty.debug(
|
||||
"Libraries in {0} and {1} are both associated"
|
||||
" with the same spec {2}"
|
||||
.format(prefix, prior_prefix, str(spec)))
|
||||
continue
|
||||
else:
|
||||
resolved_specs[spec] = prefix
|
||||
|
||||
try:
|
||||
spec.validate_detection()
|
||||
except Exception as e:
|
||||
msg = ('"{0}" has been detected on the system but will '
|
||||
'not be added to packages.yaml [reason={1}]')
|
||||
llnl.util.tty.warn(msg.format(spec, str(e)))
|
||||
continue
|
||||
|
||||
if spec.external_path:
|
||||
pkg_prefix = spec.external_path
|
||||
|
||||
pkg_to_entries[pkg.name].append(
|
||||
DetectedPackage(spec=spec, prefix=pkg_prefix)
|
||||
)
|
||||
|
||||
return pkg_to_entries
|
||||
|
||||
|
||||
def by_executable(packages_to_check, path_hints=None):
|
||||
"""Return the list of packages that have been detected on the system,
|
||||
searching by path.
|
||||
|
||||
@ -177,13 +177,21 @@ class DetectablePackageMeta(object):
|
||||
for the detection function.
|
||||
"""
|
||||
def __init__(cls, name, bases, attr_dict):
|
||||
if hasattr(cls, 'executables') and hasattr(cls, 'libraries'):
|
||||
msg = "a package can have either an 'executables' or 'libraries' attribute"
|
||||
msg += " [package '{0.name}' defines both]"
|
||||
raise spack.error.SpackError(msg.format(cls))
|
||||
|
||||
# On windows, extend the list of regular expressions to look for
|
||||
# filenames ending with ".exe"
|
||||
# (in some cases these regular expressions include "$" to avoid
|
||||
# pulling in filenames with unexpected suffixes, but this allows
|
||||
# for example detecting "foo.exe" when the package writer specified
|
||||
# that "foo" was a possible executable.
|
||||
if hasattr(cls, 'executables'):
|
||||
|
||||
# If a package has the executables or libraries attribute then it's
|
||||
# assumed to be detectable
|
||||
if hasattr(cls, 'executables') or hasattr(cls, 'libraries'):
|
||||
@property
|
||||
def platform_executables(self):
|
||||
def to_windows_exe(exe):
|
||||
@ -201,35 +209,37 @@ def to_windows_exe(exe):
|
||||
return plat_exe
|
||||
|
||||
@classmethod
|
||||
def determine_spec_details(cls, prefix, exes_in_prefix):
|
||||
def determine_spec_details(cls, prefix, objs_in_prefix):
|
||||
"""Allow ``spack external find ...`` to locate installations.
|
||||
|
||||
Args:
|
||||
prefix (str): the directory containing the executables
|
||||
exes_in_prefix (set): the executables that match the regex
|
||||
or libraries
|
||||
objs_in_prefix (set): the executables or libraries that
|
||||
match the regex
|
||||
|
||||
Returns:
|
||||
The list of detected specs for this package
|
||||
"""
|
||||
exes_by_version = collections.defaultdict(list)
|
||||
objs_by_version = collections.defaultdict(list)
|
||||
# The default filter function is the identity function for the
|
||||
# list of executables
|
||||
filter_fn = getattr(cls, 'filter_detected_exes',
|
||||
lambda x, exes: exes)
|
||||
exes_in_prefix = filter_fn(prefix, exes_in_prefix)
|
||||
for exe in exes_in_prefix:
|
||||
objs_in_prefix = filter_fn(prefix, objs_in_prefix)
|
||||
for obj in objs_in_prefix:
|
||||
try:
|
||||
version_str = cls.determine_version(exe)
|
||||
version_str = cls.determine_version(obj)
|
||||
if version_str:
|
||||
exes_by_version[version_str].append(exe)
|
||||
objs_by_version[version_str].append(obj)
|
||||
except Exception as e:
|
||||
msg = ('An error occurred when trying to detect '
|
||||
'the version of "{0}" [{1}]')
|
||||
tty.debug(msg.format(exe, str(e)))
|
||||
tty.debug(msg.format(obj, str(e)))
|
||||
|
||||
specs = []
|
||||
for version_str, exes in exes_by_version.items():
|
||||
variants = cls.determine_variants(exes, version_str)
|
||||
for version_str, objs in objs_by_version.items():
|
||||
variants = cls.determine_variants(objs, version_str)
|
||||
# Normalize output to list
|
||||
if not isinstance(variants, list):
|
||||
variants = [variants]
|
||||
@ -265,7 +275,7 @@ def determine_spec_details(cls, prefix, exes_in_prefix):
|
||||
return sorted(specs)
|
||||
|
||||
@classmethod
|
||||
def determine_variants(cls, exes, version_str):
|
||||
def determine_variants(cls, objs, version_str):
|
||||
return ''
|
||||
|
||||
# Register the class as a detectable package
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import re
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
@ -15,6 +17,7 @@ class Hipblas(CMakePackage):
|
||||
url = "https://github.com/ROCmSoftwarePlatform/hipBLAS/archive/rocm-5.0.2.tar.gz"
|
||||
|
||||
maintainers = ['srekolam', 'arjun-raj-kuppala', 'haampie']
|
||||
libraries = ['libhipblas.so']
|
||||
|
||||
version('5.0.2', sha256='201772bfc422ecb2c50e898dccd7d3d376cf34a2b795360e34bf71326aa37646')
|
||||
version('5.0.0', sha256='63cffe748ed4a86fc80f408cb9e8a9c6c55c22a2b65c0eb9a76360b97bbb9d41')
|
||||
@ -54,6 +57,18 @@ def check(self):
|
||||
depends_on('comgr@' + ver, type='build', when='@' + ver)
|
||||
depends_on('rocm-cmake@' + ver, type='build', when='@' + ver)
|
||||
|
||||
@classmethod
|
||||
def determine_version(cls, lib):
|
||||
match = re.search(r'lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)',
|
||||
lib)
|
||||
if match:
|
||||
ver = '{0}.{1}.{2}'.format(int(match.group(1)),
|
||||
int(match.group(2)),
|
||||
int(match.group(3)))
|
||||
else:
|
||||
ver = None
|
||||
return ver
|
||||
|
||||
def cmake_args(self):
|
||||
args = [
|
||||
# Make sure find_package(HIP) finds the module.
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import re
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
@ -13,6 +15,7 @@ class Nccl(MakefilePackage, CudaPackage):
|
||||
url = "https://github.com/NVIDIA/nccl/archive/v2.7.3-1.tar.gz"
|
||||
|
||||
maintainers = ['adamjstewart']
|
||||
libraries = ['libnccl.so']
|
||||
|
||||
version('2.11.4-1', sha256='db4e9a0277a64f9a31ea9b5eea22e63f10faaed36dded4587bbc8a0d8eceed10')
|
||||
version('2.10.3-1', sha256='55de166eb7dcab9ecef2629cdb5fb0c5ebec4fae03589c469ebe5dcb5716b3c5')
|
||||
@ -49,6 +52,12 @@ class Nccl(MakefilePackage, CudaPackage):
|
||||
msg='Must specify CUDA compute capabilities of your GPU, see '
|
||||
'https://developer.nvidia.com/cuda-gpus')
|
||||
|
||||
@classmethod
|
||||
def determine_version(cls, lib):
|
||||
match = re.search(r'lib\S*\.so\.(\d+\.\d+\.\d+)',
|
||||
lib)
|
||||
return match.group(1) if match else None
|
||||
|
||||
@property
|
||||
def build_targets(self):
|
||||
cuda_arch = self.spec.variants['cuda_arch'].value
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import re
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
@ -17,6 +19,7 @@ class Rccl(CMakePackage):
|
||||
url = "https://github.com/ROCmSoftwarePlatform/rccl/archive/rocm-5.0.0.tar.gz"
|
||||
|
||||
maintainers = ['srekolam', 'arjun-raj-kuppala']
|
||||
libraries = ['librccl.so']
|
||||
|
||||
version('5.0.2', sha256='a2377ad2332b93d3443a8ee74f4dd9f965ae8cbbfad473f8f57ca17905389a39')
|
||||
version('5.0.0', sha256='80eb70243f11b80e215458a67c278cd5a655f6e486289962b92ba3504e50af5c')
|
||||
@ -53,6 +56,18 @@ class Rccl(CMakePackage):
|
||||
for ver in ['4.5.0', '4.5.2', '5.0.0', '5.0.2']:
|
||||
depends_on('rocm-smi-lib@' + ver, when='@' + ver)
|
||||
|
||||
@classmethod
|
||||
def determine_version(cls, lib):
|
||||
match = re.search(r'lib\S*\.so\.\d+\.\d+\.(\d)(\d\d)(\d\d)',
|
||||
lib)
|
||||
if match:
|
||||
ver = '{0}.{1}.{2}'.format(int(match.group(1)),
|
||||
int(match.group(2)),
|
||||
int(match.group(3)))
|
||||
else:
|
||||
ver = None
|
||||
return ver
|
||||
|
||||
def setup_build_environment(self, env):
|
||||
env.set('CXX', self.spec['hip'].hipcc)
|
||||
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
|
||||
import re
|
||||
|
||||
from spack import *
|
||||
|
||||
|
||||
@ -11,6 +13,7 @@ class RdmaCore(CMakePackage):
|
||||
|
||||
homepage = "https://github.com/linux-rdma/rdma-core"
|
||||
url = "https://github.com/linux-rdma/rdma-core/releases/download/v17.1/rdma-core-17.1.tar.gz"
|
||||
libraries = ['librdmacm.so']
|
||||
|
||||
version('39.0', sha256='f6eaf0de9fe386e234e00a18a553f591143f50e03342c9fdd703fa8747bf2378')
|
||||
version('34.0', sha256='3d9ccf66468cf78f4c39bebb8bd0c5eb39150ded75f4a88a3455c4f625408be8')
|
||||
@ -34,6 +37,12 @@ class RdmaCore(CMakePackage):
|
||||
conflicts('platform=darwin', msg='rdma-core requires FreeBSD or Linux')
|
||||
conflicts('%intel', msg='rdma-core cannot be built with intel (use gcc instead)')
|
||||
|
||||
@classmethod
|
||||
def determine_version(cls, lib):
|
||||
match = re.search(r'lib\S*\.so\.\d+\.\d+\.(\d+\.\d+)',
|
||||
lib)
|
||||
return match.group(1) if match else None
|
||||
|
||||
# NOTE: specify CMAKE_INSTALL_RUNDIR explicitly to prevent rdma-core from
|
||||
# using the spack staging build dir (which may be a very long file
|
||||
# system path) as a component in compile-time static strings such as
|
||||
|
||||
Loading…
Reference in New Issue
Block a user