libc: from current python process (#43787)

If there's no compiler we currently don't have any external libc for the solver.

This commit adds a fallback on libc from the current Python process, which works if it is dynamically linked.

Co-authored-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

lib/spack/spack/compiler.py

@@ -12,7 +12,6 @@
 import shutil
 import sys
 import tempfile
-from subprocess import PIPE, run
 from typing import List, Optional, Sequence
 
 import llnl.path
@@ -24,6 +23,7 @@
 import spack.error
 import spack.spec
 import spack.util.executable
+import spack.util.libc
 import spack.util.module_cmd
 import spack.version
 from spack.util.environment import filter_system_paths
@@ -197,98 +197,6 @@ def _parse_dynamic_linker(output: str):
                 return arg.split("=", 1)[1]
 
 
-def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]:
-    try:
-        result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False)
-        stdout = result.stdout.decode("utf-8")
-    except Exception:
-        return None
-
-    if not re.search("gnu|glibc", stdout, re.IGNORECASE):
-        return None
-
-    version_str = re.match(r".+\(.+\) (.+)", stdout)
-    if not version_str:
-        return None
-    try:
-        return spack.spec.Spec(f"glibc@={version_str.group(1)}")
-    except Exception:
-        return None
-
-
-def _libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]:
-    if not os.path.exists(dynamic_linker):
-        return None
-
-    # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all
-    # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then
-    # a symlink into /usr/lib, which we use to for determining the actual install prefix of
-    # libc.
-    realpath = os.path.realpath(dynamic_linker)
-
-    prefix = os.path.dirname(realpath)
-    # Remove the multiarch suffix if it exists
-    if os.path.basename(prefix) not in ("lib", "lib64"):
-        prefix = os.path.dirname(prefix)
-
-    # Non-standard install layout -- just bail.
-    if os.path.basename(prefix) not in ("lib", "lib64"):
-        return None
-
-    prefix = os.path.dirname(prefix)
-
-    # Now try to figure out if glibc or musl, which is the only ones we support.
-    # In recent glibc we can simply execute the dynamic loader. In musl that's always the case.
-    try:
-        result = run([dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False)
-        stdout = result.stdout.decode("utf-8")
-        stderr = result.stderr.decode("utf-8")
-    except Exception:
-        return None
-
-    # musl prints to stderr
-    if stderr.startswith("musl libc"):
-        version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE)
-        if not version_str:
-            return None
-        try:
-            spec = spack.spec.Spec(f"musl@={version_str.group(1)}")
-            spec.external_path = prefix
-            return spec
-        except Exception:
-            return None
-    elif re.search("gnu|glibc", stdout, re.IGNORECASE):
-        # output is like "ld.so (...) stable release version 2.33." write a regex for it
-        match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout)
-        if not match:
-            return None
-        try:
-            version = match.group(1)
-            spec = spack.spec.Spec(f"glibc@={version}")
-            spec.external_path = prefix
-            return spec
-        except Exception:
-            return None
-    else:
-        # Could not get the version by running the dynamic linker directly. Instead locate `ldd`
-        # relative to the dynamic linker.
-        ldd = os.path.join(prefix, "bin", "ldd")
-        if not os.path.exists(ldd):
-            # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as
-            # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib.
-            if prefix != "/":
-                return None
-            prefix = "/usr"
-            ldd = os.path.join(prefix, "bin", "ldd")
-            if not os.path.exists(ldd):
-                return None
-        maybe_spec = _libc_from_ldd(ldd)
-        if not maybe_spec:
-            return None
-        maybe_spec.external_path = prefix
-        return maybe_spec
-
-
 def in_system_subdirectory(path):
     system_dirs = [
         "/lib/",
@@ -536,7 +444,9 @@ def implicit_rpaths(self) -> List[str]:
         all_required_libs = list(self.required_libs) + Compiler._all_compiler_rpath_libraries
         return list(paths_containing_libs(link_dirs, all_required_libs))
 
+    @property
     def default_libc(self) -> Optional["spack.spec.Spec"]:
+        """Determine libc targeted by the compiler from link line"""
         output = self.compiler_verbose_output
 
         if not output:
@@ -547,7 +457,7 @@ def default_libc(self) -> Optional["spack.spec.Spec"]:
         if not dynamic_linker:
             return None
 
-        return _libc_from_dynamic_linker(dynamic_linker)
+        return spack.util.libc.libc_from_dynamic_linker(dynamic_linker)
 
     @property
     def required_libs(self):

lib/spack/spack/solver/asp.py

@@ -41,6 +41,8 @@
 import spack.spec
 import spack.store
 import spack.util.crypto
+import spack.util.elf
+import spack.util.libc
 import spack.util.path
 import spack.util.timer
 import spack.variant
@@ -283,20 +285,27 @@ def all_compilers_in_config(configuration):
     return spack.compilers.all_compilers_from(configuration)
 
 
-def compatible_libc(candidate_libc_spec):
-    """Returns a list of libc specs that are compatible with the one passed as argument"""
-    result = set()
-    for compiler in all_compilers_in_config(spack.config.CONFIG):
-        libc = compiler.default_libc()
-        if not libc:
-            continue
-        if (
-            libc.name == candidate_libc_spec.name
-            and libc.version >= candidate_libc_spec.version
-            and libc.external_path == candidate_libc_spec.external_path
-        ):
-            result.add(libc)
-    return sorted(result)
+def all_libcs() -> Set[spack.spec.Spec]:
+    """Return a set of all libc specs targeted by any configured compiler. If none, fall back to
+    libc determined from the current Python process if dynamically linked."""
+
+    libcs = {
+        c.default_libc for c in all_compilers_in_config(spack.config.CONFIG) if c.default_libc
+    }
+
+    if libcs:
+        return libcs
+
+    libc = spack.util.libc.libc_from_current_python_process()
+    return {libc} if libc else set()
+
+
+def libc_is_compatible(lhs: spack.spec.Spec, rhs: spack.spec.Spec) -> List[spack.spec.Spec]:
+    return (
+        lhs.name == rhs.name
+        and lhs.external_path == rhs.external_path
+        and lhs.version >= rhs.version
+    )
 
 
 def using_libc_compatibility() -> bool:
@@ -597,7 +606,7 @@ def _external_config_with_implicit_externals(configuration):
         return packages_yaml
 
     for compiler in all_compilers_in_config(configuration):
-        libc = compiler.default_libc()
+        libc = compiler.default_libc
         if libc:
             entry = {"spec": f"{libc} %{compiler.spec}", "prefix": libc.external_path}
             packages_yaml.setdefault(libc.name, {}).setdefault("externals", []).append(entry)
@@ -1028,6 +1037,9 @@ def __init__(self, tests: bool = False):
         self.pkgs: Set[str] = set()
         self.explicitly_required_namespaces: Dict[str, str] = {}
 
+        # list of unique libc specs targeted by compilers (or an educated guess if no compiler)
+        self.libcs: List[spack.spec.Spec] = []
+
     def pkg_version_rules(self, pkg):
         """Output declared versions of a package.
 
@@ -1872,13 +1884,14 @@ def _spec_clauses(
                     if dep.name == "gcc-runtime":
                         continue
 
-                    # LIBC is also solved again by clingo, but in this case the compatibility
+                    # libc is also solved again by clingo, but in this case the compatibility
                     # is not encoded in the parent node - so we need to emit explicit facts
                     if "libc" in dspec.virtuals:
-                        for x in compatible_libc(dep):
-                            clauses.append(
-                                fn.attr("compatible_libc", spec.name, x.name, x.version)
-                            )
+                        for libc in self.libcs:
+                            if libc_is_compatible(libc, dep):
+                                clauses.append(
+                                    fn.attr("compatible_libc", spec.name, libc.name, libc.version)
+                                )
                         continue
 
                     # We know dependencies are real for concrete specs. For abstract
@@ -2336,6 +2349,7 @@ def setup(
         node_counter = _create_counter(specs, tests=self.tests)
         self.possible_virtuals = node_counter.possible_virtuals()
         self.pkgs = node_counter.possible_dependencies()
+        self.libcs = sorted(all_libcs())  # type: ignore[type-var]
 
         # Fail if we already know an unreachable node is requested
         for spec in specs:
@@ -2345,16 +2359,16 @@ def setup(
             if missing_deps:
                 raise spack.spec.InvalidDependencyError(spec.name, missing_deps)
 
-        for node in spack.traverse.traverse_nodes(specs):
+        for node in traverse.traverse_nodes(specs):
             if node.namespace is not None:
                 self.explicitly_required_namespaces[node.name] = node.namespace
 
         self.gen = ProblemInstanceBuilder()
         compiler_parser = CompilerParser(configuration=spack.config.CONFIG).with_input_specs(specs)
 
-        # Only relevant for linux
-        for libc in compiler_parser.allowed_libcs:
-            self.gen.fact(fn.allowed_libc(libc.name, libc.version))
+        if using_libc_compatibility():
+            for libc in self.libcs:
+                self.gen.fact(fn.allowed_libc(libc.name, libc.version))
 
         if not allow_deprecated:
             self.gen.fact(fn.deprecated_versions_not_allowed())
@@ -2505,15 +2519,16 @@ def define_runtime_constraints(self):
             if not compiler.available:
                 continue
 
-            if using_libc_compatibility():
-                libc = compiler.compiler_obj.default_libc()
-                if libc:
-                    recorder("*").depends_on(
-                        "libc", when=f"%{compiler.spec}", type="link", description="Add libc"
-                    )
-                    recorder("*").depends_on(
-                        str(libc), when=f"%{compiler.spec}", type="link", description="Add libc"
-                    )
+            if using_libc_compatibility() and compiler.compiler_obj.default_libc:
+                recorder("*").depends_on(
+                    "libc", when=f"%{compiler.spec}", type="link", description="Add libc"
+                )
+                recorder("*").depends_on(
+                    str(compiler.compiler_obj.default_libc),
+                    when=f"%{compiler.spec}",
+                    type="link",
+                    description="Add libc",
+                )
 
         recorder.consume_facts()
 
@@ -2890,18 +2905,13 @@ class CompilerParser:
 
     def __init__(self, configuration) -> None:
         self.compilers: Set[KnownCompiler] = set()
-        self.allowed_libcs = set()
         for c in all_compilers_in_config(configuration):
-            if using_libc_compatibility():
-                libc = c.default_libc()
-                if not libc:
-                    warnings.warn(
-                        f"cannot detect libc from {c.spec}. The compiler will not be used "
-                        f"during concretization."
-                    )
-                    continue
-
-                self.allowed_libcs.add(libc)
+            if using_libc_compatibility() and not c.default_libc:
+                warnings.warn(
+                    f"cannot detect libc from {c.spec}. The compiler will not be used "
+                    f"during concretization."
+                )
+                continue
 
             target = c.target if c.target != "any" else None
             candidate = KnownCompiler(

lib/spack/spack/solver/concretize.lp

@@ -1082,6 +1082,9 @@ error(100, "{0} compiler '{2}@{3}' incompatible with 'target={1}'", Package, Tar
      compiler_version(CompilerID, Version),
      build(node(X, Package)).
 
+#defined compiler_supports_target/2.
+#defined compiler_available/1.
+
 % if a target is set explicitly, respect it
 attr("node_target", PackageNode, Target)
  :- attr("node", PackageNode), attr("node_target_set", PackageNode, Target).

lib/spack/spack/test/concretize.py

@@ -83,7 +83,7 @@ def binary_compatibility(monkeypatch, request):
         return
 
     monkeypatch.setattr(spack.solver.asp, "using_libc_compatibility", lambda: True)
-    monkeypatch.setattr(spack.compiler.Compiler, "default_libc", lambda x: Spec("glibc@=2.28"))
+    monkeypatch.setattr(spack.compiler.Compiler, "default_libc", Spec("glibc@=2.28"))
 
 
 @pytest.fixture(

lib/spack/spack/util/elf.py

@@ -641,6 +641,20 @@ def substitute_rpath_and_pt_interp_in_place_or_raise(
         return False
 
 
+def pt_interp(path: str) -> Optional[str]:
+    """Retrieve the interpreter of an executable at `path`."""
+    try:
+        with open(path, "rb") as f:
+            elf = parse_elf(f, interpreter=True)
+    except (OSError, ElfParsingError):
+        return None
+
+    if not elf.has_pt_interp:
+        return None
+
+    return elf.pt_interp_str.decode("utf-8")
+
+
 class ElfCStringUpdatesFailed(Exception):
     def __init__(
         self, rpath: Optional[UpdateCStringAction], pt_interp: Optional[UpdateCStringAction]

lib/spack/spack/util/libc.py

@@ -0,0 +1,117 @@
+# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+import os
+import re
+import sys
+from subprocess import PIPE, run
+from typing import Optional
+
+import spack.spec
+import spack.util.elf
+
+
+def _libc_from_ldd(ldd: str) -> Optional["spack.spec.Spec"]:
+    try:
+        result = run([ldd, "--version"], stdout=PIPE, stderr=PIPE, check=False)
+        stdout = result.stdout.decode("utf-8")
+    except Exception:
+        return None
+
+    if not re.search("gnu|glibc", stdout, re.IGNORECASE):
+        return None
+
+    version_str = re.match(r".+\(.+\) (.+)", stdout)
+    if not version_str:
+        return None
+    try:
+        return spack.spec.Spec(f"glibc@={version_str.group(1)}")
+    except Exception:
+        return None
+
+
+def libc_from_dynamic_linker(dynamic_linker: str) -> Optional["spack.spec.Spec"]:
+    if not os.path.exists(dynamic_linker):
+        return None
+
+    # The dynamic linker is usually installed in the same /lib(64)?/ld-*.so path across all
+    # distros. The rest of libc is elsewhere, e.g. /usr. Typically the dynamic linker is then
+    # a symlink into /usr/lib, which we use to for determining the actual install prefix of
+    # libc.
+    realpath = os.path.realpath(dynamic_linker)
+
+    prefix = os.path.dirname(realpath)
+    # Remove the multiarch suffix if it exists
+    if os.path.basename(prefix) not in ("lib", "lib64"):
+        prefix = os.path.dirname(prefix)
+
+    # Non-standard install layout -- just bail.
+    if os.path.basename(prefix) not in ("lib", "lib64"):
+        return None
+
+    prefix = os.path.dirname(prefix)
+
+    # Now try to figure out if glibc or musl, which is the only ones we support.
+    # In recent glibc we can simply execute the dynamic loader. In musl that's always the case.
+    try:
+        result = run([dynamic_linker, "--version"], stdout=PIPE, stderr=PIPE, check=False)
+        stdout = result.stdout.decode("utf-8")
+        stderr = result.stderr.decode("utf-8")
+    except Exception:
+        return None
+
+    # musl prints to stderr
+    if stderr.startswith("musl libc"):
+        version_str = re.search(r"^Version (.+)$", stderr, re.MULTILINE)
+        if not version_str:
+            return None
+        try:
+            spec = spack.spec.Spec(f"musl@={version_str.group(1)}")
+            spec.external_path = prefix
+            return spec
+        except Exception:
+            return None
+    elif re.search("gnu|glibc", stdout, re.IGNORECASE):
+        # output is like "ld.so (...) stable release version 2.33." write a regex for it
+        match = re.search(r"version (\d+\.\d+(?:\.\d+)?)", stdout)
+        if not match:
+            return None
+        try:
+            version = match.group(1)
+            spec = spack.spec.Spec(f"glibc@={version}")
+            spec.external_path = prefix
+            return spec
+        except Exception:
+            return None
+    else:
+        # Could not get the version by running the dynamic linker directly. Instead locate `ldd`
+        # relative to the dynamic linker.
+        ldd = os.path.join(prefix, "bin", "ldd")
+        if not os.path.exists(ldd):
+            # If `/lib64/ld.so` was not a symlink to `/usr/lib/ld.so` we can try to use /usr as
+            # prefix. This is the case on ubuntu 18.04 where /lib != /usr/lib.
+            if prefix != "/":
+                return None
+            prefix = "/usr"
+            ldd = os.path.join(prefix, "bin", "ldd")
+            if not os.path.exists(ldd):
+                return None
+        maybe_spec = _libc_from_ldd(ldd)
+        if not maybe_spec:
+            return None
+        maybe_spec.external_path = prefix
+        return maybe_spec
+
+
+def libc_from_current_python_process() -> Optional["spack.spec.Spec"]:
+    if not sys.executable:
+        return None
+
+    dynamic_linker = spack.util.elf.pt_interp(sys.executable)
+
+    if not dynamic_linker:
+        return None
+
+    return libc_from_dynamic_linker(dynamic_linker)