python: allow it as a build-tool again (#49201)

Python was removed from being a build tool in #46980, due to issues
when reusing specs. This PR adds a new rule to match the interpreter
among different Python packages, in clingo.

It also adds a bunch of new "build-tools", so that specs like:
```
py-matplotlib backend=tkagg
```
can be concretized in one go.

Modifications:
- [x] Make `py-matplotlib backend=tkagg` concretizable
- [x] Add unit-tests to ensure situations like in #46980 do not happen

---------

Signed-off-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

etc/spack/defaults/concretizer.yaml

@@ -54,9 +54,15 @@ concretizer:
       # Regular packages
       cmake: 2
       gmake: 2
+      python: 2
+      python-venv: 2
       py-cython: 2
       py-flit-core: 2
+      py-pip: 2
       py-setuptools: 2
+      py-wheel: 2
+      xcb-proto: 2
+      # Compilers
       gcc: 2
       llvm: 2
   # Option to specify compatibility between operating systems for reuse of compilers and packages

lib/spack/spack/solver/asp.py

@@ -1658,13 +1658,16 @@ def track_dependencies(input_spec, requirements):
                     return requirements + [fn.attr("track_dependencies", input_spec.name)]
 
                 def dependency_holds(input_spec, requirements):
-                    return remove_node(input_spec, requirements) + [
+                    result = remove_node(input_spec, requirements) + [
                         fn.attr(
                             "dependency_holds", pkg.name, input_spec.name, dt.flag_to_string(t)
                         )
                         for t in dt.ALL_FLAGS
                         if t & depflag
                     ]
+                    if input_spec.name not in pkg.extendees:
+                        return result
+                    return result + [fn.attr("extends", pkg.name, input_spec.name)]
 
                 context = ConditionContext()
                 context.source = ConstraintOrigin.append_type_suffix(

lib/spack/spack/solver/concretize.lp

@@ -524,6 +524,16 @@ error(10, "'{0}' is not a valid dependency for any package in the DAG", Package)
   :- attr("node", node(ID, Package)),
      not needed(node(ID, Package)).
 
+
+% Extensions depending on each other must all extend the same node (e.g. all Python packages
+% depending on each other must depend on the same Python interpreter)
+error(100, "{0} and {1} must depend on the same {2}", ExtensionParent, ExtensionChild, ExtendeePackage)
+  :- depends_on(ExtensionParent, ExtensionChild),
+     attr("extends", ExtensionParent, ExtendeePackage),
+     depends_on(ExtensionParent, node(X, ExtendeePackage)),
+     depends_on(ExtensionChild,  node(Y, ExtendeePackage)),
+     X != Y.
+
 #defined dependency_type/2.
 
 %-----------------------------------------------------------------------------

lib/spack/spack/test/concretization/core.py

@@ -2888,6 +2888,23 @@ def test_specifying_different_versions_build_deps(self):
         assert any(x.satisfies(hdf5_str) for x in result.specs)
         assert any(x.satisfies(pinned_str) for x in result.specs)
 
+    @pytest.mark.regression("44289")
+    def test_all_extensions_depend_on_same_extendee(self):
+        """Tests that we don't reuse dependencies that bring in a different extendee"""
+        setuptools = spack.concretize.concretize_one("py-setuptools ^python@3.10")
+
+        solver = spack.solver.asp.Solver()
+        setup = spack.solver.asp.SpackSolverSetup()
+        result, _, _ = solver.driver.solve(
+            setup, [Spec("py-floating ^python@3.11")], reuse=list(setuptools.traverse())
+        )
+        assert len(result.specs) == 1
+
+        floating = result.specs[0]
+        assert all(setuptools.dag_hash() != x.dag_hash() for x in floating.traverse())
+        pythons = [x for x in floating.traverse() if x.name == "python"]
+        assert len(pythons) == 1 and pythons[0].satisfies("@3.11")
+
 
 @pytest.mark.parametrize(
     "v_str,v_opts,checksummed",

var/spack/repos/builtin/packages/libxcb/package.py

@@ -16,6 +16,8 @@ class Libxcb(AutotoolsPackage, XorgPackage):
 
     license("MIT")
 
+    tags = ["build-tools"]
+
     maintainers("wdconinc")
 
     version("1.17.0", sha256="599ebf9996710fea71622e6e184f3a8ad5b43d0e5fa8c4e407123c88a59a6d55")

var/spack/repos/builtin/packages/py-wheel/package.py

@@ -13,6 +13,8 @@ class PyWheel(Package, PythonExtension):
     url = "https://files.pythonhosted.org/packages/py3/w/wheel/wheel-0.41.2-py3-none-any.whl"
     list_url = "https://pypi.org/simple/wheel/"
 
+    tags = ["build-tools"]
+
     version("0.41.2", sha256="75909db2664838d015e3d9139004ee16711748a52c8f336b52882266540215d8")
     version("0.37.1", sha256="4bdcd7d840138086126cd09254dc6195fb4fc6f01c050a1d7236f2630db1d22a")
     version("0.37.0", sha256="21014b2bd93c6d0034b6ba5d35e4eb284340e09d63c59aef6fc14b0f346146fd")

var/spack/repos/builtin/packages/python-venv/package.py

@@ -14,6 +14,8 @@ class PythonVenv(Package):
     homepage = "https://docs.python.org/3/library/venv.html"
     has_code = False
 
+    tags = ["build-tools"]
+
     maintainers("haampie")
 
     version("1.0")

var/spack/repos/builtin/packages/python/package.py

@@ -45,7 +45,7 @@ class Python(Package):
     url = "https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tgz"
     list_url = "https://www.python.org/ftp/python/"
     list_depth = 1
-    tags = ["windows"]
+    tags = ["windows", "build-tools"]
 
     maintainers("skosukhin", "scheibelp")
 

var/spack/repos/builtin/packages/xcb-proto/package.py

@@ -14,6 +14,8 @@ class XcbProto(AutotoolsPackage, XorgPackage):
 
     license("MIT")
 
+    tags = ["build-tools"]
+
     maintainers("wdconinc")
 
     version("1.17.0", sha256="2c1bacd2110f4799f74de6ebb714b94cf6f80fb112316b1219480fd22562148c")

var/spack/repos/duplicates.test/packages/python/package.py

@@ -10,7 +10,10 @@ class Python(Package):
     homepage = "http://www.example.com"
     url = "http://www.example.com/tdep-1.0.tar.gz"
 
+    tags = ["build-tools"]
+
     version("3.11.2", md5="0123456789abcdef0123456789abcdef")
+    version("3.10.6", md5="0123456789abcdef0123456789abcdef")
 
     extendable = True