depfile: variable with all identifiers (#34678)

With the new variable [prefix/]SPACK_PACKAGE_IDS you can conveniently execute
things after each successful install.

For example push just-built packages to a buildcache

```
SPACK ?= spack
export SPACK_COLOR = always
MAKEFLAGS += -Orecurse
MY_BUILDCACHE := $(CURDIR)/cache

.PHONY: all clean

all: push

ifeq (,$(filter clean,$(MAKECMDGOALS)))
include env.mk
endif

# the relevant part: push has *all* example/push/<pkg identifier> as prereqs
push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))
	$(SPACK) -e . buildcache update-index --directory $(MY_BUILDCACHE)
	$(info Pushed everything, yay!)

# and each example/push/<pkg identifier> has the install target as prereq,
# and the body can use target local $(HASH) and $(SPEC) variables to do
# things, such as pushing to a build cache
example/push/%: example/install/%
	@mkdir -p $(dir $@)
	$(SPACK) -e . buildcache create --allow-root --only=package --unsigned --directory $(MY_BUILDCACHE) /$(HASH) # push $(SPEC)
	@touch $@

spack.lock: spack.yaml
	$(SPACK) -e . concretize -f

env.mk: spack.lock
	$(SPACK) -e . env depfile -o $@ --make-target-prefix example

clean:
	rm -rf spack.lock env.mk example/
``

lib/spack/docs/environments.rst

@@ -1090,3 +1090,51 @@ output (``spack install --verbose``) while its dependencies are installed silent
 
    # Install the root spec with verbose output.
    $ make -j16 install/python-3.11.0-<hash> SPACK_INSTALL_FLAGS=--verbose
+
+^^^^^^^^^^^^^^^^^^^^^^^^^
+Adding post-install hooks
+^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Another advanced use-case of generated ``Makefile``\s is running a post-install
+command for each package. These "hooks" could be anything from printing a
+post-install message, running tests, or pushing just-built binaries to a buildcache.
+
+This can be accomplished through the generated ``[<prefix>/]SPACK_PACKAGE_IDS``
+variable. Assuming we have an active and concrete environment, we generate the
+associated ``Makefile`` with a prefix ``example``:
+
+.. code:: console
+
+   $ spack env depfile -o env.mk --make-target-prefix example
+
+And we now include it in a different ``Makefile``, in which we create a target
+``example/push/%`` with ``%`` referring to a package identifier. This target
+depends on the particular package installation. In this target we automatically
+have the target-specific ``HASH`` and ``SPEC`` variables at our disposal. They
+are respectively the spec hash (excluding leading ``/``), and a human-readable spec.
+Finally, we have an entrypoint target ``push`` that will update the buildcache
+index once every package is pushed. Note how this target uses the generated
+``example/SPACK_PACKAGE_IDS`` variable to define its prerequisites.
+
+.. code:: Makefile
+
+   SPACK ?= spack
+   BUILDCACHE_DIR = $(CURDIR)/tarballs
+   
+   .PHONY: all
+   
+   all: push
+   
+   include env.mk
+   
+   example/push/%: example/install/%
+   	@mkdir -p $(dir $@)
+   	$(info About to push $(SPEC) to a buildcache)
+   	$(SPACK) -e . buildcache create --allow-root --only=package --directory $(BUILDCACHE_DIR) /$(HASH)
+   	@touch $@
+   
+   push: $(addprefix example/push/,$(example/SPACK_PACKAGE_IDS))
+   	$(info Updating the buildcache index)
+   	$(SPACK) -e . buildcache update-index --directory $(BUILDCACHE_DIR)
+   	$(info Done!)
+   	@touch $@

lib/spack/spack/cmd/env.py

@@ -735,6 +735,18 @@ def get_install_deps_target(name):
     # Root specs without deps are the prereqs for the environment target
     root_install_targets = [get_install_target(h.format("{name}-{version}-{hash}")) for h in roots]
 
+    all_pkg_identifiers = []
+
+    # The SPACK_PACKAGE_IDS variable is "exported", which can be used when including
+    # generated makefiles to add post-install hooks, like pushing to a buildcache,
+    # running tests, etc.
+    # NOTE: GNU Make allows directory separators in variable names, so for consistency
+    # we can namespace this variable with the same prefix as targets.
+    if args.make_target_prefix is None:
+        pkg_identifier_variable = "SPACK_PACKAGE_IDS"
+    else:
+        pkg_identifier_variable = os.path.join(target_prefix, "SPACK_PACKAGE_IDS")
+
     # All install and install-deps targets
     all_install_related_targets = []
 
@@ -744,6 +756,7 @@ def get_install_deps_target(name):
     phony_convenience_targets = []
 
     for tgt, _, _, _, _ in make_targets.adjacency_list:
+        all_pkg_identifiers.append(tgt)
         all_install_related_targets.append(get_install_target(tgt))
         all_install_related_targets.append(get_install_deps_target(tgt))
         if args.make_target_prefix is None:
@@ -770,6 +783,8 @@ def get_install_deps_target(name):
             "adjacency_list": make_targets.adjacency_list,
             "phony_convenience_targets": " ".join(phony_convenience_targets),
             "target_prefix": target_prefix,
+            "pkg_ids_variable": pkg_identifier_variable,
+            "pkg_ids": " ".join(all_pkg_identifiers),
         }
     )
 

lib/spack/spack/test/cmd/env.py

@@ -3146,6 +3146,54 @@ def test_environment_depfile_out(tmpdir, mock_packages):
             assert stdout == f.read()
 
 
+def test_spack_package_ids_variable(tmpdir, mock_packages):
+    # Integration test for post-install hooks through prefix/SPACK_PACKAGE_IDS
+    # variable
+    env("create", "test")
+    makefile_path = str(tmpdir.join("Makefile"))
+    include_path = str(tmpdir.join("include.mk"))
+
+    # Create env and generate depfile in include.mk with prefix example/
+    with ev.read("test"):
+        add("libdwarf")
+        concretize()
+
+    with ev.read("test"):
+        env(
+            "depfile",
+            "-G",
+            "make",
+            "--make-disable-jobserver",
+            "--make-target-prefix=example",
+            "-o",
+            include_path,
+        )
+
+    # Include in Makefile and create target that depend on SPACK_PACKAGE_IDS
+    with open(makefile_path, "w") as f:
+        f.write(
+            r"""
+all: post-install
+
+include include.mk
+
+example/post-install/%: example/install/%
+	$(info post-install: $(HASH)) # noqa: W191,E101
+
+post-install: $(addprefix example/post-install/,$(example/SPACK_PACKAGE_IDS))
+"""
+        )
+    make = Executable("make")
+
+    # Do dry run.
+    out = make("-n", "-C", str(tmpdir), output=str)
+
+    # post-install: <hash> should've been executed
+    with ev.read("test") as test:
+        for s in test.all_specs():
+            assert "post-install: {}".format(s.dag_hash()) in out
+
+
 def test_unify_when_possible_works_around_conflicts():
     e = ev.create("coconcretization")
     e.unify = "when_possible"

share/spack/templates/depfile/Makefile

@@ -1,6 +1,9 @@
 SPACK ?= spack
 SPACK_INSTALL_FLAGS ?=
 
+# This variable can be used to add post install hooks
+{{ pkg_ids_variable }} := {{ pkg_ids }}
+
 .PHONY: {{ all_target }} {{ clean_target }}
 
 {{ all_target }}: {{ env_target }}