env depfile: allow deps only install (#33245)

* env depfile: allow deps only install

- Refactor `spack env depfile` to use a Jinja template, making it a bit
  easier to follow as a human being.
- Add a layer of indirection in the generated Makefile through an
  `<prefix>/.install-deps/<hash>` target, which allows one to specify
  different options when installing dependencies. For example, only
  verbose/debug mode on when installing some particular spec:
  ```
  $ spack -e my_env env depfile -o Makefile --make-target-prefix example
  $ make example/.install-deps/<hash> -j16
  $ make example/.install/<hash> SPACK="spack -d" SPACK_INSTALL_FLAGS=--verbose -j16
  ```

This could be used to speed up `spack ci rebuild`:
- Parallel install of dependencies from buildcache
- Better readability of logs, e.g. reducing verbosity when installing
  dependencies, and splitting logs into deps.log and current_spec.log

* Silence please!

lib/spack/spack/cmd/env.py

@@ -24,6 +24,7 @@
 import spack.environment as ev
 import spack.environment.shell
 import spack.schema.env
+import spack.tengine
 import spack.util.string as string
 from spack.util.environment import EnvironmentModifications
 
@@ -641,6 +642,9 @@ def get_target(name):
     def get_install_target(name):
         return os.path.join(target_prefix, ".install", name)
 
+    def get_install_deps_target(name):
+        return os.path.join(target_prefix, ".install-deps", name)
+
     for _, spec in env.concretized_specs():
         for s in spec.traverse(root=True):
             hash_to_spec[s.dag_hash()] = s
@@ -655,76 +659,38 @@ def get_install_target(name):
 
     # All package install targets, not just roots.
     all_install_targets = [get_install_target(h) for h in hash_to_spec.keys()]
+    all_install_deps_targets = [get_install_deps_target(h) for h, _ in hash_to_prereqs.items()]
 
     buf = six.StringIO()
 
-    buf.write(
+    template = spack.tengine.make_environment().get_template(os.path.join("depfile", "Makefile"))
-        """SPACK ?= spack
 
-.PHONY: {} {}
-
-{}: {}
-
-{}: {}
-\t@touch $@
-
-{}:
-\t@mkdir -p {}
-
-{}: | {}
-\t$(info Installing $(SPEC))
-\t{}$(SPACK) -e '{}' install $(SPACK_INSTALL_FLAGS) --only-concrete --only=package \
---no-add /$(notdir $@) && touch $@
-
-""".format(
-            get_target("all"),
-            get_target("clean"),
-            get_target("all"),
-            get_target("env"),
-            get_target("env"),
-            " ".join(root_install_targets),
-            get_target("dirs"),
-            get_target(".install"),
-            get_target(".install/%"),
-            get_target("dirs"),
-            "+" if args.jobserver else "",
-            env.path,
-        )
-    )
-
-    # Targets are of the form <prefix>/<name>: [<prefix>/<depname>]...,
-    # The prefix can be an empty string, in that case we don't add the `/`.
-    # The name is currently the dag hash of the spec. In principle it
-    # could be the package name in case of `concretization: together` so
-    # it can be more easily referred to, but for now we don't special case
-    # this.
     fmt = "{name}{@version}{%compiler}{variants}{arch=architecture}"
+    hash_with_name = [(h, hash_to_spec[h].format(fmt)) for h in hash_to_prereqs.keys()]
+    targets_to_prereqs = [
+        (get_install_deps_target(h), " ".join(prereqs)) for h, prereqs in hash_to_prereqs.items()
+    ]
 
-    # Set SPEC for each hash
+    rendered = template.render(
-    buf.write("# Set the human-readable spec for each target\n")
+        {
-    for dag_hash in hash_to_prereqs.keys():
+            "all_target": get_target("all"),
-        formatted_spec = hash_to_spec[dag_hash].format(fmt)
+            "env_target": get_target("env"),
-        buf.write("{}: SPEC = {}\n".format(get_target("%/" + dag_hash), formatted_spec))
+            "clean_target": get_target("clean"),
-    buf.write("\n")
+            "all_install_targets": " ".join(all_install_targets),
-
+            "all_install_deps_targets": " ".join(all_install_deps_targets),
-    # Set install dependencies
+            "root_install_targets": " ".join(root_install_targets),
-    buf.write("# Install dependencies\n")
+            "dirs_target": get_target("dirs"),
-    for parent, children in hash_to_prereqs.items():
+            "environment": env.path,
-        if not children:
+            "install_target": get_target(".install"),
-            continue
+            "install_deps_target": get_target(".install-deps"),
-        buf.write("{}: {}\n".format(get_install_target(parent), " ".join(children)))
+            "any_hash_target": get_target("%"),
-    buf.write("\n")
+            "hash_with_name": hash_with_name,
-
+            "jobserver_support": "+" if args.jobserver else "",
-    # Clean target: remove target files but not their folders, cause
+            "targets_to_prereqs": targets_to_prereqs,
-    # --make-target-prefix can be any existing directory we do not control,
+        }
-    # including empty string (which means deleting the containing folder
-    # would delete the folder with the Makefile)
-    buf.write(
-        "{}:\n\trm -f -- {} {}\n".format(
-            get_target("clean"), get_target("env"), " ".join(all_install_targets)
-        )
     )
 
+    buf.write(rendered)
     makefile = buf.getvalue()
 
     # Finally write to stdout/file.

lib/spack/spack/tengine.py

@@ -11,6 +11,7 @@
 import llnl.util.lang
 
 import spack.config
+import spack.extensions
 from spack.util.path import canonicalize_path
 
 

share/spack/templates/depfile/Makefile

@@ -0,0 +1,37 @@
+SPACK ?= spack
+
+.PHONY: {{ all_target }} {{ clean_target }}
+
+{{ all_target }}: {{ env_target }}
+
+{{ env_target }}: {{ root_install_targets }}
+	@touch $@
+
+{{ dirs_target }}:
+	@mkdir -p {{ install_target }} {{ install_deps_target }}
+
+# The spack install commands are of the form:
+# spack -e my_env --no-add --only=package --only=concrete /hash
+# This is an involved way of expressing that Spack should only install
+# an individual concrete spec from the environment without deps.
+{{ install_target }}/%: {{ install_deps_target }}/% | {{ dirs_target }}
+	$(info Installing $(SPEC))
+	{{ jobserver_support }}$(SPACK) -e '{{ environment }}' install $(SPACK_INSTALL_FLAGS) --only-concrete --only=package --no-add /$(notdir $@)
+	@touch $@
+
+# Targets of the form {{ install_deps_target }}/<hash> install dependencies only
+{{ install_deps_target }}/%: | {{ dirs_target }}
+	@touch $@
+
+# Set a human-readable SPEC variable for each target that has a hash
+{% for (hash, name) in hash_with_name -%} 
+{{ any_hash_target }}/{{ hash }}: SPEC = {{ name }}
+{% endfor %}
+
+# The Spack DAG expressed in targets:
+{% for (target, prereqs) in targets_to_prereqs -%}
+{{ target }}: {{prereqs}}
+{% endfor %}
+
+{{ clean_target }}:
+	rm -rf {{ env_target }} {{ all_install_targets }} {{ all_install_deps_targets }}