Install: Add use-buildcache option to install (#32537)

Install: Add use-buildcache option to install

* Allow differentiating between top level packages and dependencies when
determining whether to install from the cache or not.

* Add unit test for --use-buildcache

* Use metavar to display use-buildcache options.

* Update spack-completion

lib/spack/spack/cmd/install.py

@@ -5,6 +5,7 @@
 
 import argparse
 import os
+import re
 import shutil
 import sys
 import textwrap
@@ -31,10 +32,50 @@
 level = "short"
 
 
+# Pass in the value string passed to use-buildcache and get back
+# the package and dependencies values.
+def parse_use_buildcache(opt):
+    bc_keys = ["package:", "dependencies:", ""]
+    bc_values = ["only", "never", "auto"]
+    kv_list = re.findall("([a-z]+:)?([a-z]+)", opt)
+
+    # Verify keys and values
+    bc_map = {k: v for k, v in kv_list if k in bc_keys and v in bc_values}
+    if not len(kv_list) == len(bc_map):
+        tty.error("Unrecognized arguments passed to use-buildcache")
+        tty.error(
+            "Expected: --use-buildcache "
+            "[[auto|only|never],[package:[auto|only|never]],[dependencies:[auto|only|never]]]"
+        )
+        exit(1)
+
+    for _group in ["package:", "dependencies:"]:
+        if _group not in bc_map:
+            if "" in bc_map:
+                bc_map[_group] = bc_map[""]
+            else:
+                bc_map[_group] = "auto"
+
+    return bc_map["package:"], bc_map["dependencies:"]
+
+
+# Determine value of cache flag
+def cache_opt(default_opt, use_buildcache):
+    if use_buildcache == "auto":
+        return default_opt
+    elif use_buildcache == "only":
+        return True
+    elif use_buildcache == "never":
+        return False
+
+
 def install_kwargs_from_args(args):
     """Translate command line arguments into a dictionary that will be passed
     to the package installer.
     """
+
+    pkg_use_bc, dep_use_bc = parse_use_buildcache(args.use_buildcache)
+
     return {
         "fail_fast": args.fail_fast,
         "keep_prefix": args.keep_prefix,
@@ -44,8 +85,10 @@ def install_kwargs_from_args(args):
         "verbose": args.verbose or args.install_verbose,
         "fake": args.fake,
         "dirty": args.dirty,
-        "use_cache": args.use_cache,
+        "package_use_cache": cache_opt(args.use_cache, pkg_use_bc),
-        "cache_only": args.cache_only,
+        "package_cache_only": cache_opt(args.cache_only, pkg_use_bc),
+        "dependencies_use_cache": cache_opt(args.use_cache, dep_use_bc),
+        "dependencies_cache_only": cache_opt(args.cache_only, dep_use_bc),
         "include_build_deps": args.include_build_deps,
         "explicit": True,  # Use true as a default for install command
         "stop_at": args.until,
@@ -123,6 +166,18 @@ def setup_parser(subparser):
         default=False,
         help="only install package from binary mirrors",
     )
+    cache_group.add_argument(
+        "--use-buildcache",
+        dest="use_buildcache",
+        default="package:auto,dependencies:auto",
+        metavar="[{auto,only,never},][package:{auto,only,never},][dependencies:{auto,only,never}]",
+        help="""select the mode of buildcache for the 'package' and 'dependencies'.
+Default: package:auto,dependencies:auto
+- `auto` behaves like --use-cache
+- `only` behaves like --cache-only
+- `never` behaves like --no-cache
+""",
+    )
 
     subparser.add_argument(
         "--include-build-deps",

lib/spack/spack/installer.py

@@ -1182,12 +1182,12 @@ def _install_task(self, task):
         Args:
             task (BuildTask): the installation build task for a package"""
 
-        install_args = task.request.install_args
-        cache_only = install_args.get("cache_only")
         explicit = task.explicit
+        install_args = task.request.install_args
+        cache_only = task.cache_only
+        use_cache = task.use_cache
         tests = install_args.get("tests")
         unsigned = install_args.get("unsigned")
-        use_cache = install_args.get("use_cache")
 
         pkg, pkg_id = task.pkg, task.pkg_id
 
@@ -2220,7 +2220,29 @@ def flag_installed(self, installed):
     @property
     def explicit(self):
         """The package was explicitly requested by the user."""
-        return self.pkg == self.request.pkg and self.request.install_args.get("explicit", True)
+        return self.is_root and self.request.install_args.get("explicit", True)
+
+    @property
+    def is_root(self):
+        """The package was requested directly, but may or may not be explicit
+        in an environment."""
+        return self.pkg == self.request.pkg
+
+    @property
+    def use_cache(self):
+        _use_cache = True
+        if self.is_root:
+            return self.request.install_args.get("package_use_cache", _use_cache)
+        else:
+            return self.request.install_args.get("dependencies_use_cache", _use_cache)
+
+    @property
+    def cache_only(self):
+        _cache_only = False
+        if self.is_root:
+            return self.request.install_args.get("package_cache_only", _cache_only)
+        else:
+            return self.request.install_args.get("dependencies_cache_only", _cache_only)
 
     @property
     def key(self):
@@ -2302,21 +2324,23 @@ def __str__(self):
     def _add_default_args(self):
         """Ensure standard install options are set to at least the default."""
         for arg, default in [
-            ("cache_only", False),
             ("context", "build"),  # installs *always* build
+            ("dependencies_cache_only", False),
+            ("dependencies_use_cache", True),
             ("dirty", False),
             ("fail_fast", False),
             ("fake", False),
             ("install_deps", True),
             ("install_package", True),
             ("install_source", False),
+            ("package_cache_only", False),
+            ("package_use_cache", True),
             ("keep_prefix", False),
             ("keep_stage", False),
             ("restage", False),
             ("skip_patch", False),
             ("tests", False),
             ("unsigned", False),
-            ("use_cache", True),
             ("verbose", False),
         ]:
             _ = self.install_args.setdefault(arg, default)

lib/spack/spack/test/cmd/install.py

@@ -5,6 +5,7 @@
 
 import argparse
 import filecmp
+import itertools
 import os
 import re
 import sys
@@ -14,6 +15,7 @@
 from six.moves import builtins
 
 import llnl.util.filesystem as fs
+import llnl.util.tty as tty
 
 import spack.cmd.install
 import spack.compilers as compilers
@@ -1090,3 +1092,80 @@ def test_install_callbacks_fail(install_mockery, mock_fetch, name, method):
     assert output.count(method) == 2
     assert output.count("method not implemented") == 1
     assert output.count("TestFailure: 1 tests failed") == 1
+
+
+def test_install_use_buildcache(
+    capsys,
+    mock_packages,
+    mock_fetch,
+    mock_archive,
+    mock_binary_index,
+    tmpdir,
+    install_mockery_mutable_config,
+):
+    """
+    Make sure installing with use-buildcache behaves correctly.
+    """
+
+    package_name = "dependent-install"
+    dependency_name = "dependency-install"
+
+    def validate(mode, out, pkg):
+        def assert_auto(pkg, out):
+            assert "==> Extracting {0}".format(pkg) in out
+
+        def assert_only(pkg, out):
+            assert "==> Extracting {0}".format(pkg) in out
+
+        def assert_never(pkg, out):
+            assert "==> {0}: Executing phase: 'install'".format(pkg) in out
+
+        if mode == "auto":
+            assert_auto(pkg, out)
+        elif mode == "only":
+            assert_only(pkg, out)
+        else:
+            assert_never(pkg, out)
+
+    def install_use_buildcache(opt):
+        out = install(
+            "--no-check-signature", "--use-buildcache", opt, package_name, fail_on_error=True
+        )
+
+        pkg_opt, dep_opt = spack.cmd.install.parse_use_buildcache(opt)
+        validate(dep_opt, out, dependency_name)
+        validate(pkg_opt, out, package_name)
+
+        # Clean up installed packages
+        uninstall("-y", "-a")
+
+    # Setup the mirror
+    # Create a temp mirror directory for buildcache usage
+    mirror_dir = tmpdir.join("mirror_dir")
+    mirror_url = "file://{0}".format(mirror_dir.strpath)
+
+    # Populate the buildcache
+    install(package_name)
+    buildcache("create", "-u", "-a", "-f", "-d", mirror_dir.strpath, package_name, dependency_name)
+
+    # Uninstall the all of the packages for clean slate
+    uninstall("-y", "-a")
+
+    # Configure the mirror where we put that buildcache w/ the compiler
+    mirror("add", "test-mirror", mirror_url)
+
+    with capsys.disabled():
+        # Install using the matrix of possible combinations with --use-buildcache
+        for pkg, deps in itertools.product(["auto", "only", "never"], repeat=2):
+            tty.debug(
+                "Testing `spack install --use-buildcache package:{0},dependencies:{1}`".format(
+                    pkg, deps
+                )
+            )
+            install_use_buildcache("package:{0},dependencies:{1}".format(pkg, deps))
+            install_use_buildcache("dependencies:{0},package:{1}".format(deps, pkg))
+
+        # Install using a default override option
+        # Alternative to --cache-only (always) or --no-cache (never)
+        for opt in ["auto", "only", "never"]:
+            install_use_buildcache(opt)

share/spack/spack-completion.bash

@@ -1188,7 +1188,7 @@ _spack_info() {
 _spack_install() {
     if $list_options
     then
-        SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --include-build-deps --no-check-signature --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse"
+        SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --use-buildcache --include-build-deps --no-check-signature --show-log-on-error --source -n --no-checksum --deprecated -v --verbose --fake --only-concrete --no-add -f --file --clean --dirty --test --log-format --log-file --help-cdash --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp -y --yes-to-all -U --fresh --reuse"
     else
         _all_packages
     fi