Auto install available pre-built packages from binary cache (#5242)

* basic functionality to install spec from a binary cache when it's available; this spiders each cache for each package and could likely be more efficient by caching the results of the first check

* add spec to db after installing from binary cache

* cache (in memory) spec listings retrieved from binary caches

* print a warning vs. failing when no mirrors are configured to retrieve pre-built Spack packages

* make automatic retrieval of pre-built spack packages from mirrors optional

* no code was using the links stored in the dictionary returned by get_specs, so this simplifies the logic to return only a set of specs

* print package prefix after installing from binary cache

* provide more information to user on install progress

lib/spack/spack/__init__.py

@@ -134,6 +134,9 @@
 misc_cache = FileCache(misc_cache_path)
 
 
+binary_cache_retrieved_specs = set()
+
+
 #: Directories where to search for templates
 template_dirs = spack.config.get_config('config')['template_dirs']
 template_dirs = [canonicalize_path(x) for x in template_dirs]

lib/spack/spack/binary_distribution.py

@@ -417,15 +417,18 @@ def get_specs(force=False):
     """
     Get spec.yaml's for build caches available on mirror
     """
+    if spack.binary_cache_retrieved_specs:
+        tty.debug("Using previously-retrieved specs")
+        previously_retrieved = spack.binary_cache_retrieved_specs
+        return previously_retrieved
+
     mirrors = spack.config.get_config('mirrors')
     if len(mirrors) == 0:
-        tty.die("Please add a spack mirror to allow " +
-                "download of build caches.")
+        tty.warn("No Spack mirrors are currently configured")
+        return {}
+
     path = str(spack.architecture.sys_type())
-    specs = set()
     urls = set()
-    from collections import defaultdict
-    durls = defaultdict(list)
     for key in mirrors:
         url = mirrors[key]
         if url.startswith('file'):
@@ -442,25 +445,27 @@ def get_specs(force=False):
             for link in links:
                 if re.search("spec.yaml", link) and re.search(path, link):
                     urls.add(link)
-        for link in urls:
-            with Stage(link, name="build_cache", keep=True) as stage:
-                if force and os.path.exists(stage.save_filename):
-                    os.remove(stage.save_filename)
-                if not os.path.exists(stage.save_filename):
-                    try:
-                        stage.fetch()
-                    except fs.FetchError:
-                        continue
-                with open(stage.save_filename, 'r') as f:
-                    # read the spec from the build cache file. All specs
-                    # in build caches are concrete (as they aer built) so
-                    # we need to mark this spec concrete on read-in.
-                    spec = spack.spec.Spec.from_yaml(f)
-                    spec._mark_concrete()
 
-                    specs.add(spec)
-                    durls[spec].append(link)
-    return specs, durls
+    specs = set()
+    for link in urls:
+        with Stage(link, name="build_cache", keep=True) as stage:
+            if force and os.path.exists(stage.save_filename):
+                os.remove(stage.save_filename)
+            if not os.path.exists(stage.save_filename):
+                try:
+                    stage.fetch()
+                except fs.FetchError:
+                    continue
+            with open(stage.save_filename, 'r') as f:
+                # read the spec from the build cache file. All specs
+                # in build caches are concrete (as they are built) so
+                # we need to mark this spec concrete on read-in.
+                spec = spack.spec.Spec.from_yaml(f)
+                spec._mark_concrete()
+                specs.add(spec)
+
+    spack.binary_cache_retrieved_specs = specs
+    return specs
 
 
 def get_keys(install=False, yes_to_all=False, force=False):

lib/spack/spack/cmd/buildcache.py

@@ -146,7 +146,7 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False):
     # List of specs that match expressions given via command line
     specs_from_cli = []
     has_errors = False
-    specs, links = bindist.get_specs(force)
+    specs = bindist.get_specs(force)
     for pkg in pkgs:
         matches = []
         tty.msg("buildcache spec(s) matching %s \n" % pkg)
@@ -296,10 +296,7 @@ def install_tarball(spec, args):
 
 
 def listspecs(args):
-    force = False
-    if args.force:
-        force = True
-    specs, links = bindist.get_specs(force)
+    specs = bindist.get_specs(args.force)
     if args.packages:
         pkgs = set(args.packages)
         for pkg in pkgs:

lib/spack/spack/cmd/install.py

@@ -73,6 +73,9 @@ def setup_parser(subparser):
     subparser.add_argument(
         '--restage', action='store_true',
         help="if a partial install is detected, delete prior state")
+    subparser.add_argument(
+        '--use-cache', action='store_true', dest='use_cache',
+        help="check for pre-built Spack packages in mirrors")
     subparser.add_argument(
         '--show-log-on-error', action='store_true',
         help="print full build log to stderr if build fails")
@@ -395,7 +398,8 @@ def install(parser, args, **kwargs):
         'make_jobs': args.jobs,
         'verbose': args.verbose,
         'fake': args.fake,
-        'dirty': args.dirty
+        'dirty': args.dirty,
+        'use_cache': args.use_cache
     })
 
     if args.run_tests:

lib/spack/spack/package.py

@@ -60,6 +60,7 @@
 import spack.url
 import spack.util.web
 import spack.multimethod
+import spack.binary_distribution as binary_distribution
 
 from llnl.util.filesystem import mkdirp, join_path, touch, ancestor
 from llnl.util.filesystem import working_dir, install_tree, install
@@ -1266,6 +1267,18 @@ def _update_explicit_entry_in_db(self, rec, explicit):
                 message = '{s.name}@{s.version} : marking the package explicit'
                 tty.msg(message.format(s=self))
 
+    def try_install_from_binary_cache(self, explicit):
+        tty.msg('Searching for binary cache of %s' % self.name)
+        specs = binary_distribution.get_specs()
+        if self.spec not in specs:
+            return False
+        tty.msg('Installing %s from binary cache' % self.name)
+        tarball = binary_distribution.download_tarball(self.spec)
+        binary_distribution.extract_tarball(
+            self.spec, tarball, yes_to_all=False, force=False)
+        spack.store.db.add(self.spec, spack.store.layout, explicit=explicit)
+        return True
+
     def do_install(self,
                    keep_prefix=False,
                    keep_stage=False,
@@ -1349,6 +1362,16 @@ def do_install(self,
 
         tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
 
+        if kwargs.get('use_cache', False):
+            if self.try_install_from_binary_cache(explicit):
+                tty.msg('Successfully installed %s from binary cache'
+                        % self.name)
+                print_pkg(self.prefix)
+                return
+
+            tty.msg('No binary for %s found: installing from source'
+                    % self.name)
+
         # Set run_tests flag before starting build.
         self.run_tests = spack.package_testing.check(self.name)