buildcache command updates (#5860)

* Add better spec matching to spack buildcache

* skip download of spec.yaml and keys if they exist

* autopep8

lib/spack/spack/binary_distribution.py

@@ -412,7 +412,7 @@ def extract_tarball(spec, filename, yes_to_all=False, force=False):
     relocate_package(installpath)
 
 
-def get_specs():
+def get_specs(force=False):
     """
     Get spec.yaml's for build caches available on mirror
     """
@@ -443,10 +443,13 @@ def get_specs():
                     urls.add(link)
         for link in urls:
             with Stage(link, name="build_cache", keep=True) as stage:
-                try:
+                if force and os.path.exists(stage.save_filename):
-                    stage.fetch()
+                    os.remove(stage.save_filename)
-                except fs.FetchError:
+                if not os.path.exists(stage.save_filename):
-                    continue
+                    try:
+                        stage.fetch()
+                    except fs.FetchError:
+                        continue
                 with open(stage.save_filename, 'r') as f:
                     # read the spec from the build cache file. All specs
                     # in build caches are concrete (as they aer built) so
@@ -459,7 +462,7 @@ def get_specs():
     return specs, durls
 
 
-def get_keys(install=False, yes_to_all=False):
+def get_keys(install=False, yes_to_all=False, force=False):
     """
     Get pgp public keys available on mirror
     """
@@ -487,10 +490,13 @@ def get_keys(install=False, yes_to_all=False):
                     keys.add(link)
         for link in keys:
             with Stage(link, name="build_cache", keep=True) as stage:
-                try:
+                if os.path.exists(stage.save_filename) and force:
-                    stage.fetch()
+                    os.remove(stage.save_filename)
-                except fs.FetchError:
+                if not os.path.exists(stage.save_filename):
-                    continue
+                    try:
+                        stage.fetch()
+                    except fs.FetchError:
+                        continue
             tty.msg('Found key %s' % link)
             if install:
                 if yes_to_all:

lib/spack/spack/cmd/buildcache.py

@@ -25,7 +25,6 @@
 import argparse
 
 import os
-import re
 import llnl.util.tty as tty
 
 import spack
@@ -39,6 +38,18 @@
 section = "caching"
 level = "long"
 
+# Arguments for display_specs when we find ambiguity
+display_args = {
+    'long': True,
+    'show_flags': True,
+    'variants': True
+}
+
+error_message = """You can either:
+    a) use a more specific spec, or
+    b) use the package hash with a leading /
+"""
+
 
 def setup_parser(subparser):
     setup_parser.parser = subparser
@@ -76,6 +87,8 @@ def setup_parser(subparser):
     install.set_defaults(func=installtarball)
 
     listcache = subparsers.add_parser('list')
+    listcache.add_argument('-f', '--force', action='store_true',
+                           help="force new download of specs")
     listcache.add_argument(
         'packages', nargs=argparse.REMAINDER,
         help="specs of packages to search for")
@@ -88,9 +101,91 @@ def setup_parser(subparser):
     dlkeys.add_argument(
         '-y', '--yes-to-all', action='store_true',
         help="answer yes to all trust questions")
+    dlkeys.add_argument('-f', '--force', action='store_true',
+                        help="force new download of keys")
     dlkeys.set_defaults(func=getkeys)
 
 
+def find_matching_specs(specs, allow_multiple_matches=False, force=False):
+    """Returns a list of specs matching the not necessarily
+       concretized specs given from cli
+
+    Args:
+        specs: list of specs to be matched against installed packages
+        allow_multiple_matches : if True multiple matches are admitted
+
+    Return:
+        list of specs
+    """
+    # List of specs that match expressions given via command line
+    specs_from_cli = []
+    has_errors = False
+    for spec in specs:
+        matching = spack.store.db.query(spec)
+        # For each spec provided, make sure it refers to only one package.
+        # Fail and ask user to be unambiguous if it doesn't
+        if not allow_multiple_matches and len(matching) > 1:
+            tty.error('{0} matches multiple packages:'.format(spec))
+            print()
+            spack.cmd.display_specs(matching, **display_args)
+            print()
+            has_errors = True
+
+        # No installed package matches the query
+        if len(matching) == 0 and spec is not any:
+            tty.error('{0} does not match any installed packages.'.format(
+                spec))
+            has_errors = True
+
+        specs_from_cli.extend(matching)
+    if has_errors:
+        tty.die(error_message)
+
+    return specs_from_cli
+
+
+def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False):
+    """Returns a list of specs matching the not necessarily
+       concretized specs given from cli
+
+    Args:
+        specs: list of specs to be matched against buildcaches on mirror
+        allow_multiple_matches : if True multiple matches are admitted
+
+    Return:
+        list of specs
+    """
+    # List of specs that match expressions given via command line
+    specs_from_cli = []
+    has_errors = False
+    specs, links = bindist.get_specs(force)
+    for pkg in pkgs:
+        matches = []
+        tty.msg("buildcache spec(s) matching %s \n" % pkg)
+        for spec in sorted(specs):
+            if spec.satisfies(pkg):
+                matches.append(spec)
+        # For each pkg provided, make sure it refers to only one package.
+        # Fail and ask user to be unambiguous if it doesn't
+        if not allow_multiple_matches and len(matches) > 1:
+            tty.error('%s matches multiple downloaded packages:' % pkg)
+            print()
+            spack.cmd.display_specs(matches, **display_args)
+            print()
+            has_errors = True
+
+        # No downloaded package matches the query
+        if len(matches) == 0:
+            tty.error('%s does not match any downloaded packages.' % pkg)
+            has_errors = True
+
+        specs_from_cli.extend(matches)
+    if has_errors:
+        tty.die(error_message)
+
+    return specs_from_cli
+
+
 def createtarball(args):
     if not args.packages:
         tty.die("build cache file creation requires at least one" +
@@ -112,16 +207,22 @@ def createtarball(args):
         force = True
     if args.rel:
         relative = True
-    for pkg in pkgs:
+
-        for spec in spack.cmd.parse_specs(pkg, concretize=True):
+    matches = find_matching_specs(pkgs, False, False)
-            specs.add(spec)
+    for match in matches:
-            tty.msg('recursing dependencies')
+        tty.msg('adding matching spec %s' % match.format())
-            for d, node in spec.traverse(order='post',
+        specs.add(match)
-                                         depth=True,
+        tty.msg('recursing dependencies')
-                                         deptype=('link', 'run')):
+        for d, node in match.traverse(order='post',
-                if not node.external:
+                                      depth=True,
-                    tty.msg('adding dependency %s' % node.format())
+                                      deptype=('link', 'run')):
-                    specs.add(node)
+            if node.external or node.virtual:
+                tty.msg('Skipping external or virtual dependency %s' %
+                        node.format())
+            else:
+                tty.msg('adding dependency %s' % node.format())
+                specs.add(node)
+
     for spec in specs:
         tty.msg('creating binary cache file for package %s ' % spec.format())
         try:
@@ -147,14 +248,13 @@ def installtarball(args):
         tty.die("build cache file installation requires" +
                 " at least one package spec argument")
     pkgs = set(args.packages)
-    specs, links = bindist.get_specs()
+    yes_to_all = False
-    matches = set()
+    if args.yes_to_all:
-    for spec in specs:
+        yes_to_all = True
-        for pkg in pkgs:
+    force = False
-            if re.match(re.escape(pkg), str(spec)):
+    if args.force:
-                matches.add(spec)
+        force = True
-            if re.match(re.escape(pkg), '/%s' % spec.dag_hash()):
+    matches = match_downloaded_specs(pkgs, yes_to_all, force)
-                matches.add(spec)
 
     for match in matches:
         install_tarball(match, args)
@@ -162,10 +262,13 @@ def installtarball(args):
 
 def install_tarball(spec, args):
     s = spack.spec.Spec(spec)
+    if s.external or s.virtual:
+        tty.warn("Skipping external or virtual package %s" % spec.format())
+        return
     yes_to_all = False
-    force = False
     if args.yes_to_all:
         yes_to_all = True
+    force = False
     if args.force:
         force = True
     for d in s.dependencies():
@@ -197,20 +300,24 @@ def install_tarball(spec, args):
 
 
 def listspecs(args):
-    specs, links = bindist.get_specs()
+    force = False
+    if args.force:
+        force = True
+    specs, links = bindist.get_specs(force)
     if args.packages:
         pkgs = set(args.packages)
         for pkg in pkgs:
-            tty.msg("buildcache spec(s) matching %s \n" % pkg)
+            tty.msg("buildcache spec(s) matching %s \n" % pkgs)
             for spec in sorted(specs):
-                if re.search("^" + re.escape(pkg), str(spec)):
+                if spec.satisfies(pkg):
-                    tty.msg('run "spack buildcache install /%s"' %
+                    tty.msg('spack buildcache install /%s\n' %
-                            spec.dag_hash(7) + ' to install  %s\n' %
+                            spec.dag_hash(7) +
+                            '   to install %s' %
                             spec.format())
     else:
         tty.msg("buildcache specs ")
         for spec in sorted(specs):
-            tty.msg('run "spack buildcache install /%s" to install  %s\n' %
+            tty.msg('spack buildcache install /%s\n   to install %s' %
                     (spec.dag_hash(7), spec.format()))
 
 
@@ -221,7 +328,10 @@ def getkeys(args):
     yes_to_all = False
     if args.yes_to_all:
         yes_to_all = True
-    bindist.get_keys(install, yes_to_all)
+    force = False
+    if args.force:
+        force = True
+    bindist.get_keys(install, yes_to_all, force)
 
 
 def buildcache(parser, args):

lib/spack/spack/test/packaging.py

@@ -189,6 +189,9 @@ def test_packaging(mock_archive, tmpdir):
     args = parser.parse_args(['list'])
     buildcache.buildcache(parser, args)
 
+    args = parser.parse_args(['list', '-f'])
+    buildcache.buildcache(parser, args)
+
     args = parser.parse_args(['list', 'trivial'])
     buildcache.buildcache(parser, args)
 
@@ -199,6 +202,9 @@ def test_packaging(mock_archive, tmpdir):
     args = parser.parse_args(['keys'])
     buildcache.buildcache(parser, args)
 
+    args = parser.parse_args(['keys', '-f'])
+    buildcache.buildcache(parser, args)
+
     # unregister mirror with spack config
     mirrors = {}
     spack.config.update_config('mirrors', mirrors)