zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto install available pre-built packages from binary cache (#5242)

Browse Source 
* basic functionality to install spec from a binary cache when it's available; this spiders each cache for each package and could likely be more efficient by caching the results of the first check

* add spec to db after installing from binary cache

* cache (in memory) spec listings retrieved from binary caches

* print a warning vs. failing when no mirrors are configured to retrieve pre-built Spack packages

* make automatic retrieval of pre-built spack packages from mirrors optional

* no code was using the links stored in the dictionary returned by get_specs, so this simplifies the logic to return only a set of specs

* print package prefix after installing from binary cache

* provide more information to user on install progress
This commit is contained in:
scheibelp 2017-10-30 14:16:46 -07:00 committed by becker33
parent 347c76b7a0
commit e5b8312de3
5 changed files with 61 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
lib/spack/spack/__init__.py
View File

@ -134,6 +134,9 @@
misc_cache = FileCache(misc_cache_path)
binary_cache_retrieved_specs = set()
#: Directories where to search for templates
template_dirs = spack.config.get_config('config')['template_dirs']
template_dirs = [canonicalize_path(x) for x in template_dirs]

23
lib/spack/spack/binary_distribution.py
View File

@ -417,15 +417,18 @@ def get_specs(force=False):
"""
Get spec.yaml's for build caches available on mirror
"""
if spack.binary_cache_retrieved_specs:
tty.debug("Using previously-retrieved specs")
previously_retrieved = spack.binary_cache_retrieved_specs
return previously_retrieved
mirrors = spack.config.get_config('mirrors')
if len(mirrors) == 0:
tty.die("Please add a spack mirror to allow " +
"download of build caches.")
tty.warn("No Spack mirrors are currently configured")
return {}
path = str(spack.architecture.sys_type())
specs = set()
urls = set()
from collections import defaultdict
durls = defaultdict(list)
for key in mirrors:
url = mirrors[key]
if url.startswith('file'):
@ -442,6 +445,8 @@ def get_specs(force=False):
for link in links:
if re.search("spec.yaml", link) and re.search(path, link):
urls.add(link)
specs = set()
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
if force and os.path.exists(stage.save_filename):
@ -453,14 +458,14 @@ def get_specs(force=False):
continue
with open(stage.save_filename, 'r') as f:
# read the spec from the build cache file. All specs
# in build caches are concrete (as they aer built) so
# in build caches are concrete (as they are built) so
# we need to mark this spec concrete on read-in.
spec = spack.spec.Spec.from_yaml(f)
spec._mark_concrete()
specs.add(spec)
durls[spec].append(link)
return specs, durls
spack.binary_cache_retrieved_specs = specs
return specs
def get_keys(install=False, yes_to_all=False, force=False):

7
lib/spack/spack/cmd/buildcache.py
View File

@ -146,7 +146,7 @@ def match_downloaded_specs(pkgs, allow_multiple_matches=False, force=False):
# List of specs that match expressions given via command line
specs_from_cli = []
has_errors = False
specs, links = bindist.get_specs(force)
specs = bindist.get_specs(force)
for pkg in pkgs:
matches = []
tty.msg("buildcache spec(s) matching %s \n" % pkg)
@ -296,10 +296,7 @@ def install_tarball(spec, args):
def listspecs(args):
force = False
if args.force:
force = True
specs, links = bindist.get_specs(force)
specs = bindist.get_specs(args.force)
if args.packages:
pkgs = set(args.packages)
for pkg in pkgs:

6
lib/spack/spack/cmd/install.py
View File

@ -73,6 +73,9 @@ def setup_parser(subparser):
subparser.add_argument(
'--restage', action='store_true',
help="if a partial install is detected, delete prior state")
subparser.add_argument(
'--use-cache', action='store_true', dest='use_cache',
help="check for pre-built Spack packages in mirrors")
subparser.add_argument(
'--show-log-on-error', action='store_true',
help="print full build log to stderr if build fails")
@ -395,7 +398,8 @@ def install(parser, args, **kwargs):
'make_jobs': args.jobs,
'verbose': args.verbose,
'fake': args.fake,
'dirty': args.dirty
'dirty': args.dirty,
'use_cache': args.use_cache
})
if args.run_tests:

23
lib/spack/spack/package.py
View File

@ -60,6 +60,7 @@
import spack.url
import spack.util.web
import spack.multimethod
import spack.binary_distribution as binary_distribution
from llnl.util.filesystem import mkdirp, join_path, touch, ancestor
from llnl.util.filesystem import working_dir, install_tree, install
@ -1266,6 +1267,18 @@ def _update_explicit_entry_in_db(self, rec, explicit):
message = '{s.name}@{s.version} : marking the package explicit'
tty.msg(message.format(s=self))
def try_install_from_binary_cache(self, explicit):
tty.msg('Searching for binary cache of %s' % self.name)
specs = binary_distribution.get_specs()
if self.spec not in specs:
return False
tty.msg('Installing %s from binary cache' % self.name)
tarball = binary_distribution.download_tarball(self.spec)
binary_distribution.extract_tarball(
self.spec, tarball, yes_to_all=False, force=False)
spack.store.db.add(self.spec, spack.store.layout, explicit=explicit)
return True
def do_install(self,
keep_prefix=False,
keep_stage=False,
@ -1349,6 +1362,16 @@ def do_install(self,
tty.msg(colorize('@*{Installing} @*g{%s}' % self.name))
if kwargs.get('use_cache', False):
if self.try_install_from_binary_cache(explicit):
tty.msg('Successfully installed %s from binary cache'
% self.name)
print_pkg(self.prefix)
return
tty.msg('No binary for %s found: installing from source'
% self.name)
# Set run_tests flag before starting build.
self.run_tests = spack.package_testing.check(self.name)