zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #18359 from scottwittenburg/add-binary-distribution-cache-manager

Browse Source 
Add binary distribution cache manager
This commit is contained in:
Scott Wittenburg 2020-09-30 16:37:35 -06:00 committed by GitHub
parent 5558bc2b50
commit 075c3e0d92
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 571 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

524
lib/spack/spack/binary_distribution.py
View File

@ -17,8 +17,9 @@
import json import json
from six.moves.urllib.error import URLError from six.moves.urllib.error import URLError, HTTPError
import llnl.util.lang
import llnl.util.tty as tty import llnl.util.tty as tty
from llnl.util.filesystem import mkdirp from llnl.util.filesystem import mkdirp
@ -26,6 +27,7 @@
import spack.config as config import spack.config as config
import spack.database as spack_db import spack.database as spack_db
import spack.fetch_strategy as fs import spack.fetch_strategy as fs
import spack.util.file_cache as file_cache
import spack.relocate as relocate import spack.relocate as relocate
import spack.util.gpg import spack.util.gpg
import spack.util.spack_json as sjson import spack.util.spack_json as sjson
@ -36,23 +38,296 @@
from spack.spec import Spec from spack.spec import Spec
from spack.stage import Stage from spack.stage import Stage
#: default root, relative to the Spack install path
default_manager_root = os.path.join(spack.paths.opt_path, 'spack')
_build_cache_relative_path = 'build_cache' _build_cache_relative_path = 'build_cache'
_build_cache_keys_relative_path = '_pgp' _build_cache_keys_relative_path = '_pgp'
BUILD_CACHE_INDEX_TEMPLATE = '''
<html>
<head>
<title>{title}</title>
</head>
<body>
<ul>
{path_list}
</ul>
</body>
</html>
'''
BUILD_CACHE_INDEX_ENTRY_TEMPLATE = ' <li><a href="{path}">{path}</a></li>' class BinaryDistributionCacheManager(object):
"""
The BinaryDistributionCacheManager stores and manages both:
1. cached buildcache index files (the index.json describing the built
specs available on a mirror)
2. a cache of all the conrcrete built specs available on all the
configured mirrors.
"""
def __init__(self, cache_root=None):
self._cache_root = cache_root or default_manager_root
self._index_cache_root = os.path.join(self._cache_root, 'indices')
self._index_contents_key = 'contents.json'
self._index_file_cache = None
self._local_index_cache = None
self._built_spec_cache = {}
def _init_local_index_cache(self):
if not self._index_file_cache:
self._index_file_cache = file_cache.FileCache(
self._index_cache_root)
cache_key = self._index_contents_key
self._index_file_cache.init_entry(cache_key)
cache_path = self._index_file_cache.cache_path(cache_key)
self._local_index_cache = {}
if os.path.isfile(cache_path):
with self._index_file_cache.read_transaction(
cache_key) as cache_file:
self._local_index_cache = json.load(cache_file)
@property
def spec_cache(self):
"""The cache of specs and mirrors they live on"""
return self._built_spec_cache
def clear_spec_cache(self):
self._built_spec_cache = {}
def _write_local_index_cache(self):
self._init_local_index_cache()
cache_key = self._index_contents_key
with self._index_file_cache.write_transaction(cache_key) as (old, new):
json.dump(self._local_index_cache, new)
def regenerate_spec_cache(self):
self._init_local_index_cache()
for mirror_url in self._local_index_cache:
cache_entry = self._local_index_cache[mirror_url]
cached_index_path = cache_entry['index_path']
self._associate_built_specs_with_mirror(cached_index_path,
mirror_url)
def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
self._init_local_index_cache()
tmpdir = tempfile.mkdtemp()
try:
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False)
self._index_file_cache.init_entry(cache_key)
cache_path = self._index_file_cache.cache_path(cache_key)
with self._index_file_cache.read_transaction(cache_key):
db._read_from_file(cache_path)
spec_list = db.query_local(installed=False)
for indexed_spec in spec_list:
indexed_spec_dag_hash = indexed_spec.dag_hash()
if indexed_spec_dag_hash not in self._built_spec_cache:
self._built_spec_cache[indexed_spec_dag_hash] = []
for entry in self._built_spec_cache[indexed_spec_dag_hash]:
if entry['mirror_url'] == mirror_url:
break
else:
self._built_spec_cache[indexed_spec_dag_hash].append({
"mirror_url": mirror_url,
"spec": indexed_spec,
})
finally:
shutil.rmtree(tmpdir)
def get_all_built_specs(self):
spec_list = []
for dag_hash in self._built_spec_cache:
# in the absence of further information, all concrete specs
# with the same DAG hash are equivalent, so we can just
# return the first one in the list.
if len(self._built_spec_cache[dag_hash]) > 0:
spec_list.append(self._built_spec_cache[dag_hash][0]['spec'])
return spec_list
def find_built_spec(self, spec):
"""Find the built spec corresponding to ``spec``.
If the spec can be found among the configured binary mirrors, an
object is returned that contains the concrete spec and the mirror url
where it can be found. Otherwise, ``None`` is returned.
Args:
spec (Spec): Concrete spec to find
Returns:
An list of objects containing the found specs and mirror url where
each can be found, e.g.:
.. code-block:: python
[
{
"spec": <concrete-spec>,
"mirror_url": <mirror-root-url>
}
]
"""
find_hash = spec.dag_hash()
if find_hash not in self._built_spec_cache:
return None
return self._built_spec_cache[find_hash]
def update_spec(self, spec, found_list):
"""
Take list of {'mirror_url': m, 'spec': s} objects and update the local
built_spec_cache
"""
spec_dag_hash = spec.dag_hash()
if spec_dag_hash not in self._built_spec_cache:
self._built_spec_cache[spec_dag_hash] = found_list
else:
current_list = self._built_spec_cache[spec_dag_hash]
for new_entry in found_list:
for cur_entry in current_list:
if new_entry['mirror_url'] == cur_entry['mirror_url']:
cur_entry['spec'] = new_entry['spec']
break
else:
current_list.append = {
'mirror_url': new_entry['mirror_url'],
'spec': new_entry['spec'],
}
def update_local_index_cache(self):
""" Make sure local cache of buildcache index files is up to date."""
self._init_local_index_cache()
mirrors = spack.mirror.MirrorCollection()
configured_mirror_urls = [m.fetch_url for m in mirrors.values()]
items_to_remove = []
# First compare the mirror urls currently present in the cache to the
# configured mirrors. If we have a cached index for a mirror which is
# no longer configured, we should remove it from the cache. For any
# cached indices corresponding to currently configured mirrors, we need
# to check if the cache is still good, or needs to be updated.
# Finally, if there are configured mirrors for which we don't have a
# cache entry, we need to fetch and cache the indices from those
# mirrors.
for cached_mirror_url in self._local_index_cache:
cache_entry = self._local_index_cache[cached_mirror_url]
cached_index_hash = cache_entry['index_hash']
cached_index_path = cache_entry['index_path']
if cached_mirror_url in configured_mirror_urls:
# May need to fetch the index and update the local caches
self.fetch_and_cache_index(
cached_mirror_url, expect_hash=cached_index_hash)
else:
# No longer have this mirror, cached index should be removed
items_to_remove.append({
'url': cached_mirror_url,
'cache_key': os.path.join(self._index_cache_root,
cached_index_path)
})
# Clean up items to be removed, identified above
for item in items_to_remove:
url = item['url']
cache_key = item['cache_key']
self._index_file_cache.remove(cache_key)
del self._local_index_cache[url]
# Iterate the configured mirrors now. Any mirror urls we do not
# already have in our cache must be fetched, stored, and represented
# locally.
for mirror_url in configured_mirror_urls:
if mirror_url not in self._local_index_cache:
# Need to fetch the index and update the local caches
self.fetch_and_cache_index(mirror_url)
self._write_local_index_cache()
def fetch_and_cache_index(self, mirror_url, expect_hash=None):
""" Fetch a buildcache index file from a remote mirror and cache it.
If we already have a cached index from this mirror, then we first
check if the hash has changed, and we avoid fetching it if not.
Args:
mirror_url (str): Base url of mirror
expect_hash (str): If provided, this hash will be compared against
the index hash we retrieve from the mirror, to determine if we
need to fetch the index or not.
"""
index_fetch_url = url_util.join(
mirror_url, _build_cache_relative_path, 'index.json')
hash_fetch_url = url_util.join(
mirror_url, _build_cache_relative_path, 'index.json.hash')
fetched_hash = ''
# Fetch the hash first so we can check if we actually need to fetch
# the index itself.
try:
_, _, fs = web_util.read_from_url(
hash_fetch_url, 'text/plain')
fetched_hash = codecs.getreader('utf-8')(fs).read()
except (URLError, web_util.SpackWebError) as url_err:
tty.debug('Unable to read index hash {0}'.format(
hash_fetch_url), url_err, 1)
# If we were expecting some hash and found that we got it, we're done
if expect_hash and fetched_hash == expect_hash:
tty.debug('Cached index for {0} already up to date'.format(
mirror_url))
return
tty.debug('Fetching index from {0}'.format(index_fetch_url))
# Fetch index itself
try:
_, _, fs = web_util.read_from_url(
index_fetch_url, 'application/json')
index_object_str = codecs.getreader('utf-8')(fs).read()
except (URLError, web_util.SpackWebError) as url_err:
tty.debug('Unable to read index {0}'.format(index_fetch_url),
url_err, 1)
return
locally_computed_hash = compute_hash(index_object_str)
if fetched_hash is not None and locally_computed_hash != fetched_hash:
msg_tmpl = ('Computed hash ({0}) did not match remote ({1}), '
'indicating error in index transmission')
tty.error(msg_tmpl.format(locally_computed_hash, expect_hash))
return
url_hash = compute_hash(mirror_url)
cache_key = '{0}_{1}.json'.format(
url_hash[:10], locally_computed_hash[:10])
self._index_file_cache.init_entry(cache_key)
with self._index_file_cache.write_transaction(cache_key) as (old, new):
new.write(index_object_str)
self._local_index_cache[mirror_url] = {
'index_hash': locally_computed_hash,
'index_path': cache_key,
}
def _cache_manager():
"""Get the singleton store instance."""
cache_root = spack.config.get(
'config:binary_distribution_cache_root', default_manager_root)
cache_root = spack.util.path.canonicalize_path(cache_root)
return BinaryDistributionCacheManager(cache_root)
#: Singleton cache_manager instance
cache_manager = llnl.util.lang.Singleton(_cache_manager)
class NoOverwriteException(spack.error.SpackError): class NoOverwriteException(spack.error.SpackError):
@ -118,6 +393,10 @@ def __init__(self, msg):
super(NewLayoutException, self).__init__(msg) super(NewLayoutException, self).__init__(msg)
def compute_hash(data):
return hashlib.sha256(data.encode('utf-8')).hexdigest()
def build_cache_relative_path(): def build_cache_relative_path():
return _build_cache_relative_path return _build_cache_relative_path
@ -315,11 +594,29 @@ def generate_package_index(cache_prefix):
with open(index_json_path, 'w') as f: with open(index_json_path, 'w') as f:
db._write_to_file(f) db._write_to_file(f)
# Read the index back in and compute it's hash
with open(index_json_path) as f:
index_string = f.read()
index_hash = compute_hash(index_string)
# Write the hash out to a local file
index_hash_path = os.path.join(db_root_dir, 'index.json.hash')
with open(index_hash_path, 'w') as f:
f.write(index_hash)
# Push the index itself
web_util.push_to_url( web_util.push_to_url(
index_json_path, index_json_path,
url_util.join(cache_prefix, 'index.json'), url_util.join(cache_prefix, 'index.json'),
keep_original=False, keep_original=False,
extra_args={'ContentType': 'application/json'}) extra_args={'ContentType': 'application/json'})
# Push the hash
web_util.push_to_url(
index_hash_path,
url_util.join(cache_prefix, 'index.json.hash'),
keep_original=False,
extra_args={'ContentType': 'text/plain'})
finally: finally:
shutil.rmtree(tmpdir) shutil.rmtree(tmpdir)
@ -470,13 +767,6 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
spec.prefix, spack.store.layout.root) spec.prefix, spack.store.layout.root)
buildinfo['relative_rpaths'] = rel buildinfo['relative_rpaths'] = rel
spec_dict['buildinfo'] = buildinfo spec_dict['buildinfo'] = buildinfo
spec_dict['full_hash'] = spec.full_hash()
tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format(
spec_dict['full_hash'],
spec.name,
url_util.format(remote_specfile_path)))
tty.debug(spec.tree())
with open(specfile_path, 'w') as outfile: with open(specfile_path, 'w') as outfile:
outfile.write(syaml.dump(spec_dict)) outfile.write(syaml.dump(spec_dict))
@ -527,10 +817,20 @@ def build_tarball(spec, outdir, force=False, rel=False, unsigned=False,
return None return None
def download_tarball(spec): def download_tarball(spec, url=None):
""" """
Download binary tarball for given package into stage area Download binary tarball for given package into stage area
Return True if successful Return True if successful
Args:
spec (Spec): Concrete spec
url (str): If provided, this is the preferred url, other configured
mirrors will only be used if the tarball can't be retrieved from
this preferred url.
Returns:
Path to the downloaded tarball, or ``None`` if the tarball could not
be downloaded from any configured mirrors.
""" """
if not spack.mirror.MirrorCollection(): if not spack.mirror.MirrorCollection():
tty.die("Please add a spack mirror to allow " + tty.die("Please add a spack mirror to allow " +
@ -538,12 +838,20 @@ def download_tarball(spec):
tarball = tarball_path_name(spec, '.spack') tarball = tarball_path_name(spec, '.spack')
for mirror in spack.mirror.MirrorCollection().values(): urls_to_try = []
url = url_util.join(
mirror.fetch_url, _build_cache_relative_path, tarball)
if url:
urls_to_try.append(url_util.join(
url, _build_cache_relative_path, tarball))
for mirror in spack.mirror.MirrorCollection().values():
if not url or url != mirror.fetch_url:
urls_to_try.append(url_util.join(
mirror.fetch_url, _build_cache_relative_path, tarball))
for try_url in urls_to_try:
# stage the tarball into standard place # stage the tarball into standard place
stage = Stage(url, name="build_cache", keep=True) stage = Stage(try_url, name="build_cache", keep=True)
stage.create() stage.create()
try: try:
stage.fetch() stage.fetch()
@ -551,6 +859,8 @@ def download_tarball(spec):
except fs.FetchError: except fs.FetchError:
continue continue
# If we arrive here, something went wrong, maybe this should be an
# exception?
return None return None
@ -834,117 +1144,95 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
os.remove(filename) os.remove(filename)
# Internal cache for downloaded specs def try_direct_fetch(spec, force=False, full_hash_match=False):
_cached_specs = set()
def try_download_specs(urls=None, force=False):
'''
Try to download the urls and cache them
'''
global _cached_specs
if urls is None:
return {}
for link in urls:
with Stage(link, name="build_cache", keep=True) as stage:
if force and os.path.exists(stage.save_filename):
os.remove(stage.save_filename)
if not os.path.exists(stage.save_filename):
try:
stage.fetch()
except fs.FetchError:
continue
with open(stage.save_filename, 'r') as f:
# read the spec from the build cache file. All specs
# in build caches are concrete (as they are built) so
# we need to mark this spec concrete on read-in.
spec = Spec.from_yaml(f)
spec._mark_concrete()
_cached_specs.add(spec)
return _cached_specs
def get_spec(spec=None, force=False):
""" """
Check if spec.yaml exists on mirrors and return it if it does Try to find the spec directly on the configured mirrors
""" """
global _cached_specs
urls = set()
if spec is None:
return {}
specfile_name = tarball_name(spec, '.spec.yaml') specfile_name = tarball_name(spec, '.spec.yaml')
lenient = not full_hash_match
found_specs = []
for mirror in spack.mirror.MirrorCollection().values():
buildcache_fetch_url = url_util.join(
mirror.fetch_url, _build_cache_relative_path, specfile_name)
try:
_, _, fs = web_util.read_from_url(
buildcache_fetch_url, 'text/plain')
fetched_spec_yaml = codecs.getreader('utf-8')(fs).read()
except (URLError, web_util.SpackWebError, HTTPError) as url_err:
tty.debug('Did not find {0} on {1}'.format(
specfile_name, buildcache_fetch_url), url_err)
continue
# read the spec from the build cache file. All specs in build caches
# are concrete (as they are built) so we need to mark this spec
# concrete on read-in.
fetched_spec = Spec.from_yaml(fetched_spec_yaml)
fetched_spec._mark_concrete()
# Do not recompute the full hash for the fetched spec, instead just
# read the property.
if lenient or fetched_spec._full_hash == spec.full_hash():
found_specs.append({
'mirror_url': mirror.fetch_url,
'spec': fetched_spec,
})
return found_specs
def get_spec(spec=None, force=False, full_hash_match=False):
"""
Check if concrete spec exists on mirrors and return an object
indicating the mirrors on which it can be found
"""
if spec is None:
return []
if not spack.mirror.MirrorCollection(): if not spack.mirror.MirrorCollection():
tty.debug("No Spack mirrors are currently configured") tty.debug("No Spack mirrors are currently configured")
return {} return {}
if _cached_specs and spec in _cached_specs: results = []
return _cached_specs lenient = not full_hash_match
for mirror in spack.mirror.MirrorCollection().values(): def filter_candidates(possibles):
fetch_url_build_cache = url_util.join( filtered_candidates = []
mirror.fetch_url, _build_cache_relative_path) for candidate in possibles:
spec_full_hash = spec.full_hash()
candidate_full_hash = candidate['spec']._full_hash
if lenient or spec_full_hash == candidate_full_hash:
filtered_candidates.append(candidate)
return filtered_candidates
mirror_dir = url_util.local_file_path(fetch_url_build_cache) candidates = cache_manager.find_built_spec(spec)
if mirror_dir: if candidates:
tty.debug('Finding buildcaches in {0}'.format(mirror_dir)) results = filter_candidates(candidates)
link = url_util.join(fetch_url_build_cache, specfile_name)
urls.add(link)
else: # Maybe we just didn't have the latest information from the mirror, so
tty.debug('Finding buildcaches at {0}' # try to fetch directly.
.format(url_util.format(fetch_url_build_cache))) if not results:
link = url_util.join(fetch_url_build_cache, specfile_name) results = try_direct_fetch(spec,
urls.add(link) force=force,
full_hash_match=full_hash_match)
if results:
cache_manager.update_spec(spec, results)
return try_download_specs(urls=urls, force=force) return results
def get_specs(): def get_specs():
""" """
Get spec.yaml's for build caches available on mirror Get concrete specs for build caches available on mirrors
""" """
global _cached_specs cache_manager.update_local_index_cache()
cache_manager.regenerate_spec_cache()
return cache_manager.get_all_built_specs()
if not spack.mirror.MirrorCollection():
tty.debug("No Spack mirrors are currently configured")
return {}
for mirror in spack.mirror.MirrorCollection().values(): def clear_spec_cache():
fetch_url_build_cache = url_util.join( cache_manager.clear_spec_cache()
mirror.fetch_url, _build_cache_relative_path)
tty.debug('Finding buildcaches at {0}'
.format(url_util.format(fetch_url_build_cache)))
index_url = url_util.join(fetch_url_build_cache, 'index.json')
try:
_, _, file_stream = web_util.read_from_url(
index_url, 'application/json')
index_object = codecs.getreader('utf-8')(file_stream).read()
except (URLError, web_util.SpackWebError) as url_err:
tty.debug('Failed to read index {0}'.format(index_url), url_err, 1)
# Continue on to the next mirror
continue
tmpdir = tempfile.mkdtemp()
index_file_path = os.path.join(tmpdir, 'index.json')
with open(index_file_path, 'w') as fd:
fd.write(index_object)
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False)
db._read_from_file(index_file_path)
spec_list = db.query_local(installed=False)
for indexed_spec in spec_list:
_cached_specs.add(indexed_spec)
return _cached_specs
def get_keys(install=False, trust=False, force=False, mirrors=None): def get_keys(install=False, trust=False, force=False, mirrors=None):

6
lib/spack/spack/cmd/install.py
View File

@ -45,6 +45,7 @@ def update_kwargs_from_args(args, kwargs):
'explicit': True, # Always true for install command 'explicit': True, # Always true for install command
'stop_at': args.until, 'stop_at': args.until,
'unsigned': args.unsigned, 'unsigned': args.unsigned,
'full_hash_match': args.full_hash_match,
}) })
kwargs.update({ kwargs.update({
@ -107,6 +108,11 @@ def setup_parser(subparser):
'--no-check-signature', action='store_true', '--no-check-signature', action='store_true',
dest='unsigned', default=False, dest='unsigned', default=False,
help="do not check signatures of binary packages") help="do not check signatures of binary packages")
subparser.add_argument(
'--require-full-hash-match', action='store_true',
dest='full_hash_match', default=False, help="""when installing from
binary mirrors, do not install binary package unless the full hash of the
remote spec matches that of the local spec""")
subparser.add_argument( subparser.add_argument(
'--show-log-on-error', action='store_true', '--show-log-on-error', action='store_true',
help="print full build log to stderr if build fails") help="print full build log to stderr if build fails")

1
lib/spack/spack/database.py
View File

@ -1112,6 +1112,7 @@ def _add(
# the original hash of concrete specs. # the original hash of concrete specs.
new_spec._mark_concrete() new_spec._mark_concrete()
new_spec._hash = key new_spec._hash = key
new_spec._full_hash = spec._full_hash
else: else:
# If it is already there, mark it as installed. # If it is already there, mark it as installed.

35
lib/spack/spack/installer.py
View File

@ -213,7 +213,8 @@ def _hms(seconds):
return ' '.join(parts) return ' '.join(parts)
def _install_from_cache(pkg, cache_only, explicit, unsigned=False): def _install_from_cache(pkg, cache_only, explicit, unsigned=False,
full_hash_match=False):
""" """
Extract the package from binary cache Extract the package from binary cache
@ -229,8 +230,8 @@ def _install_from_cache(pkg, cache_only, explicit, unsigned=False):
(bool) ``True`` if the package was extract from binary cache, (bool) ``True`` if the package was extract from binary cache,
``False`` otherwise ``False`` otherwise
""" """
installed_from_cache = _try_install_from_binary_cache(pkg, explicit, installed_from_cache = _try_install_from_binary_cache(
unsigned) pkg, explicit, unsigned=unsigned, full_hash_match=full_hash_match)
pkg_id = package_id(pkg) pkg_id = package_id(pkg)
if not installed_from_cache: if not installed_from_cache:
pre = 'No binary for {0} found'.format(pkg_id) pre = 'No binary for {0} found'.format(pkg_id)
@ -302,7 +303,8 @@ def _process_external_package(pkg, explicit):
spack.store.db.add(spec, None, explicit=explicit) spack.store.db.add(spec, None, explicit=explicit)
def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned): def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned,
preferred_mirror_url=None):
""" """
Process the binary cache tarball. Process the binary cache tarball.
@ -317,7 +319,8 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned):
(bool) ``True`` if the package was extracted from binary cache, (bool) ``True`` if the package was extracted from binary cache,
else ``False`` else ``False``
""" """
tarball = binary_distribution.download_tarball(binary_spec) tarball = binary_distribution.download_tarball(binary_spec,
url=preferred_mirror_url)
# see #10063 : install from source if tarball doesn't exist # see #10063 : install from source if tarball doesn't exist
if tarball is None: if tarball is None:
tty.msg('{0} exists in binary cache but with different hash' tty.msg('{0} exists in binary cache but with different hash'
@ -333,7 +336,8 @@ def _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned):
return True return True
def _try_install_from_binary_cache(pkg, explicit, unsigned=False): def _try_install_from_binary_cache(pkg, explicit, unsigned=False,
full_hash_match=False):
""" """
Try to extract the package from binary cache. Try to extract the package from binary cache.
@ -345,13 +349,18 @@ def _try_install_from_binary_cache(pkg, explicit, unsigned=False):
""" """
pkg_id = package_id(pkg) pkg_id = package_id(pkg)
tty.debug('Searching for binary cache of {0}'.format(pkg_id)) tty.debug('Searching for binary cache of {0}'.format(pkg_id))
specs = binary_distribution.get_spec(pkg.spec, force=False) matches = binary_distribution.get_spec(
binary_spec = spack.spec.Spec.from_dict(pkg.spec.to_dict()) pkg.spec, force=False, full_hash_match=full_hash_match)
binary_spec._mark_concrete()
if binary_spec not in specs: if not matches:
return False return False
return _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned) # In the absence of guidance from user or some other reason to prefer one
# mirror over another, any match will suffice, so just pick the first one.
preferred_mirror = matches[0]['mirror_url']
binary_spec = matches[0]['spec']
return _process_binary_cache_tarball(pkg, binary_spec, explicit, unsigned,
preferred_mirror_url=preferred_mirror)
def _update_explicit_entry_in_db(pkg, rec, explicit): def _update_explicit_entry_in_db(pkg, rec, explicit):
@ -1044,6 +1053,7 @@ def _install_task(self, task, **kwargs):
unsigned = kwargs.get('unsigned', False) unsigned = kwargs.get('unsigned', False)
use_cache = kwargs.get('use_cache', True) use_cache = kwargs.get('use_cache', True)
verbose = kwargs.get('verbose', False) verbose = kwargs.get('verbose', False)
full_hash_match = kwargs.get('full_hash_match', False)
pkg = task.pkg pkg = task.pkg
pkg_id = package_id(pkg) pkg_id = package_id(pkg)
@ -1055,7 +1065,8 @@ def _install_task(self, task, **kwargs):
# Use the binary cache if requested # Use the binary cache if requested
if use_cache and \ if use_cache and \
_install_from_cache(pkg, cache_only, explicit, unsigned): _install_from_cache(pkg, cache_only, explicit, unsigned,
full_hash_match):
self._update_installed(task) self._update_installed(task)
if task.compiler: if task.compiler:
spack.compilers.add_compilers_to_config( spack.compilers.add_compilers_to_config(

1
lib/spack/spack/schema/buildcache_spec.py
View File

@ -26,7 +26,6 @@
'relative_rpaths': {'type': 'boolean'}, 'relative_rpaths': {'type': 'boolean'},
}, },
}, },
'full_hash': {'type': 'string'},
'spec': { 'spec': {
'type': 'array', 'type': 'array',
'items': spack.schema.spec.properties, 'items': spack.schema.spec.properties,

1
lib/spack/spack/schema/config.py
View File

@ -87,6 +87,7 @@
], ],
}, },
'allow_sgid': {'type': 'boolean'}, 'allow_sgid': {'type': 'boolean'},
'binary_distribution_cache_root': {'type': 'string'},
}, },
}, },
} }

1
lib/spack/spack/schema/spec.py
View File

@ -81,6 +81,7 @@
], ],
'properties': { 'properties': {
'hash': {'type': 'string'}, 'hash': {'type': 'string'},
'full_hash': {'type': 'string'},
'version': { 'version': {
'oneOf': [ 'oneOf': [
{'type': 'string'}, {'type': 'string'},

3
lib/spack/spack/spec.py
View File

@ -1552,6 +1552,8 @@ def to_node_dict(self, hash=ht.dag_hash):
if not self._concrete: if not self._concrete:
d['concrete'] = False d['concrete'] = False
else:
d['full_hash'] = self._full_hash
if 'patches' in self.variants: if 'patches' in self.variants:
variant = self.variants['patches'] variant = self.variants['patches']
@ -1734,6 +1736,7 @@ def from_node_dict(node):
# specs read in are concrete unless marked abstract # specs read in are concrete unless marked abstract
spec._concrete = node.get('concrete', True) spec._concrete = node.get('concrete', True)
spec._full_hash = node.get('full_hash', None)
if 'patches' in node: if 'patches' in node:
patches = node['patches'] patches = node['patches']

69
lib/spack/spack/test/bindist.py
View File

@ -302,7 +302,7 @@ def test_default_rpaths_install_nondefault_layout(tmpdir,
args = parser.parse_args(install_args) args = parser.parse_args(install_args)
buildcache.buildcache(parser, args) buildcache.buildcache(parser, args)
bindist._cached_specs = set() bindist.clear_spec_cache()
spack.stage.purge() spack.stage.purge()
margs = mparser.parse_args( margs = mparser.parse_args(
['rm', '--scope', 'site', 'test-mirror-def']) ['rm', '--scope', 'site', 'test-mirror-def'])
@ -358,7 +358,7 @@ def test_relative_rpaths_create_default_layout(tmpdir,
uargs = uparser.parse_args(['-y', '--dependents', gspec.name]) uargs = uparser.parse_args(['-y', '--dependents', gspec.name])
uninstall.uninstall(uparser, uargs) uninstall.uninstall(uparser, uargs)
bindist._cached_specs = set() bindist.clear_spec_cache()
spack.stage.purge() spack.stage.purge()
@ -395,7 +395,7 @@ def test_relative_rpaths_install_default_layout(tmpdir,
buildcache.setup_parser(parser) buildcache.setup_parser(parser)
# set default buildcache args # set default buildcache args
install_args = ['install', '-a', '-u', install_args = ['install', '-a', '-u', '-f',
cspec.name] cspec.name]
# install buildcache created with relativized rpaths # install buildcache created with relativized rpaths
@ -460,7 +460,7 @@ def test_relative_rpaths_install_nondefault(tmpdir,
buildcache.setup_parser(parser) buildcache.setup_parser(parser)
# Set default buildcache args # Set default buildcache args
install_args = ['install', '-a', '-u', '%s' % cspec.name] install_args = ['install', '-a', '-u', '-f', '%s' % cspec.name]
# test install in non-default install path scheme and relative path # test install in non-default install path scheme and relative path
args = parser.parse_args(install_args) args = parser.parse_args(install_args)
@ -508,3 +508,64 @@ def test_push_and_fetch_keys(mock_gnupghome):
new_keys = spack.util.gpg.public_keys() new_keys = spack.util.gpg.public_keys()
assert len(new_keys) == 1 assert len(new_keys) == 1
assert new_keys[0] == fpr assert new_keys[0] == fpr
@pytest.mark.requires_executables(*args)
@pytest.mark.disable_clean_stage_check
@pytest.mark.maybeslow
@pytest.mark.nomockstage
@pytest.mark.usefixtures('default_config', 'cache_directory',
'install_dir_non_default_layout')
def test_built_spec_cache(tmpdir,
install_mockery):
""" Test what's the situation now """
global mirror_path_rel
mparser = argparse.ArgumentParser()
mirror.setup_parser(mparser)
margs = mparser.parse_args(
['add', '--scope', 'site', 'test-mirror-rel', 'file://%s' % mirror_path_rel])
mirror.mirror(mparser, margs)
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
list_args = ['list', '-a', '-l']
args = parser.parse_args(list_args)
buildcache.buildcache(parser, args)
gspec = Spec('garply')
gspec.concretize()
cspec = Spec('corge')
cspec.concretize()
full_hash_map = {
'garply': gspec.full_hash(),
'corge': cspec.full_hash(),
}
gspec_results = bindist.get_spec(gspec)
gspec_mirrors = {}
for result in gspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in gspec_mirrors)
gspec_mirrors[result['mirror_url']] = True
cspec_results = bindist.get_spec(cspec, full_hash_match=True)
cspec_mirrors = {}
for result in cspec_results:
s = result['spec']
assert(s._full_hash == full_hash_map[s.name])
assert(result['mirror_url'] not in cspec_mirrors)
cspec_mirrors[result['mirror_url']] = True
bindist.clear_spec_cache()
margs = mparser.parse_args(
['rm', '--scope', 'site', 'test-mirror-rel'])
mirror.mirror(mparser, margs)

58
lib/spack/spack/test/cmd/install.py
View File

@ -809,3 +809,61 @@ def test_install_fails_no_args_suggests_env_activation(tmpdir):
assert 'requires a package argument or active environment' in output assert 'requires a package argument or active environment' in output
assert 'spack env activate .' in output assert 'spack env activate .' in output
assert 'using the `spack.yaml` in this directory' in output assert 'using the `spack.yaml` in this directory' in output
def test_cache_install_full_hash_match(
install_mockery_mutable_config, mock_packages, mock_fetch,
mock_archive, mutable_config, monkeypatch, tmpdir):
"""Make sure installing from cache respects full hash argument"""
# Create a temp mirror directory for buildcache usage
mirror_dir = tmpdir.join('mirror_dir')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
s = Spec('libdwarf').concretized()
# Install a package
install(s.name)
# Put installed package in the buildcache
buildcache('create', '-u', '-a', '-f', '-d', mirror_dir.strpath, s.name)
# Now uninstall the package
uninstall('-y', s.name)
# Configure the mirror with the binary package in it
mirror('add', 'test-mirror', mirror_url)
# Make sure we get the binary version by default
install_output = install('--no-check-signature', s.name, output=str)
expect_msg = 'Extracting {0} from binary cache'.format(s.name)
assert expect_msg in install_output
uninstall('-y', s.name)
# Now monkey patch Spec to change the full hash on the package
def fake_full_hash(spec):
print('fake_full_hash')
return 'tal4c7h4z0gqmixb1eqa92mjoybxn5l6'
monkeypatch.setattr(spack.spec.Spec, 'full_hash', fake_full_hash)
# Check that even if the full hash changes, we install from binary when
# we don't explicitly require the full hash to match
install_output = install('--no-check-signature', s.name, output=str)
expect_msg = 'Extracting {0} from binary cache'.format(s.name)
assert expect_msg in install_output
uninstall('-y', s.name)
# Finally, make sure that if we insist on the full hash match, spack
# installs from source.
install_output = install('--require-full-hash-match', s.name, output=str)
expect_msg = 'No binary for {0} found: installing from source'.format(
s.name)
assert expect_msg in install_output
uninstall('-y', s.name)
mirror('rm', 'test-mirror')

9
lib/spack/spack/test/installer.py
View File

@ -177,7 +177,7 @@ def test_process_binary_cache_tarball_none(install_mockery, monkeypatch,
def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd): def test_process_binary_cache_tarball_tar(install_mockery, monkeypatch, capfd):
"""Tests of _process_binary_cache_tarball with a tar file.""" """Tests of _process_binary_cache_tarball with a tar file."""
def _spec(spec): def _spec(spec, url=None):
return spec return spec
# Skip binary distribution functionality since assume tested elsewhere # Skip binary distribution functionality since assume tested elsewhere
@ -196,9 +196,12 @@ def _spec(spec):
def test_try_install_from_binary_cache(install_mockery, mock_packages, def test_try_install_from_binary_cache(install_mockery, mock_packages,
monkeypatch, capsys): monkeypatch, capsys):
"""Tests SystemExit path for_try_install_from_binary_cache.""" """Tests SystemExit path for_try_install_from_binary_cache."""
def _spec(spec, force): def _spec(spec, force, full_hash_match=False):
spec = spack.spec.Spec('mpi').concretized() spec = spack.spec.Spec('mpi').concretized()
return {spec: None} return [{
'mirror_url': 'notused',
'spec': spec,
}]
spec = spack.spec.Spec('mpich') spec = spack.spec.Spec('mpich')
spec.concretize() spec.concretize()

2
share/spack/spack-completion.bash
View File

@ -1011,7 +1011,7 @@ _spack_info() {
_spack_install() { _spack_install() {
if $list_options if $list_options
then then
SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --no-check-signature --show-log-on-error --source -n --no-checksum -v --verbose --fake --only-concrete -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash -y --yes-to-all --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp" SPACK_COMPREPLY="-h --help --only -u --until -j --jobs --overwrite --fail-fast --keep-prefix --keep-stage --dont-restage --use-cache --no-cache --cache-only --no-check-signature --require-full-hash-match --show-log-on-error --source -n --no-checksum -v --verbose --fake --only-concrete -f --file --clean --dirty --test --run-tests --log-format --log-file --help-cdash -y --yes-to-all --cdash-upload-url --cdash-build --cdash-site --cdash-track --cdash-buildstamp"
else else
_all_packages _all_packages
fi fi