bugfix: mirror index shows missing packages (#23939)

- [x] add `in_buildcache` field to DB records to indicate what parts of an index, 
       which includes roots and dependencies, are in the buildcache.
- [x] add `mark()` method to DB for setting values on single nodes of the DAG.
This commit is contained in:
Greg Becker 2021-05-27 13:38:13 -07:00 committed by GitHub
parent 214182529f
commit b91dff4aaf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 66 additions and 4 deletions

View File

@ -156,7 +156,7 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
with self._index_file_cache.read_transaction(cache_key):
db._read_from_file(cache_path)
spec_list = db.query_local(installed=False)
spec_list = db.query_local(installed=False, in_buildcache=True)
for indexed_spec in spec_list:
dag_hash = indexed_spec.dag_hash()
@ -716,7 +716,7 @@ def generate_package_index(cache_prefix):
db_root_dir = os.path.join(tmpdir, 'db_root')
db = spack_db.Database(None, db_dir=db_root_dir,
enable_transaction_locking=False,
record_fields=['spec', 'ref_count'])
record_fields=['spec', 'ref_count', 'in_buildcache'])
try:
file_list = (
@ -748,6 +748,7 @@ def generate_package_index(cache_prefix):
# s = Spec.from_yaml(yaml_obj)
s = Spec.from_yaml(yaml_contents)
db.add(s, None)
db.mark(s, 'in_buildcache', True)
except (URLError, web_util.SpackWebError) as url_err:
tty.error('Error reading spec.yaml: {0}'.format(file_path))
tty.error(url_err)

View File

@ -187,7 +187,8 @@ def __init__(
ref_count=0,
explicit=False,
installation_time=None,
deprecated_for=None
deprecated_for=None,
in_buildcache=False,
):
self.spec = spec
self.path = str(path) if path else None
@ -196,6 +197,7 @@ def __init__(
self.explicit = explicit
self.installation_time = installation_time or _now()
self.deprecated_for = deprecated_for
self.in_buildcache = in_buildcache
def install_type_matches(self, installed):
installed = InstallStatuses.canonicalize(installed)
@ -282,6 +284,11 @@ def __getattribute__(self, name):
hashes (container): list or set of hashes that we can use to
restrict the search
in_buildcache (bool or any, optional): Specs that are marked in
this database as part of an associated binary cache are
``in_buildcache``. All other specs are not. This field is used
for querying mirror indices. Default is ``any``.
Returns:
list of specs that match the query
@ -1255,6 +1262,16 @@ def _deprecate(self, spec, deprecator):
spec_rec.installed = False
self._data[spec_key] = spec_rec
@_autospec
def mark(self, spec, key, value):
"""Mark an arbitrary record on a spec."""
with self.write_transaction():
return self._mark(spec, key, value)
def _mark(self, spec, key, value):
record = self._data[self._get_matching_spec_key(spec)]
setattr(record, key, value)
@_autospec
def deprecate(self, spec, deprecator):
"""Marks a spec as deprecated in favor of its deprecator"""
@ -1415,7 +1432,8 @@ def _query(
explicit=any,
start_date=None,
end_date=None,
hashes=None
hashes=None,
in_buildcache=any,
):
"""Run a query on the database."""
@ -1447,6 +1465,9 @@ def _query(
if not rec.install_type_matches(installed):
continue
if in_buildcache is not any and rec.in_buildcache != in_buildcache:
continue
if explicit is not any and rec.explicit != explicit:
continue

View File

@ -2,6 +2,7 @@
# Spack Project Developers. See the top-level COPYRIGHT file for details.
#
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
import glob
import os
import sys
import platform
@ -435,6 +436,45 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
assert rebuild
@pytest.mark.usefixtures(
'install_mockery_mutable_config', 'mock_packages', 'mock_fetch',
)
def test_generate_index_missing(monkeypatch, tmpdir, mutable_config):
"""Ensure spack buildcache index only reports available packages"""
# Create a temp mirror directory for buildcache usage
mirror_dir = tmpdir.join('mirror_dir')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
spack.config.set('mirrors', {'test': mirror_url})
s = Spec('libdwarf').concretized()
# Install a package
install_cmd('--no-cache', s.name)
# Create a buildcache and update index
buildcache_cmd('create', '-uad', mirror_dir.strpath, s.name)
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
# Check package and dependency in buildcache
cache_list = buildcache_cmd('list', '--allarch')
assert 'libdwarf' in cache_list
assert 'libelf' in cache_list
# Remove dependency from cache
libelf_files = glob.glob(
os.path.join(mirror_dir.join('build_cache').strpath, '*libelf*'))
os.remove(*libelf_files)
# Update index
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
# Check dependency not in buildcache
cache_list = buildcache_cmd('list', '--allarch')
assert 'libdwarf' in cache_list
assert 'libelf' not in cache_list
def test_generate_indices_key_error(monkeypatch, capfd):
def mock_list_url(url, recursive=False):