bugfix: mirror index shows missing packages (#23939)
- [x] add `in_buildcache` field to DB records to indicate what parts of an index, which includes roots and dependencies, are in the buildcache. - [x] add `mark()` method to DB for setting values on single nodes of the DAG.
This commit is contained in:
parent
214182529f
commit
b91dff4aaf
@ -156,7 +156,7 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
|
||||
with self._index_file_cache.read_transaction(cache_key):
|
||||
db._read_from_file(cache_path)
|
||||
|
||||
spec_list = db.query_local(installed=False)
|
||||
spec_list = db.query_local(installed=False, in_buildcache=True)
|
||||
|
||||
for indexed_spec in spec_list:
|
||||
dag_hash = indexed_spec.dag_hash()
|
||||
@ -716,7 +716,7 @@ def generate_package_index(cache_prefix):
|
||||
db_root_dir = os.path.join(tmpdir, 'db_root')
|
||||
db = spack_db.Database(None, db_dir=db_root_dir,
|
||||
enable_transaction_locking=False,
|
||||
record_fields=['spec', 'ref_count'])
|
||||
record_fields=['spec', 'ref_count', 'in_buildcache'])
|
||||
|
||||
try:
|
||||
file_list = (
|
||||
@ -748,6 +748,7 @@ def generate_package_index(cache_prefix):
|
||||
# s = Spec.from_yaml(yaml_obj)
|
||||
s = Spec.from_yaml(yaml_contents)
|
||||
db.add(s, None)
|
||||
db.mark(s, 'in_buildcache', True)
|
||||
except (URLError, web_util.SpackWebError) as url_err:
|
||||
tty.error('Error reading spec.yaml: {0}'.format(file_path))
|
||||
tty.error(url_err)
|
||||
|
@ -187,7 +187,8 @@ def __init__(
|
||||
ref_count=0,
|
||||
explicit=False,
|
||||
installation_time=None,
|
||||
deprecated_for=None
|
||||
deprecated_for=None,
|
||||
in_buildcache=False,
|
||||
):
|
||||
self.spec = spec
|
||||
self.path = str(path) if path else None
|
||||
@ -196,6 +197,7 @@ def __init__(
|
||||
self.explicit = explicit
|
||||
self.installation_time = installation_time or _now()
|
||||
self.deprecated_for = deprecated_for
|
||||
self.in_buildcache = in_buildcache
|
||||
|
||||
def install_type_matches(self, installed):
|
||||
installed = InstallStatuses.canonicalize(installed)
|
||||
@ -282,6 +284,11 @@ def __getattribute__(self, name):
|
||||
hashes (container): list or set of hashes that we can use to
|
||||
restrict the search
|
||||
|
||||
in_buildcache (bool or any, optional): Specs that are marked in
|
||||
this database as part of an associated binary cache are
|
||||
``in_buildcache``. All other specs are not. This field is used
|
||||
for querying mirror indices. Default is ``any``.
|
||||
|
||||
Returns:
|
||||
list of specs that match the query
|
||||
|
||||
@ -1255,6 +1262,16 @@ def _deprecate(self, spec, deprecator):
|
||||
spec_rec.installed = False
|
||||
self._data[spec_key] = spec_rec
|
||||
|
||||
@_autospec
|
||||
def mark(self, spec, key, value):
|
||||
"""Mark an arbitrary record on a spec."""
|
||||
with self.write_transaction():
|
||||
return self._mark(spec, key, value)
|
||||
|
||||
def _mark(self, spec, key, value):
|
||||
record = self._data[self._get_matching_spec_key(spec)]
|
||||
setattr(record, key, value)
|
||||
|
||||
@_autospec
|
||||
def deprecate(self, spec, deprecator):
|
||||
"""Marks a spec as deprecated in favor of its deprecator"""
|
||||
@ -1415,7 +1432,8 @@ def _query(
|
||||
explicit=any,
|
||||
start_date=None,
|
||||
end_date=None,
|
||||
hashes=None
|
||||
hashes=None,
|
||||
in_buildcache=any,
|
||||
):
|
||||
"""Run a query on the database."""
|
||||
|
||||
@ -1447,6 +1465,9 @@ def _query(
|
||||
if not rec.install_type_matches(installed):
|
||||
continue
|
||||
|
||||
if in_buildcache is not any and rec.in_buildcache != in_buildcache:
|
||||
continue
|
||||
|
||||
if explicit is not any and rec.explicit != explicit:
|
||||
continue
|
||||
|
||||
|
@ -2,6 +2,7 @@
|
||||
# Spack Project Developers. See the top-level COPYRIGHT file for details.
|
||||
#
|
||||
# SPDX-License-Identifier: (Apache-2.0 OR MIT)
|
||||
import glob
|
||||
import os
|
||||
import sys
|
||||
import platform
|
||||
@ -435,6 +436,45 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
|
||||
assert rebuild
|
||||
|
||||
|
||||
@pytest.mark.usefixtures(
|
||||
'install_mockery_mutable_config', 'mock_packages', 'mock_fetch',
|
||||
)
|
||||
def test_generate_index_missing(monkeypatch, tmpdir, mutable_config):
|
||||
"""Ensure spack buildcache index only reports available packages"""
|
||||
|
||||
# Create a temp mirror directory for buildcache usage
|
||||
mirror_dir = tmpdir.join('mirror_dir')
|
||||
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
|
||||
spack.config.set('mirrors', {'test': mirror_url})
|
||||
|
||||
s = Spec('libdwarf').concretized()
|
||||
|
||||
# Install a package
|
||||
install_cmd('--no-cache', s.name)
|
||||
|
||||
# Create a buildcache and update index
|
||||
buildcache_cmd('create', '-uad', mirror_dir.strpath, s.name)
|
||||
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
|
||||
|
||||
# Check package and dependency in buildcache
|
||||
cache_list = buildcache_cmd('list', '--allarch')
|
||||
assert 'libdwarf' in cache_list
|
||||
assert 'libelf' in cache_list
|
||||
|
||||
# Remove dependency from cache
|
||||
libelf_files = glob.glob(
|
||||
os.path.join(mirror_dir.join('build_cache').strpath, '*libelf*'))
|
||||
os.remove(*libelf_files)
|
||||
|
||||
# Update index
|
||||
buildcache_cmd('update-index', '-d', mirror_dir.strpath)
|
||||
|
||||
# Check dependency not in buildcache
|
||||
cache_list = buildcache_cmd('list', '--allarch')
|
||||
assert 'libdwarf' in cache_list
|
||||
assert 'libelf' not in cache_list
|
||||
|
||||
|
||||
def test_generate_indices_key_error(monkeypatch, capfd):
|
||||
|
||||
def mock_list_url(url, recursive=False):
|
||||
|
Loading…
Reference in New Issue
Block a user