bugfix: mirror index shows missing packages (#23939)

- [x] add `in_buildcache` field to DB records to indicate what parts of an index, 
       which includes roots and dependencies, are in the buildcache.
- [x] add `mark()` method to DB for setting values on single nodes of the DAG.

lib/spack/spack/binary_distribution.py

@@ -156,7 +156,7 @@ def _associate_built_specs_with_mirror(self, cache_key, mirror_url):
             with self._index_file_cache.read_transaction(cache_key):
                 db._read_from_file(cache_path)
 
-            spec_list = db.query_local(installed=False)
+            spec_list = db.query_local(installed=False, in_buildcache=True)
 
             for indexed_spec in spec_list:
                 dag_hash = indexed_spec.dag_hash()
@@ -716,7 +716,7 @@ def generate_package_index(cache_prefix):
     db_root_dir = os.path.join(tmpdir, 'db_root')
     db = spack_db.Database(None, db_dir=db_root_dir,
                            enable_transaction_locking=False,
-                           record_fields=['spec', 'ref_count'])
+                           record_fields=['spec', 'ref_count', 'in_buildcache'])
 
     try:
         file_list = (
@@ -748,6 +748,7 @@ def generate_package_index(cache_prefix):
             # s = Spec.from_yaml(yaml_obj)
             s = Spec.from_yaml(yaml_contents)
             db.add(s, None)
+            db.mark(s, 'in_buildcache', True)
         except (URLError, web_util.SpackWebError) as url_err:
             tty.error('Error reading spec.yaml: {0}'.format(file_path))
             tty.error(url_err)

lib/spack/spack/database.py

@@ -187,7 +187,8 @@ def __init__(
             ref_count=0,
             explicit=False,
             installation_time=None,
-            deprecated_for=None
+            deprecated_for=None,
+            in_buildcache=False,
     ):
         self.spec = spec
         self.path = str(path) if path else None
@@ -196,6 +197,7 @@ def __init__(
         self.explicit = explicit
         self.installation_time = installation_time or _now()
         self.deprecated_for = deprecated_for
+        self.in_buildcache = in_buildcache
 
     def install_type_matches(self, installed):
         installed = InstallStatuses.canonicalize(installed)
@@ -282,6 +284,11 @@ def __getattribute__(self, name):
             hashes (container): list or set of hashes that we can use to
                 restrict the search
 
+            in_buildcache (bool or any, optional): Specs that are marked in
+                this database as part of an associated binary cache are
+                ``in_buildcache``. All other specs are not. This field is used
+                for querying mirror indices. Default is ``any``.
+
         Returns:
             list of specs that match the query
 
@@ -1255,6 +1262,16 @@ def _deprecate(self, spec, deprecator):
         spec_rec.installed = False
         self._data[spec_key] = spec_rec
 
+    @_autospec
+    def mark(self, spec, key, value):
+        """Mark an arbitrary record on a spec."""
+        with self.write_transaction():
+            return self._mark(spec, key, value)
+
+    def _mark(self, spec, key, value):
+        record = self._data[self._get_matching_spec_key(spec)]
+        setattr(record, key, value)
+
     @_autospec
     def deprecate(self, spec, deprecator):
         """Marks a spec as deprecated in favor of its deprecator"""
@@ -1415,7 +1432,8 @@ def _query(
             explicit=any,
             start_date=None,
             end_date=None,
-            hashes=None
+            hashes=None,
+            in_buildcache=any,
     ):
         """Run a query on the database."""
 
@@ -1447,6 +1465,9 @@ def _query(
             if not rec.install_type_matches(installed):
                 continue
 
+            if in_buildcache is not any and rec.in_buildcache != in_buildcache:
+                continue
+
             if explicit is not any and rec.explicit != explicit:
                 continue
 

lib/spack/spack/test/bindist.py

@@ -2,6 +2,7 @@
 # Spack Project Developers. See the top-level COPYRIGHT file for details.
 #
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
+import glob
 import os
 import sys
 import platform
@@ -435,6 +436,45 @@ def test_spec_needs_rebuild(monkeypatch, tmpdir):
     assert rebuild
 
 
+@pytest.mark.usefixtures(
+    'install_mockery_mutable_config', 'mock_packages', 'mock_fetch',
+)
+def test_generate_index_missing(monkeypatch, tmpdir, mutable_config):
+    """Ensure spack buildcache index only reports available packages"""
+
+    # Create a temp mirror directory for buildcache usage
+    mirror_dir = tmpdir.join('mirror_dir')
+    mirror_url = 'file://{0}'.format(mirror_dir.strpath)
+    spack.config.set('mirrors', {'test': mirror_url})
+
+    s = Spec('libdwarf').concretized()
+
+    # Install a package
+    install_cmd('--no-cache', s.name)
+
+    # Create a buildcache and update index
+    buildcache_cmd('create', '-uad', mirror_dir.strpath, s.name)
+    buildcache_cmd('update-index', '-d', mirror_dir.strpath)
+
+    # Check package and dependency in buildcache
+    cache_list = buildcache_cmd('list', '--allarch')
+    assert 'libdwarf' in cache_list
+    assert 'libelf' in cache_list
+
+    # Remove dependency from cache
+    libelf_files = glob.glob(
+        os.path.join(mirror_dir.join('build_cache').strpath, '*libelf*'))
+    os.remove(*libelf_files)
+
+    # Update index
+    buildcache_cmd('update-index', '-d', mirror_dir.strpath)
+
+    # Check dependency not in buildcache
+    cache_list = buildcache_cmd('list', '--allarch')
+    assert 'libdwarf' in cache_list
+    assert 'libelf' not in cache_list
+
+
 def test_generate_indices_key_error(monkeypatch, capfd):
 
     def mock_list_url(url, recursive=False):