binary_distribution: Initialize _cached_specs at the module level and only search the mirrors in get_spec if spec is not in _cached_specs. (#14714)

* Initialize _cached_specs at the file level and check for spec in it before searching mirrors in try_download_spec.

* Make _cached_specs a set to avoid duplicates

* Fix packaging test

* Ignore build_cache in stage when spec.yaml files are downloaded.

lib/spack/spack/binary_distribution.py

@@ -661,7 +661,7 @@ def extract_tarball(spec, filename, allow_root=False, unsigned=False,
 
 
 # Internal cache for downloaded specs
-_cached_specs = None
+_cached_specs = set()
 
 
 def try_download_specs(urls=None, force=False):
@@ -669,7 +669,6 @@ def try_download_specs(urls=None, force=False):
     Try to download the urls and cache them
     '''
     global _cached_specs
-    _cached_specs = []
     if urls is None:
         return {}
     for link in urls:
@@ -687,7 +686,7 @@ def try_download_specs(urls=None, force=False):
                 # we need to mark this spec concrete on read-in.
                 spec = Spec.from_yaml(f)
                 spec._mark_concrete()
-                _cached_specs.append(spec)
+                _cached_specs.add(spec)
 
     return _cached_specs
 
@@ -701,14 +700,14 @@ def get_spec(spec=None, force=False):
     if spec is None:
         return {}
     specfile_name = tarball_name(spec, '.spec.yaml')
-    if _cached_specs:
-        tty.debug("Using previously-retrieved specs")
-        return _cached_specs
 
     if not spack.mirror.MirrorCollection():
         tty.debug("No Spack mirrors are currently configured")
         return {}
 
+    if spec in _cached_specs:
+        return _cached_specs
+
     for mirror in spack.mirror.MirrorCollection().values():
         fetch_url_build_cache = url_util.join(
             mirror.fetch_url, _build_cache_relative_path)
@@ -732,7 +731,6 @@ def get_specs(force=False, use_arch=False, names=None):
     """
     Get spec.yaml's for build caches available on mirror
     """
-    global _cached_specs
     arch = architecture.Arch(architecture.platform(),
                              'default_os', 'default_target')
     arch_pattern = ('([^-]*-[^-]*-[^-]*)')
@@ -747,10 +745,6 @@ def get_specs(force=False, use_arch=False, names=None):
                                                     names_pattern)
     name_re = re.compile(regex_pattern)
 
-    if _cached_specs:
-        tty.debug("Using previously-retrieved specs")
-        return _cached_specs
-
     if not spack.mirror.MirrorCollection():
         tty.debug("No Spack mirrors are currently configured")
         return {}

lib/spack/spack/test/conftest.py

@@ -170,7 +170,7 @@ def ignore_stage_files():
     Used to track which leftover files in the stage have been seen.
     """
     # to start with, ignore the .lock file at the stage root.
-    return set(['.lock', spack.stage._source_path_subdir])
+    return set(['.lock', spack.stage._source_path_subdir, 'build_cache'])
 
 
 def remove_whatever_it_is(path):

lib/spack/spack/test/packaging.py

@@ -214,7 +214,7 @@ def test_buildcache(mock_archive, tmpdir):
     stage.destroy()
 
     # Remove cached binary specs since we deleted the mirror
-    bindist._cached_specs = None
+    bindist._cached_specs = set()
 
 
 def test_relocate_text(tmpdir):