Optimize concurrent misc_cache provider index rebuild (#32874)

When concurrent misc_cache provider index rebuilds happen, try to
rebuild it only once, so we don't exceed misc_cache lock timeout.

For example, when using `spack env depfile`, with no previous
misc_cache, running `make -f depfile -j8` could run at most 8 concurrent
`spack install` locking on misc_cache to rebuild the provider index. If
one rebuild takes 30s, before this fix, the "worst" lock could wait up
to 30s * 7, easily exceeding misc_cache lock timeout. Now, the "worst"
lock would take 30s * 1 + ~1s * 6.
This commit is contained in:
Jordan Galby 2022-10-05 14:01:59 +02:00 committed by GitHub
parent 53cea629b7
commit 6c12630e95
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

View File

@ -559,6 +559,9 @@ def _create_new_cache(self): # type: () -> Dict[str, os.stat_result]
def last_mtime(self):
return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values())
def modified_since(self, since):
return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since]
def __getitem__(self, item):
return self._packages_to_stats[item]
@ -739,8 +742,7 @@ def _build_index(self, name, indexer):
# Compute which packages needs to be updated in the cache
misc_cache = spack.caches.misc_cache
index_mtime = misc_cache.mtime(cache_filename)
needs_update = [x for x, sinfo in self.checker.items() if sinfo.st_mtime > index_mtime]
needs_update = self.checker.modified_since(index_mtime)
index_existed = misc_cache.init_entry(cache_filename)
if index_existed and not needs_update:
@ -753,6 +755,12 @@ def _build_index(self, name, indexer):
with misc_cache.write_transaction(cache_filename) as (old, new):
indexer.read(old) if old else indexer.create()
# Compute which packages needs to be updated **again** in case someone updated them
# while we waited for the lock
new_index_mtime = misc_cache.mtime(cache_filename)
if new_index_mtime != index_mtime:
needs_update = self.checker.modified_since(new_index_mtime)
for pkg_name in needs_update:
namespaced_name = "%s.%s" % (self.namespace, pkg_name)
indexer.update(namespaced_name)