Optimize concurrent misc_cache provider index rebuild (#32874)
When concurrent misc_cache provider index rebuilds happen, try to rebuild it only once, so we don't exceed misc_cache lock timeout. For example, when using `spack env depfile`, with no previous misc_cache, running `make -f depfile -j8` could run at most 8 concurrent `spack install` locking on misc_cache to rebuild the provider index. If one rebuild takes 30s, before this fix, the "worst" lock could wait up to 30s * 7, easily exceeding misc_cache lock timeout. Now, the "worst" lock would take 30s * 1 + ~1s * 6.
This commit is contained in:
parent
53cea629b7
commit
6c12630e95
@ -559,6 +559,9 @@ def _create_new_cache(self): # type: () -> Dict[str, os.stat_result]
|
||||
def last_mtime(self):
|
||||
return max(sinfo.st_mtime for sinfo in self._packages_to_stats.values())
|
||||
|
||||
def modified_since(self, since):
|
||||
return [name for name, sinfo in self._packages_to_stats.items() if sinfo.st_mtime > since]
|
||||
|
||||
def __getitem__(self, item):
|
||||
return self._packages_to_stats[item]
|
||||
|
||||
@ -739,8 +742,7 @@ def _build_index(self, name, indexer):
|
||||
# Compute which packages needs to be updated in the cache
|
||||
misc_cache = spack.caches.misc_cache
|
||||
index_mtime = misc_cache.mtime(cache_filename)
|
||||
|
||||
needs_update = [x for x, sinfo in self.checker.items() if sinfo.st_mtime > index_mtime]
|
||||
needs_update = self.checker.modified_since(index_mtime)
|
||||
|
||||
index_existed = misc_cache.init_entry(cache_filename)
|
||||
if index_existed and not needs_update:
|
||||
@ -753,6 +755,12 @@ def _build_index(self, name, indexer):
|
||||
with misc_cache.write_transaction(cache_filename) as (old, new):
|
||||
indexer.read(old) if old else indexer.create()
|
||||
|
||||
# Compute which packages needs to be updated **again** in case someone updated them
|
||||
# while we waited for the lock
|
||||
new_index_mtime = misc_cache.mtime(cache_filename)
|
||||
if new_index_mtime != index_mtime:
|
||||
needs_update = self.checker.modified_since(new_index_mtime)
|
||||
|
||||
for pkg_name in needs_update:
|
||||
namespaced_name = "%s.%s" % (self.namespace, pkg_name)
|
||||
indexer.update(namespaced_name)
|
||||
|
Loading…
Reference in New Issue
Block a user