zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Specs now cache result of "fast" in-memory hash.

Browse Source 
- Hash causes major slowdown for reading/setting up large DBs

- New version caches hash for concrete specs, which includes all specs in
  the install DB
This commit is contained in:
Todd Gamblin 2016-08-16 11:29:36 -07:00
parent c46a15b574
commit a8aad95d41
1 changed files with 22 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
lib/spack/spack/spec.py
View File

@ -504,6 +504,7 @@ def __init__(self, spec_like, *dep_like, **kwargs):
self.variants.spec = self self.variants.spec = self
self.namespace = other.namespace self.namespace = other.namespace
self._hash = other._hash self._hash = other._hash
self._cmp_key_cache = other._cmp_key_cache
# Specs are by default not assumed to be normal, but in some # Specs are by default not assumed to be normal, but in some
# cases we've read them from a file want to assume normal. # cases we've read them from a file want to assume normal.
@ -858,9 +859,10 @@ def return_val(res):
# Edge traversal yields but skips children of visited nodes # Edge traversal yields but skips children of visited nodes
if not (key in visited and cover == 'edges'): if not (key in visited and cover == 'edges'):
# This code determines direction and yields the children/parents # This code determines direction and yields the children/parents
successors = deps successors = deps
if direction == 'parents': if direction == 'parents':
successors = self.dependents_dict() successors = self.dependents_dict() # TODO: deptype?
visited.add(key) visited.add(key)
for name in sorted(successors): for name in sorted(successors):
@ -1278,15 +1280,15 @@ def concretize(self):
# Mark everything in the spec as concrete, as well. # Mark everything in the spec as concrete, as well.
self._mark_concrete() self._mark_concrete()
def _mark_concrete(self): def _mark_concrete(self, value=True):
"""Mark this spec and its dependencies as concrete. """Mark this spec and its dependencies as concrete.
Only for internal use -- client code should use "concretize" Only for internal use -- client code should use "concretize"
unless there is a need to force a spec to be concrete. unless there is a need to force a spec to be concrete.
""" """
for s in self.traverse(deptype_query=alldeps): for s in self.traverse(deptype_query=alldeps):
s._normal = True s._normal = value
s._concrete = True s._concrete = value
def concretized(self): def concretized(self):
"""This is a non-destructive version of concretize(). First clones, """This is a non-destructive version of concretize(). First clones,
@ -1533,6 +1535,10 @@ def normalize(self, force=False):
if self._normal and not force: if self._normal and not force:
return False return False
# avoid any assumptions about concreteness when forced
if force:
self._mark_concrete(False)
# Ensure first that all packages & compilers in the DAG exist. # Ensure first that all packages & compilers in the DAG exist.
self.validate_names() self.validate_names()
# Get all the dependencies into one DependencyMap # Get all the dependencies into one DependencyMap
@ -2059,10 +2065,17 @@ def _cmp_key(self):
1. A tuple describing this node in the DAG. 1. A tuple describing this node in the DAG.
2. The hash of each of this node's dependencies' cmp_keys. 2. The hash of each of this node's dependencies' cmp_keys.
""" """
dep_dict = self.dependencies_dict(deptype=('link', 'run')) if self._cmp_key_cache:
return self._cmp_node() + ( return self._cmp_key_cache
tuple(hash(dep_dict[name])
for name in sorted(dep_dict)),) dep_tuple = tuple(
(d.spec.name, hash(d.spec), tuple(sorted(d.deptypes)))
for name, d in sorted(self._dependencies.items()))
key = (self._cmp_node(), dep_tuple)
if self._concrete:
self._cmp_key_cache = key
return key
def colorized(self): def colorized(self):
return colorize_spec(self) return colorize_spec(self)
@ -2457,6 +2470,7 @@ def spec(self, name, check_valid_token=False):
spec._dependencies = DependencyMap() spec._dependencies = DependencyMap()
spec.namespace = spec_namespace spec.namespace = spec_namespace
spec._hash = None spec._hash = None
spec._cmp_key_cache = None
spec._normal = False spec._normal = False
spec._concrete = False spec._concrete = False