zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Clean up stable hashing so that specs don't contain !!python/object/apply

Browse Source 
- only output basic lists, dicts, etc.
- spec and database now parse and write specs as ordered data.
This commit is contained in:
Todd Gamblin 2016-09-02 02:54:03 -07:00
parent 674434b0fc
commit c8b4f978e1
4 changed files with 51 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/spack/spack/architecture.py
View File

@ -87,6 +87,7 @@
from spack.util.naming import mod_to_class
from spack.util.environment import get_path
from spack.util.multiproc import parmap
from spack.util.spack_yaml import syaml_dict
import spack.error as serr
@ -407,12 +408,13 @@ def _cmp_key(self):
return (platform, platform_os, target)
def to_dict(self):
d = {}
d['platform'] = str(self.platform) if self.platform else None
d['platform_os'] = str(self.platform_os) if self.platform_os else None
d['target'] = str(self.target) if self.target else None
return d
return syaml_dict((
('platform',
str(self.platform) if self.platform else None),
('platform_os',
str(self.platform_os) if self.platform_os else None),
('target',
str(self.target) if self.target else None)))
def _target_from_dict(target_name, plat=None):

10
lib/spack/spack/database.py
View File

@ -42,7 +42,6 @@
import os
import socket
import yaml
from yaml.error import MarkedYAMLError, YAMLError
import llnl.util.tty as tty
@ -54,7 +53,7 @@
from spack.spec import Spec
from spack.error import SpackError
from spack.repository import UnknownPackageError
import spack.util.spack_yaml as syaml
# DB goes in this directory underneath the root
_db_dirname = '.spack-db'
@ -197,7 +196,8 @@ def _write_to_yaml(self, stream):
}
try:
return yaml.dump(database, stream=stream, default_flow_style=False)
return syaml.dump(
database, stream=stream, default_flow_style=False)
except YAMLError as e:
raise SpackYAMLError("error writing YAML database:", str(e))
@ -247,9 +247,9 @@ def _read_from_yaml(self, stream):
try:
if isinstance(stream, basestring):
with open(stream, 'r') as f:
yfile = yaml.load(f)
yfile = syaml.load(f)
else:
yfile = yaml.load(stream)
yfile = syaml.load(stream)
except MarkedYAMLError as e:
raise SpackYAMLError("error parsing YAML database:", str(e))

68
lib/spack/spack/spec.py
View File

@ -102,7 +102,6 @@
from StringIO import StringIO
from operator import attrgetter
import yaml
from yaml.error import MarkedYAMLError
import llnl.util.tty as tty
@ -119,6 +118,7 @@
from spack.util.naming import mod_to_class
from spack.util.prefix import Prefix
from spack.util.string import *
import spack.util.spack_yaml as syaml
from spack.util.spack_yaml import syaml_dict
from spack.version import *
from spack.provider_index import ProviderIndex
@ -267,9 +267,10 @@ def _cmp_key(self):
return (self.name, self.versions)
def to_dict(self):
d = {'name': self.name}
d = syaml_dict([('name', self.name)])
d.update(self.versions.to_dict())
return {'compiler': d}
return syaml_dict([('compiler', d)])
@staticmethod
def from_dict(d):
@ -903,7 +904,7 @@ def dag_hash(self, length=None):
return self._hash[:length]
else:
# XXX(deptype): ignore 'build' dependencies here
yaml_text = yaml.dump(
yaml_text = syaml.dump(
self.to_node_dict(), default_flow_style=True, width=sys.maxint)
sha = hashlib.sha1(yaml_text)
b32_hash = base64.b32encode(sha.digest()).lower()[:length]
@ -912,45 +913,37 @@ def dag_hash(self, length=None):
return b32_hash
def to_node_dict(self):
ordered_dict = lambda d: syaml_dict(sorted(d.items()))
d = syaml_dict()
params = syaml_dict(sorted(
(name, v.value) for name, v in self.variants.items()))
params.update(ordered_dict(self.compiler_flags))
if self.versions:
d.update(self.versions.to_dict())
if params:
d['parameters'] = params
deps = self.dependencies_dict(deptype=('link', 'run'))
if deps:
d['dependencies'] = syaml_dict(sorted((
(
name,
ordered_dict({
'hash': dspec.spec.dag_hash(),
'type': sorted([str(s) for s in dspec.deptypes])
})
)
for name, dspec in deps.items()
)))
if self.compiler:
d.update(self.compiler.to_dict())
if self.namespace:
d['namespace'] = self.namespace
params = syaml_dict(sorted(
(name, v.value) for name, v in self.variants.items()))
params.update(sorted(self.compiler_flags.items()))
if params:
d['parameters'] = params
if self.architecture:
# TODO: Fix the target.to_dict to account for the tuple
# Want it to be a dict of dicts
d['arch'] = ordered_dict(self.architecture.to_dict())
d['arch'] = self.architecture.to_dict()
if self.compiler:
d['compiler'] = syaml_dict(self.compiler.to_dict()['compiler'])
deps = self.dependencies_dict(deptype=('link', 'run'))
if deps:
d['dependencies'] = syaml_dict([
(name,
syaml_dict([
('hash', dspec.spec.dag_hash()),
('type', sorted(str(s) for s in dspec.deptypes))])
) for name, dspec in sorted(deps.items())
])
if self.versions:
d.update(ordered_dict(self.versions.to_dict()))
return syaml_dict({self.name: d})
return syaml_dict([(self.name, d)])
def to_yaml(self, stream=None):
node_list = []
@ -958,8 +951,9 @@ def to_yaml(self, stream=None):
node = s.to_node_dict()
node[s.name]['hash'] = s.dag_hash()
node_list.append(node)
return yaml.dump({'spec': node_list},
stream=stream, default_flow_style=False)
return syaml.dump(
syaml_dict([('spec', node_list)]),
stream=stream, default_flow_style=False)
@staticmethod
def from_node_dict(node):
@ -1033,7 +1027,7 @@ def from_yaml(stream):
"""
try:
yfile = yaml.load(stream)
yfile = syaml.load(stream)
except MarkedYAMLError as e:
raise SpackYAMLError("error parsing YAML spec:", str(e))
@ -1952,7 +1946,7 @@ def _dup(self, other, deps=True, cleardeps=True):
# These fields are all cached results of expensive operations.
# If we preserved the original structure, we can copy them
# safely. If not, they need to be recomputed.
if deps == True or deps == alldeps:
if deps is True or deps == alldeps:
self._hash = other._hash
self._cmp_key_cache = other._cmp_key_cache
self._normal = other._normal

9
lib/spack/spack/version.py
View File

@ -49,6 +49,7 @@
from functools import wraps
from functools_backport import total_ordering
from spack.util.spack_yaml import syaml_dict
__all__ = ['Version', 'VersionRange', 'VersionList', 'ver']
@ -593,9 +594,13 @@ def overlaps(self, other):
def to_dict(self):
"""Generate human-readable dict for YAML."""
if self.concrete:
return {'version': str(self[0])}
return syaml_dict([
('version', str(self[0]))
])
else:
return {'versions': [str(v) for v in self]}
return syaml_dict([
('versions', [str(v) for v in self])
])
@staticmethod
def from_dict(dictionary):