Clean up stable hashing so that specs don't contain !!python/object/apply

- only output basic lists, dicts, etc.
- spec and database now parse and write specs as ordered data.

lib/spack/spack/architecture.py

@@ -87,6 +87,7 @@
 from spack.util.naming import mod_to_class
 from spack.util.environment import get_path
 from spack.util.multiproc import parmap
+from spack.util.spack_yaml import syaml_dict
 import spack.error as serr
 
 
@@ -407,12 +408,13 @@ def _cmp_key(self):
         return (platform, platform_os, target)
 
     def to_dict(self):
-        d = {}
+        return syaml_dict((
-        d['platform'] = str(self.platform) if self.platform else None
+            ('platform',
-        d['platform_os'] = str(self.platform_os) if self.platform_os else None
+             str(self.platform) if self.platform else None),
-        d['target'] = str(self.target) if self.target else None
+            ('platform_os',
-
+             str(self.platform_os) if self.platform_os else None),
-        return d
+            ('target',
+             str(self.target) if self.target else None)))
 
 
 def _target_from_dict(target_name, plat=None):

lib/spack/spack/database.py

@@ -42,7 +42,6 @@
 import os
 import socket
 
-import yaml
 from yaml.error import MarkedYAMLError, YAMLError
 
 import llnl.util.tty as tty
@@ -54,7 +53,7 @@
 from spack.spec import Spec
 from spack.error import SpackError
 from spack.repository import UnknownPackageError
-
+import spack.util.spack_yaml as syaml
 
 # DB goes in this directory underneath the root
 _db_dirname = '.spack-db'
@@ -197,7 +196,8 @@ def _write_to_yaml(self, stream):
         }
 
         try:
-            return yaml.dump(database, stream=stream, default_flow_style=False)
+            return syaml.dump(
+                database, stream=stream, default_flow_style=False)
         except YAMLError as e:
             raise SpackYAMLError("error writing YAML database:", str(e))
 
@@ -247,9 +247,9 @@ def _read_from_yaml(self, stream):
         try:
             if isinstance(stream, basestring):
                 with open(stream, 'r') as f:
-                    yfile = yaml.load(f)
+                    yfile = syaml.load(f)
             else:
-                yfile = yaml.load(stream)
+                yfile = syaml.load(stream)
 
         except MarkedYAMLError as e:
             raise SpackYAMLError("error parsing YAML database:", str(e))

lib/spack/spack/spec.py

@@ -102,7 +102,6 @@
 from StringIO import StringIO
 from operator import attrgetter
 
-import yaml
 from yaml.error import MarkedYAMLError
 
 import llnl.util.tty as tty
@@ -119,6 +118,7 @@
 from spack.util.naming import mod_to_class
 from spack.util.prefix import Prefix
 from spack.util.string import *
+import spack.util.spack_yaml as syaml
 from spack.util.spack_yaml import syaml_dict
 from spack.version import *
 from spack.provider_index import ProviderIndex
@@ -267,9 +267,10 @@ def _cmp_key(self):
         return (self.name, self.versions)
 
     def to_dict(self):
-        d = {'name': self.name}
+        d = syaml_dict([('name', self.name)])
         d.update(self.versions.to_dict())
-        return {'compiler': d}
+
+        return syaml_dict([('compiler', d)])
 
     @staticmethod
     def from_dict(d):
@@ -903,7 +904,7 @@ def dag_hash(self, length=None):
             return self._hash[:length]
         else:
             # XXX(deptype): ignore 'build' dependencies here
-            yaml_text = yaml.dump(
+            yaml_text = syaml.dump(
                 self.to_node_dict(), default_flow_style=True, width=sys.maxint)
             sha = hashlib.sha1(yaml_text)
             b32_hash = base64.b32encode(sha.digest()).lower()[:length]
@@ -912,45 +913,37 @@ def dag_hash(self, length=None):
             return b32_hash
 
     def to_node_dict(self):
-        ordered_dict = lambda d: syaml_dict(sorted(d.items()))
-
         d = syaml_dict()
 
-        params = syaml_dict(sorted(
+        if self.versions:
-            (name, v.value) for name, v in self.variants.items()))
+            d.update(self.versions.to_dict())
-        params.update(ordered_dict(self.compiler_flags))
 
-        if params:
+        if self.compiler:
-            d['parameters'] = params
+            d.update(self.compiler.to_dict())
-
-        deps = self.dependencies_dict(deptype=('link', 'run'))
-        if deps:
-            d['dependencies'] = syaml_dict(sorted((
-                (
-                    name,
-                    ordered_dict({
-                        'hash': dspec.spec.dag_hash(),
-                        'type': sorted([str(s) for s in dspec.deptypes])
-                    })
-                )
-                for name, dspec in deps.items()
-            )))
 
         if self.namespace:
             d['namespace'] = self.namespace
 
+        params = syaml_dict(sorted(
+            (name, v.value) for name, v in self.variants.items()))
+        params.update(sorted(self.compiler_flags.items()))
+        if params:
+            d['parameters'] = params
+
         if self.architecture:
-            # TODO: Fix the target.to_dict to account for the tuple
+            d['arch'] = self.architecture.to_dict()
-            # Want it to be a dict of dicts
-            d['arch'] = ordered_dict(self.architecture.to_dict())
 
-        if self.compiler:
+        deps = self.dependencies_dict(deptype=('link', 'run'))
-            d['compiler'] = syaml_dict(self.compiler.to_dict()['compiler'])
+        if deps:
+            d['dependencies'] = syaml_dict([
+                (name,
+                 syaml_dict([
+                     ('hash', dspec.spec.dag_hash()),
+                     ('type', sorted(str(s) for s in dspec.deptypes))])
+                 ) for name, dspec in sorted(deps.items())
+            ])
 
-        if self.versions:
+        return syaml_dict([(self.name, d)])
-            d.update(ordered_dict(self.versions.to_dict()))
-
-        return syaml_dict({self.name: d})
 
     def to_yaml(self, stream=None):
         node_list = []
@@ -958,8 +951,9 @@ def to_yaml(self, stream=None):
             node = s.to_node_dict()
             node[s.name]['hash'] = s.dag_hash()
             node_list.append(node)
-        return yaml.dump({'spec': node_list},
+        return syaml.dump(
-                         stream=stream, default_flow_style=False)
+            syaml_dict([('spec', node_list)]),
+            stream=stream, default_flow_style=False)
 
     @staticmethod
     def from_node_dict(node):
@@ -1033,7 +1027,7 @@ def from_yaml(stream):
 
         """
         try:
-            yfile = yaml.load(stream)
+            yfile = syaml.load(stream)
         except MarkedYAMLError as e:
             raise SpackYAMLError("error parsing YAML spec:", str(e))
 
@@ -1952,7 +1946,7 @@ def _dup(self, other, deps=True, cleardeps=True):
         # These fields are all cached results of expensive operations.
         # If we preserved the original structure, we can copy them
         # safely. If not, they need to be recomputed.
-        if deps == True or deps == alldeps:
+        if deps is True or deps == alldeps:
             self._hash = other._hash
             self._cmp_key_cache = other._cmp_key_cache
             self._normal = other._normal

lib/spack/spack/version.py

@@ -49,6 +49,7 @@
 from functools import wraps
 
 from functools_backport import total_ordering
+from spack.util.spack_yaml import syaml_dict
 
 __all__ = ['Version', 'VersionRange', 'VersionList', 'ver']
 
@@ -593,9 +594,13 @@ def overlaps(self, other):
     def to_dict(self):
         """Generate human-readable dict for YAML."""
         if self.concrete:
-            return {'version': str(self[0])}
+            return syaml_dict([
+                ('version', str(self[0]))
+            ])
         else:
-            return {'versions': [str(v) for v in self]}
+            return syaml_dict([
+                ('versions', [str(v) for v in self])
+            ])
 
     @staticmethod
     def from_dict(dictionary):