WIP

lib/spack/llnl/util/lang.py

@@ -1111,3 +1111,32 @@ def __init__(self, callback):
 
     def __get__(self, instance, owner):
         return self.callback(owner)
+
+
+def dict_list_to_tuple(dlist):
+    if isinstance(dlist, (list, tuple)):
+        return tuple(dict_list_to_tuple(elt) for elt in dlist)
+    elif isinstance(dlist, dict):
+        return tuple((key, dict_list_to_tuple(val)) for key, val in dlist.items())
+    else:
+        return dlist
+
+
+class dict_list(list):
+    """A list with an interface like an ordereddict that can be turned into a tuple."""
+
+    def __setitem__(self, key, value):
+        self.append((key, value))
+
+    def update(self, dict_like):
+        if isinstance(dict_like, (list, tuple)):
+            iterable = dict_like
+        else:
+            iterable = dict_like.items()
+
+        for i in iterable:
+            self.append(i)
+
+    def items(self):
+        for i in self:
+            yield i

lib/spack/spack/spec.py

@@ -77,6 +77,7 @@
 expansion when it is the first character in an id typed on the command line.
 """
 import collections
+import hashlib
 import itertools
 import operator
 import os
@@ -521,15 +522,18 @@ def target_concrete(self):
         """True if the target is not a range or list."""
         return ":" not in str(self.target) and "," not in str(self.target)
 
-    def to_dict(self):
-        d = syaml.syaml_dict(
-            [
+    def to_dict(self, dict_type=syaml.syaml_dict):
+        d = dict_type(
+            (
                 ("platform", self.platform),
                 ("platform_os", self.os),
-                ("target", self.target.to_dict_or_value()),
-            ]
+                (
+                    "target",
+                    self.target.to_dict_or_value(dict_type=dict_type) if self.target else None,
+                ),
+            )
         )
-        return syaml.syaml_dict([("arch", d)])
+        return dict_type([("arch", d)])
 
     @staticmethod
     def from_dict(d):
@@ -643,11 +647,11 @@ def _cmp_iter(self):
         yield self.name
         yield self.versions
 
-    def to_dict(self):
-        d = syaml.syaml_dict([("name", self.name)])
+    def to_dict(self, dict_type=syaml.syaml_dict):
+        d = dict_type([("name", self.name)])
         d.update(self.versions.to_dict())
 
-        return syaml.syaml_dict([("compiler", d)])
+        return dict_type([("compiler", d)])
 
     @staticmethod
     def from_dict(d):
@@ -1853,7 +1857,13 @@ def process_hash_bit_prefix(self, bits):
         """Get the first <bits> bits of the DAG hash as an integer type."""
         return spack.util.hash.base32_prefix_bits(self.process_hash(), bits)
 
-    def to_node_dict(self, hash=ht.dag_hash):
+    def dict_tuple(self, hash=ht.dag_hash):
+        #        return lang.dict_list_to_tuple(self.to_node_dict())
+
+        dlist = self.to_node_dict(hash=hash, dict_type=lang.dict_list)
+        return lang.dict_list_to_tuple(dlist)
+
+    def to_node_dict(self, hash=ht.dag_hash, dict_type=syaml.syaml_dict):
         """Create a dictionary representing the state of this Spec.
 
         ``to_node_dict`` creates the content that is eventually hashed by
@@ -1905,30 +1915,30 @@ def to_node_dict(self, hash=ht.dag_hash):
         Arguments:
             hash (spack.hash_types.SpecHashDescriptor) type of hash to generate.
         """
-        d = syaml.syaml_dict()
+        d = dict_type()
 
         d["name"] = self.name
 
         if self.versions:
-            d.update(self.versions.to_dict())
+            d.update(self.versions.to_dict(dict_type=dict_type))
 
         if self.architecture:
-            d.update(self.architecture.to_dict())
+            d.update(self.architecture.to_dict(dict_type=dict_type))
 
         if self.compiler:
-            d.update(self.compiler.to_dict())
+            d.update(self.compiler.to_dict(dict_type=dict_type))
 
         if self.namespace:
             d["namespace"] = self.namespace
 
-        params = syaml.syaml_dict(sorted(v.yaml_entry() for _, v in self.variants.items()))
+        params = dict_type(sorted(v.yaml_entry() for _, v in self.variants.items()))
 
         params.update(sorted(self.compiler_flags.items()))
         if params:
             d["parameters"] = params
 
         if self.external:
-            d["external"] = syaml.syaml_dict(
+            d["external"] = dict_type(
                 [
                     ("path", self.external_path),
                     ("module", self.external_modules),
@@ -1968,12 +1978,12 @@ def to_node_dict(self, hash=ht.dag_hash):
                 for dspec in edges_for_name:
                     hash_tuple = (hash.name, dspec.spec._cached_hash(hash))
                     type_tuple = ("type", sorted(str(s) for s in dspec.deptypes))
-                    deps_list.append(syaml.syaml_dict([name_tuple, hash_tuple, type_tuple]))
+                    deps_list.append(dict_type([name_tuple, hash_tuple, type_tuple]))
             d["dependencies"] = deps_list
 
         # Name is included in case this is replacing a virtual.
         if self._build_spec:
-            d["build_spec"] = syaml.syaml_dict(
+            d["build_spec"] = dict_type(
                 [("name", self.build_spec.name), (hash.name, self.build_spec._cached_hash(hash))]
             )
         return d
@@ -4012,6 +4022,7 @@ def _cmp_node(self):
         yield self.compiler
         yield self.compiler_flags
         yield self.architecture
+        yield self._package_hash
 
     def eq_node(self, other):
         """Equality with another spec, not including dependencies."""
@@ -4783,6 +4794,15 @@ def clear_cached_hashes(self, ignore=()):
         self._dunder_hash = None
 
     def __hash__(self):
+        return hash(self.dict_tuple())
+
+        #        node_dict = self.to_node_dict(hash=ht.dag_hash)
+        #        json_text = sjson.dump(node_dict)
+        #        sha = hashlib.sha1(json_text.encode("utf-8"))
+        #        h = int.from_bytes(sha.digest()[:8], byteorder=sys.byteorder)
+        #        print(h)
+        #        return h
+
         # If the spec is concrete, we leverage the process hash and just use
         # a 64-bit prefix of it. The process hash has the advantage that it's
         # computed once per concrete spec, and it's saved -- so if we read

lib/spack/spack/target.py

@@ -91,7 +91,7 @@ def from_dict_or_value(dict_or_value):
         target_info = dict_or_value
         return Target(target_info["name"])
 
-    def to_dict_or_value(self):
+    def to_dict_or_value(self, dict_type=syaml.syaml_dict):
         """Returns a dict or a value representing the current target.
 
         String values are used to keep backward compatibility with generic
@@ -104,7 +104,7 @@ def to_dict_or_value(self):
         if self.microarchitecture.vendor == "generic":
             return str(self)
 
-        return syaml.syaml_dict(self.microarchitecture.to_dict(return_list_of_items=True))
+        return dict_type(self.microarchitecture.to_dict(return_list_of_items=True))
 
     def __repr__(self):
         cls_name = self.__class__.__name__

lib/spack/spack/variant.py

@@ -277,7 +277,7 @@ def yaml_entry(self):
         Returns:
             tuple: (name, value_representation)
         """
-        return self.name, list(self.value)
+        return (self.name, tuple(self.value))
 
     @property
     def value(self):

lib/spack/spack/version.py

@@ -1011,12 +1011,12 @@ def overlaps(self, other):
                 o += 1
         return False
 
-    def to_dict(self):
+    def to_dict(self, dict_type=syaml_dict):
         """Generate human-readable dict for YAML."""
         if self.concrete:
-            return syaml_dict([("version", str(self[0]))])
+            return dict_type([("version", str(self[0]))])
         else:
-            return syaml_dict([("versions", [str(v) for v in self])])
+            return dict_type([("versions", [str(v) for v in self])])
 
     @staticmethod
     def from_dict(dictionary):

test.py

@@ -0,0 +1,296 @@
+#!/usr/bin/env spack-python
+
+import ast
+import contextlib
+
+from typing import Dict, List
+
+import spack.directives
+import spack.repo
+import spack.util.package_hash as ph
+
+
+def is_directive(node):
+    return (
+        hasattr(node, "value")
+        and node.value
+        and isinstance(node.value, ast.Call)
+        and isinstance(node.value.func, ast.Name)
+        and node.value.func.id in spack.directives.directive_names
+    )
+
+
+class NameDescriptor(object):
+    """Name in a scope, with global/nonlocal and const information."""
+
+    def __init__(self, name, const, isglobal, isnonlocal):
+        # type: (str, bool, bool, bool) -> None
+        self.name = name
+        self.const = const
+        self.isglobal = isglobal
+        self.isnonlocal = isnonlocal
+
+
+class ScopeStack(object):
+    """Simple implementation of a stack of lexical scopes.
+
+    We use this for very simple constant tracking in packages.
+
+    """
+
+    def __init__(self):
+        self.scopes = []  # type: List[Dict[str, NameDescriptor]]
+
+    def push(self, name):
+        """Add a scope with a name for debugging."""
+        self.scopes.append((name, {}))
+
+    def pop(self, name):
+        """Remove the top scope."""
+        key, scope = self.scopes.pop()
+        assert name == key
+        return scope
+
+    @contextlib.contextmanager
+    def scope(self, name):
+        try:
+            self.push(name)
+            yield self.top
+        finally:
+            self.pop(name)
+
+    @property
+    def top(self):
+        """Get the top scope on the stack"""
+        assert self.scopes
+        _, scope = self.scopes[-1]
+        return scope
+
+    def define(self, name, const=None, isglobal=None, isnonlocal=None):
+        # type: NameDescriptor -> None
+        self.top[name] = NameDescriptor(name, const, isglobal, isnonlocal)
+
+    def assign(self, name, const=None, isglobal=None, isnonlocal=None):
+
+        self.top.add(name)
+
+    def scope_for(self, name):
+        # type: str -> Optional[Dict[str, NameDescriptor]]
+        for _, scope in reversed(self.scopes):
+            if name in scope:
+                return scope
+        else:
+            return None
+
+    def get(self, name):
+        # type: str -> Optional[NameDescriptor]
+        scope = self.scope_for(name)
+        return scope[name] if scope else None
+
+    def delete(self, name):
+        """Delete name from scope it lives in."""
+        scope = self.scope_for(name)
+        if not scope:
+            raise KeyError("No name '%s' in any scope" % name)
+
+        del scope[name]
+
+
+def constexpr(node):
+    if isinstance(node, ast.Constant):
+        return True
+
+    elif isinstance(node, (list, tuple)):
+        return all(constexpr(arg) for arg in node)
+
+    elif isinstance(node, (ast.Tuple, ast.List)):
+        return all(constexpr(arg) for arg in node.elts)
+
+    elif isinstance(node, ast.BinOp):
+        return constexpr(node.left) and constexpr(node.right)
+
+    elif isinstance(node, ast.Compare):
+        return constexpr(node.left) and constexpr(node.right)
+
+    #    elif isinstance(node, ast.Call):
+    #        # allow some function calls in packages to be constexpr
+    #        if isinstance(node.func, ast.Name):
+    #            name = node.func.id
+    #        elif isinstance(node.func, ast.Attribute):
+    #            name =
+    #            # common directives
+    #            # TODO: we should really check that they're actually Spack's functions
+    #            # and not some overridden name
+    #            return node.func.id in (
+    #                "any_combination_of",
+    #                "auto_or_any_combination_of",
+    #                "bool",
+    #                "conditional",
+    #                "disjoint_sets",
+    #                "join_path",
+    #                "str",
+    #                "tuple",
+    #                "patch",
+    #            )
+
+    else:
+        return False
+
+
+class ConstTracker(ast.NodeVisitor):
+    """AST traversal that tracks the const-ness of variables defined in scopes."""
+
+    def __init__(self, name=None):
+        self.name = name or "<module>"
+        self.scopes = ScopeStack()
+
+    # These constructs create/destroy scopes and may define names
+
+    def visit_Module(self, node):
+        with self.scopes.scope("module:%s" % self.name):
+            self.generic_visit(node)
+
+    def visit_ClassDef(self, node):
+        self.scopes.define(node.name, const=False)
+
+        with self.scopes.scope("class:%s" % node.name):
+            self.generic_visit(node)
+
+    def visit_AsyncClassDef(self, node):
+        self.visit_ClassDef(node)
+
+    def visit_FunctionDef(self, node):
+        with self.scopes.scope("func:%s" % node.name):
+            self.generic_visit(node)
+
+    def visit_AsyncFunctionDef(self, node):
+        self.visit_FunctionDef(node)
+
+    def visit_GeneratorExp(self, node, leak=False):
+        for comp in node.generators:
+            if not leak:
+                self.scopes.push("<generatorexp>")
+
+            self.generic_visit(comp.iter)
+
+            # TODO: if we need this to handle different targets separately , e.g. x and y in:
+            #
+            #    [for x,y in [(a, 1), (b, 2), (c, 3)]
+            #
+            # Then this needs to understand unpacking. Currently it's either all const or not,
+            # So both x and y would be non-const here.
+
+            const = constexpr(comp.iter)
+            if isinstance(comp.target, (ast.List, ast.Tuple)):
+                for name in comp.target.elts:
+                    self.scopes.top.define(name.id, const=const)
+
+        # visit element expressions once the generator clauses are done
+        self.generic_visit(node.elt)
+
+        # in Python 2, Generator expressions do not leak but comprehensions do.
+        # in Python 3, none of these leak.
+        if not leak:
+            for comp in node.generators:
+                self.scopes.pop("<generatorexp>")
+
+        self.generic_visit(node)
+
+    def visit_ListComp(self, node):
+        # use leak-True b/c we support Python 2
+        self.visit_GeneratorExp(node, leak=True)
+
+    def visit_SetComp(self, node):
+        # use leak-True b/c we support Python 2
+        self.visit_GeneratorExp(node, leak=True)
+
+    def visit_DictComp(self, node):
+        # use leak-True b/c we support Python 2
+        self.visit_GeneratorExp(node, leak=True)
+
+    def visit_Lambda(self, node):
+        self.generic_visit(node)
+
+    # These constructs define names in scopes
+
+    def visit_Assign(self, node):
+        self.generic_visit(node)
+
+    def visit_AugAssign(self, node):
+        """Operators like +="""
+        self.generic_visit(node)
+
+    def visit_AnnAssign(self, node):
+        self.generic_visit(node)
+
+    def visit_NamedExpr(self, node):
+        """Walrus operator :="""
+        self.generic_visit(node)
+
+    def visit_withitem(self, node):
+        self.generic_visit(node)
+
+    def visit_For(self, node):
+        self.generic_visit(node)
+
+    def visit_AsyncFor(self, node):
+        self.generic_visit(node)
+
+    def visit_Import(self, node):
+        self.generic_visit(node)
+
+    def visit_ImportFrom(self, node):
+        self.generic_visit(node)
+
+    def visit_Delete(self, node):
+        self.generic_visit(node)
+
+    # these change the scope of names
+    def visit_Global(self, node):
+        self.generic_visit(node)
+
+    def visit_Nonlocal(self, node):
+        self.generic_visit(node)
+
+
+class ConstDirectives(ConstTracker):
+    def __init__(self, name):
+        super(ConstDirectives, self).__init__(name)
+
+        self.const = True
+        self.name = name
+        self.in_classdef = False
+
+        self.issues = []
+
+    def visit_Expr(self, node):
+        # Directives are represented in the AST as named function call expressions (as
+        # opposed to function calls through a variable callback).
+        if is_directive(node):
+            for arg in node.value.args:
+                if not constexpr(arg):
+                    #                    self.issues.append("ARG:   %s" % ast.dump(arg))
+                    self.issues.append("ISSUE: " + ast.dump(arg))
+                    self.const = False
+
+            for k in node.value.keywords:
+                if not constexpr(k.value):
+                    # self.issues.append("KWARG: %s=%s" % (k.arg, ast.dump(k.value)))
+                    self.issues.append("ISSUE: " + ast.dump(k.value))
+                    self.const = False
+
+
+for pkg_name in spack.repo.all_package_names():
+    # for pkg_name in ["boost"]:
+    filename = spack.repo.path.filename_for_package_name(pkg_name)
+    with open(filename) as f:
+        source = f.read()
+
+    root = ast.parse(source)
+    const = ConstDirectives(pkg_name)
+
+    const.visit(root)
+    if not const.const:
+        print("PACKAGE:", pkg_name)
+        for issue in const.issues:
+            print("    ", issue)