package_hash: add code to generate a hash for a package file

This will be included in the full hash of packages.

lib/spack/spack/test/package_hash.py

@@ -0,0 +1,84 @@
+##############################################################################
+# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://software.llnl.gov/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+from spack.util.package_hash import package_hash, package_content
+from spack.spec import Spec
+
+
+def test_hash(tmpdir, builtin_mock, config):
+    package_hash("hash-test1@1.2")
+
+
+def test_different_variants(tmpdir, builtin_mock, config):
+    spec1 = Spec("hash-test1@1.2 +variantx")
+    spec2 = Spec("hash-test1@1.2 +varianty")
+    assert package_hash(spec1) == package_hash(spec2)
+
+
+def test_all_same_but_name(tmpdir, builtin_mock, config):
+    spec1 = Spec("hash-test1@1.2")
+    spec2 = Spec("hash-test2@1.2")
+    compare_sans_name(True, spec1, spec2)
+
+    spec1 = Spec("hash-test1@1.2 +varianty")
+    spec2 = Spec("hash-test2@1.2 +varianty")
+    compare_sans_name(True, spec1, spec2)
+
+
+def test_all_same_but_archive_hash(tmpdir, builtin_mock, config):
+    """
+    Archive hash is not intended to be reflected in Package hash.
+    """
+    spec1 = Spec("hash-test1@1.3")
+    spec2 = Spec("hash-test2@1.3")
+    compare_sans_name(True, spec1, spec2)
+
+
+def test_all_same_but_patch_contents(tmpdir, builtin_mock, config):
+    spec1 = Spec("hash-test1@1.1")
+    spec2 = Spec("hash-test2@1.1")
+    compare_sans_name(True, spec1, spec2)
+
+
+def test_all_same_but_patches_to_apply(tmpdir, builtin_mock, config):
+    spec1 = Spec("hash-test1@1.4")
+    spec2 = Spec("hash-test2@1.4")
+    compare_sans_name(True, spec1, spec2)
+
+
+def test_all_same_but_install(tmpdir, builtin_mock, config):
+    spec1 = Spec("hash-test1@1.5")
+    spec2 = Spec("hash-test2@1.5")
+    compare_sans_name(False, spec1, spec2)
+
+
+def compare_sans_name(eq, spec1, spec2):
+    content1 = package_content(spec1)
+    content1 = content1.replace(spec1.package.__class__.__name__, '')
+    content2 = package_content(spec2)
+    content2 = content2.replace(spec2.package.__class__.__name__, '')
+    if eq:
+        assert content1 == content2
+    else:
+        assert content1 != content2

lib/spack/spack/util/package_hash.py

@@ -0,0 +1,151 @@
+##############################################################################
+# Copyright (c) 2016, Lawrence Livermore National Security, LLC.
+# Produced at the Lawrence Livermore National Laboratory.
+#
+# This file is part of Spack.
+# Written by Todd Gamblin, tgamblin@llnl.gov, All rights reserved.
+# LLNL-CODE-647188
+#
+# For details, see https://software.llnl.gov/spack
+# Please also see the LICENSE file for our notice and the LGPL.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License (as published by
+# the Free Software Foundation) version 2.1 dated February 1999.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the IMPLIED WARRANTY OF
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the terms and
+# conditions of the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software Foundation,
+# Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
+##############################################################################
+import spack
+from spack import directives
+from spack.error import SpackError
+from spack.spec import Spec
+from spack.util.naming import mod_to_class
+
+import ast
+import hashlib
+
+
+class RemoveDocstrings(ast.NodeTransformer):
+    """Transformer that removes docstrings from a Python AST."""
+    def remove_docstring(self, node):
+        if node.body:
+            if isinstance(node.body[0], ast.Expr) and \
+               isinstance(node.body[0].value, ast.Str):
+                node.body.pop(0)
+
+        self.generic_visit(node)
+        return node
+
+    def visit_FunctionDef(self, node):
+        return self.remove_docstring(node)
+
+    def visit_ClassDef(self, node):
+        return self.remove_docstring(node)
+
+    def visit_Module(self, node):
+        return self.remove_docstring(node)
+
+
+class RemoveDirectives(ast.NodeTransformer):
+    """Remove Spack directives from a package AST."""
+    def __init__(self, spec):
+        self.spec = spec
+
+    def is_directive(self, node):
+        return (isinstance(node, ast.Expr) and
+                node.value and isinstance(node.value, ast.Call) and
+                node.value.func.id in directives.__all__)
+
+    def is_spack_attr(self, node):
+        return (isinstance(node, ast.Assign) and
+                node.targets and isinstance(node.targets[0], ast.Name) and
+                node.targets[0].id in spack.Package.metadata_attrs)
+
+    def visit_ClassDef(self, node):
+        if node.name == mod_to_class(self.spec.name):
+            node.body = [
+                c for c in node.body
+                if (not self.is_directive(c) and not self.is_spack_attr(c))]
+        return node
+
+
+class TagMultiMethods(ast.NodeVisitor):
+    """Tag @when-decorated methods in a spec."""
+    def __init__(self, spec):
+        self.spec = spec
+        self.methods = {}
+
+    def visit_FunctionDef(self, node):
+        nodes = self.methods.setdefault(node.name, [])
+        if node.decorator_list:
+            dec = node.decorator_list[0]
+            if isinstance(dec, ast.Call) and dec.func.id == 'when':
+                cond = dec.args[0].s
+                nodes.append((node, self.spec.satisfies(cond, strict=True)))
+        else:
+            nodes.append((node, None))
+
+
+class ResolveMultiMethods(ast.NodeTransformer):
+    """Remove methods which do not exist if their @when is not satisfied."""
+    def __init__(self, methods):
+        self.methods = methods
+
+    def resolve(self, node):
+        if node.name not in self.methods:
+            raise PackageHashError(
+                "Future traversal visited new node: %s" % node.name)
+
+        result = None
+        for n, cond in self.methods[node.name]:
+            if cond:
+                return n
+            if cond is None:
+                result = n
+        return result
+
+    def visit_FunctionDef(self, node):
+        if self.resolve(node) is node:
+            node.decorator_list = []
+            return node
+        return None
+
+
+def package_content(spec):
+    return ast.dump(package_ast(spec))
+
+
+def package_hash(spec, content=None):
+    if content is None:
+        content = package_content(spec)
+    return hashlib.sha256(content.encode('utf-8')).digest().lower()
+
+
+def package_ast(spec):
+    spec = Spec(spec)
+
+    filename = spack.repo.filename_for_package_name(spec.name)
+    with open(filename) as f:
+        text = f.read()
+        root = ast.parse(text)
+
+    root = RemoveDocstrings().visit(root)
+
+    RemoveDirectives(spec).visit(root)
+
+    fmm = TagMultiMethods(spec)
+    fmm.visit(root)
+
+    root = ResolveMultiMethods(fmm.methods).visit(root)
+    return root
+
+
+class PackageHashError(SpackError):
+    """Raised for all errors encountered during package hashing."""

var/spack/repos/builtin.mock/packages/hash-test1/package.py

@@ -0,0 +1,34 @@
+from spack import *
+
+import os
+
+
+class HashTest1(Package):
+    """Used to test package hashing
+    """
+
+    homepage = "http://www.hashtest1.org"
+    url = "http://www.hashtest1.org/downloads/hashtest1-1.1.tar.bz2"
+
+    version('1.1', 'a' * 32)
+    version('1.2', 'b' * 32)
+    version('1.3', 'c' * 32)
+    version('1.4', 'd' * 32)
+
+    patch('patch1.patch', when="@1.1")
+    patch('patch2.patch', when="@1.4")
+
+    variant('variantx', default=False, description='Test variant X')
+    variant('varianty', default=False, description='Test variant Y')
+
+    def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
+        pass
+
+    @when('@:1.4')
+    def install(self, spec, prefix):
+        print("install 1")
+        os.listdir(os.getcwd())
+
+    @when('@1.5')
+    def install(self, spec, prefix):
+        os.listdir(os.getcwd())

var/spack/repos/builtin.mock/packages/hash-test1/patch1.patch

@@ -0,0 +1 @@
+the contents of patch 1 (not a valid diff, but sufficient for testing)

var/spack/repos/builtin.mock/packages/hash-test1/patch2.patch

@@ -0,0 +1 @@
+the contents of patch 2 (not a valid diff, but sufficient for testing)

var/spack/repos/builtin.mock/packages/hash-test2/package.py

@@ -0,0 +1,28 @@
+from spack import *
+
+import os
+
+
+class HashTest2(Package):
+    """Used to test package hashing
+    """
+
+    homepage = "http://www.hashtest2.org"
+    url = "http://www.hashtest1.org/downloads/hashtest2-1.1.tar.bz2"
+
+    version('1.1', 'a' * 32)
+    version('1.2', 'b' * 32)
+    version('1.3', 'c' * 31 + 'x')  # Source hash differs from hash-test1@1.3
+    version('1.4', 'd' * 32)
+
+    patch('patch1.patch', when="@1.1")
+
+    variant('variantx', default=False, description='Test variant X')
+    variant('varianty', default=False, description='Test variant Y')
+
+    def setup_dependent_environment(self, spack_env, run_env, dependent_spec):
+        pass
+
+    def install(self, spec, prefix):
+        print("install 1")
+        os.listdir(os.getcwd())

var/spack/repos/builtin.mock/packages/hash-test2/patch1.patch

@@ -0,0 +1,2 @@
+the different contents of patch 1 (not a valid diff, but sufficient for testing,
+and different from patch 1 of hash-test1)