content_hash()
: make it work on abstract specs
Some test cases had to be modified in a kludgy way so that abstract specs made concrete would have versions on them. We shouldn't *need* to do this, as the only reason we care is because the content hash has to be able to get an archive for a version. This modifies the content hash so that it can be called on abstract specs, including only relevant content. This does NOT add a partial content hash to the DAG hash, as we do not really want that -- we don't need in-memory spec hashes to need to load package files. It just makes `Package.content_hash()` less prickly and tests easier to understand.
This commit is contained in:
parent
6db215dd89
commit
7ab46e26b5
@ -1673,44 +1673,62 @@ def all_patches(cls):
|
|||||||
return patches
|
return patches
|
||||||
|
|
||||||
def content_hash(self, content=None):
|
def content_hash(self, content=None):
|
||||||
"""Create a hash based on the sources and logic used to build the
|
"""Create a hash based on the artifacts and patches used to build this package.
|
||||||
package. This includes the contents of all applied patches and the
|
|
||||||
contents of applicable functions in the package subclass."""
|
|
||||||
if not self.spec.concrete:
|
|
||||||
err_msg = ("Cannot invoke content_hash on a package"
|
|
||||||
" if the associated spec is not concrete")
|
|
||||||
raise spack.error.SpackError(err_msg)
|
|
||||||
|
|
||||||
hash_content = list()
|
This includes:
|
||||||
try:
|
* source artifacts (tarballs, repositories) used to build;
|
||||||
source_id = fs.for_package_version(self, self.version).source_id()
|
* content hashes (``sha256``'s) of all patches applied by Spack; and
|
||||||
except (fs.ExtrapolationError, fs.InvalidArgsError):
|
* canonicalized contents the ``package.py`` recipe used to build.
|
||||||
# ExtrapolationError happens if the package has no fetchers defined.
|
|
||||||
# InvalidArgsError happens when there are version directives with args,
|
|
||||||
# but none of them identifies an actual fetcher.
|
|
||||||
source_id = None
|
|
||||||
|
|
||||||
if not source_id:
|
This hash is only included in Spack's DAG hash for concrete specs, but if it
|
||||||
# TODO? in cases where a digest or source_id isn't available,
|
happens to be called on a package with an abstract spec, only applicable (i.e.,
|
||||||
# should this attempt to download the source and set one? This
|
determinable) portions of the hash will be included.
|
||||||
# probably only happens for source repositories which are
|
|
||||||
# referenced by branch name rather than tag or commit ID.
|
|
||||||
env = spack.environment.active_environment()
|
|
||||||
from_local_sources = env and env.is_develop(self.spec)
|
|
||||||
if not self.spec.external and not from_local_sources:
|
|
||||||
message = 'Missing a source id for {s.name}@{s.version}'
|
|
||||||
tty.warn(message.format(s=self))
|
|
||||||
hash_content.append(''.encode('utf-8'))
|
|
||||||
else:
|
|
||||||
hash_content.append(source_id.encode('utf-8'))
|
|
||||||
|
|
||||||
hash_content.extend(':'.join((p.sha256, str(p.level))).encode('utf-8')
|
"""
|
||||||
for p in self.spec.patches)
|
# list of components to make up the hash
|
||||||
|
hash_content = []
|
||||||
|
|
||||||
|
# source artifacts/repositories
|
||||||
|
# TODO: resources
|
||||||
|
if self.spec.versions.concrete:
|
||||||
|
try:
|
||||||
|
source_id = fs.for_package_version(self, self.version).source_id()
|
||||||
|
except (fs.ExtrapolationError, fs.InvalidArgsError):
|
||||||
|
# ExtrapolationError happens if the package has no fetchers defined.
|
||||||
|
# InvalidArgsError happens when there are version directives with args,
|
||||||
|
# but none of them identifies an actual fetcher.
|
||||||
|
source_id = None
|
||||||
|
|
||||||
|
if not source_id:
|
||||||
|
# TODO? in cases where a digest or source_id isn't available,
|
||||||
|
# should this attempt to download the source and set one? This
|
||||||
|
# probably only happens for source repositories which are
|
||||||
|
# referenced by branch name rather than tag or commit ID.
|
||||||
|
env = spack.environment.active_environment()
|
||||||
|
from_local_sources = env and env.is_develop(self.spec)
|
||||||
|
if not self.spec.external and not from_local_sources:
|
||||||
|
message = 'Missing a source id for {s.name}@{s.version}'
|
||||||
|
tty.warn(message.format(s=self))
|
||||||
|
hash_content.append(''.encode('utf-8'))
|
||||||
|
else:
|
||||||
|
hash_content.append(source_id.encode('utf-8'))
|
||||||
|
|
||||||
|
# patch sha256's
|
||||||
|
if self.spec.concrete:
|
||||||
|
hash_content.extend(
|
||||||
|
':'.join((p.sha256, str(p.level))).encode('utf-8')
|
||||||
|
for p in self.spec.patches
|
||||||
|
)
|
||||||
|
|
||||||
|
# package.py contents
|
||||||
hash_content.append(package_hash(self.spec, source=content).encode('utf-8'))
|
hash_content.append(package_hash(self.spec, source=content).encode('utf-8'))
|
||||||
|
|
||||||
|
# put it all together and encode as base32
|
||||||
b32_hash = base64.b32encode(
|
b32_hash = base64.b32encode(
|
||||||
hashlib.sha256(bytes().join(
|
hashlib.sha256(
|
||||||
sorted(hash_content))).digest()).lower()
|
bytes().join(sorted(hash_content))
|
||||||
|
).digest()
|
||||||
|
).lower()
|
||||||
|
|
||||||
# convert from bytes if running python 3
|
# convert from bytes if running python 3
|
||||||
if sys.version_info[0] >= 3:
|
if sys.version_info[0] >= 3:
|
||||||
|
@ -215,9 +215,9 @@ def test_python_ignore_namespace_init_conflict(
|
|||||||
python_spec = spack.spec.Spec('python@2.7.12')
|
python_spec = spack.spec.Spec('python@2.7.12')
|
||||||
python_spec._concrete = True
|
python_spec._concrete = True
|
||||||
|
|
||||||
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
|
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
|
||||||
monkeypatch, py_namespace)
|
monkeypatch, py_namespace)
|
||||||
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
|
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
|
||||||
monkeypatch, py_namespace)
|
monkeypatch, py_namespace)
|
||||||
|
|
||||||
view_dir = str(tmpdir.join('view'))
|
view_dir = str(tmpdir.join('view'))
|
||||||
@ -250,9 +250,9 @@ def test_python_keep_namespace_init(
|
|||||||
python_spec = spack.spec.Spec('python@2.7.12')
|
python_spec = spack.spec.Spec('python@2.7.12')
|
||||||
python_spec._concrete = True
|
python_spec._concrete = True
|
||||||
|
|
||||||
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
|
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
|
||||||
monkeypatch, py_namespace)
|
monkeypatch, py_namespace)
|
||||||
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
|
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
|
||||||
monkeypatch, py_namespace)
|
monkeypatch, py_namespace)
|
||||||
|
|
||||||
view_dir = str(tmpdir.join('view'))
|
view_dir = str(tmpdir.join('view'))
|
||||||
@ -293,9 +293,9 @@ def test_python_namespace_conflict(tmpdir, namespace_extensions,
|
|||||||
python_spec = spack.spec.Spec('python@2.7.12')
|
python_spec = spack.spec.Spec('python@2.7.12')
|
||||||
python_spec._concrete = True
|
python_spec._concrete = True
|
||||||
|
|
||||||
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec,
|
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
|
||||||
monkeypatch, py_namespace)
|
monkeypatch, py_namespace)
|
||||||
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec,
|
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
|
||||||
monkeypatch, other_namespace)
|
monkeypatch, other_namespace)
|
||||||
|
|
||||||
view_dir = str(tmpdir.join('view'))
|
view_dir = str(tmpdir.join('view'))
|
||||||
|
@ -96,6 +96,26 @@ def test_content_hash_all_same_but_patch_contents(mock_packages, config):
|
|||||||
compare_hash_sans_name(False, spec1, spec2)
|
compare_hash_sans_name(False, spec1, spec2)
|
||||||
|
|
||||||
|
|
||||||
|
def test_content_hash_not_concretized(mock_packages, config):
|
||||||
|
"""Check that Package.content_hash() works on abstract specs."""
|
||||||
|
# these are different due to the package hash
|
||||||
|
spec1 = Spec("hash-test1@1.1")
|
||||||
|
spec2 = Spec("hash-test2@1.3")
|
||||||
|
compare_hash_sans_name(False, spec1, spec2)
|
||||||
|
|
||||||
|
# at v1.1 these are actually the same package when @when's are removed
|
||||||
|
# and the name isn't considered
|
||||||
|
spec1 = Spec("hash-test1@1.1")
|
||||||
|
spec2 = Spec("hash-test2@1.1")
|
||||||
|
compare_hash_sans_name(True, spec1, spec2)
|
||||||
|
|
||||||
|
# these end up being different b/c we can't eliminate much of the package.py
|
||||||
|
# without a version.
|
||||||
|
spec1 = Spec("hash-test1")
|
||||||
|
spec2 = Spec("hash-test2")
|
||||||
|
compare_hash_sans_name(False, spec1, spec2)
|
||||||
|
|
||||||
|
|
||||||
def test_content_hash_different_variants(mock_packages, config):
|
def test_content_hash_different_variants(mock_packages, config):
|
||||||
spec1 = Spec("hash-test1@1.2 +variantx").concretized()
|
spec1 = Spec("hash-test1@1.2 +variantx").concretized()
|
||||||
spec2 = Spec("hash-test2@1.2 ~variantx").concretized()
|
spec2 = Spec("hash-test2@1.2 ~variantx").concretized()
|
||||||
|
@ -143,7 +143,7 @@ def test_check_prefix_manifest(tmpdir):
|
|||||||
prefix_path = tmpdir.join('prefix')
|
prefix_path = tmpdir.join('prefix')
|
||||||
prefix = str(prefix_path)
|
prefix = str(prefix_path)
|
||||||
|
|
||||||
spec = spack.spec.Spec('libelf@0.8.13')
|
spec = spack.spec.Spec('libelf')
|
||||||
spec._mark_concrete()
|
spec._mark_concrete()
|
||||||
spec.prefix = prefix
|
spec.prefix = prefix
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user