content_hash(): make it work on abstract specs

Some test cases had to be modified in a kludgy way so that abstract specs made
concrete would have versions on them. We shouldn't *need* to do this, as the
only reason we care is because the content hash has to be able to get an archive
for a version.

This modifies the content hash so that it can be called on abstract specs,
including only relevant content.

This does NOT add a partial content hash to the DAG hash, as we do not really
want that -- we don't need in-memory spec hashes to need to load package files.
It just makes `Package.content_hash()` less prickly and tests easier to
understand.
This commit is contained in:
Todd Gamblin 2022-04-23 18:09:08 -07:00
parent 6db215dd89
commit 7ab46e26b5
4 changed files with 77 additions and 39 deletions

View File

@ -1673,44 +1673,62 @@ def all_patches(cls):
return patches return patches
def content_hash(self, content=None): def content_hash(self, content=None):
"""Create a hash based on the sources and logic used to build the """Create a hash based on the artifacts and patches used to build this package.
package. This includes the contents of all applied patches and the
contents of applicable functions in the package subclass."""
if not self.spec.concrete:
err_msg = ("Cannot invoke content_hash on a package"
" if the associated spec is not concrete")
raise spack.error.SpackError(err_msg)
hash_content = list() This includes:
try: * source artifacts (tarballs, repositories) used to build;
source_id = fs.for_package_version(self, self.version).source_id() * content hashes (``sha256``'s) of all patches applied by Spack; and
except (fs.ExtrapolationError, fs.InvalidArgsError): * canonicalized contents the ``package.py`` recipe used to build.
# ExtrapolationError happens if the package has no fetchers defined.
# InvalidArgsError happens when there are version directives with args,
# but none of them identifies an actual fetcher.
source_id = None
if not source_id: This hash is only included in Spack's DAG hash for concrete specs, but if it
# TODO? in cases where a digest or source_id isn't available, happens to be called on a package with an abstract spec, only applicable (i.e.,
# should this attempt to download the source and set one? This determinable) portions of the hash will be included.
# probably only happens for source repositories which are
# referenced by branch name rather than tag or commit ID.
env = spack.environment.active_environment()
from_local_sources = env and env.is_develop(self.spec)
if not self.spec.external and not from_local_sources:
message = 'Missing a source id for {s.name}@{s.version}'
tty.warn(message.format(s=self))
hash_content.append(''.encode('utf-8'))
else:
hash_content.append(source_id.encode('utf-8'))
hash_content.extend(':'.join((p.sha256, str(p.level))).encode('utf-8') """
for p in self.spec.patches) # list of components to make up the hash
hash_content = []
# source artifacts/repositories
# TODO: resources
if self.spec.versions.concrete:
try:
source_id = fs.for_package_version(self, self.version).source_id()
except (fs.ExtrapolationError, fs.InvalidArgsError):
# ExtrapolationError happens if the package has no fetchers defined.
# InvalidArgsError happens when there are version directives with args,
# but none of them identifies an actual fetcher.
source_id = None
if not source_id:
# TODO? in cases where a digest or source_id isn't available,
# should this attempt to download the source and set one? This
# probably only happens for source repositories which are
# referenced by branch name rather than tag or commit ID.
env = spack.environment.active_environment()
from_local_sources = env and env.is_develop(self.spec)
if not self.spec.external and not from_local_sources:
message = 'Missing a source id for {s.name}@{s.version}'
tty.warn(message.format(s=self))
hash_content.append(''.encode('utf-8'))
else:
hash_content.append(source_id.encode('utf-8'))
# patch sha256's
if self.spec.concrete:
hash_content.extend(
':'.join((p.sha256, str(p.level))).encode('utf-8')
for p in self.spec.patches
)
# package.py contents
hash_content.append(package_hash(self.spec, source=content).encode('utf-8')) hash_content.append(package_hash(self.spec, source=content).encode('utf-8'))
# put it all together and encode as base32
b32_hash = base64.b32encode( b32_hash = base64.b32encode(
hashlib.sha256(bytes().join( hashlib.sha256(
sorted(hash_content))).digest()).lower() bytes().join(sorted(hash_content))
).digest()
).lower()
# convert from bytes if running python 3 # convert from bytes if running python 3
if sys.version_info[0] >= 3: if sys.version_info[0] >= 3:

View File

@ -215,9 +215,9 @@ def test_python_ignore_namespace_init_conflict(
python_spec = spack.spec.Spec('python@2.7.12') python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec, ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
monkeypatch, py_namespace) monkeypatch, py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec, ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
monkeypatch, py_namespace) monkeypatch, py_namespace)
view_dir = str(tmpdir.join('view')) view_dir = str(tmpdir.join('view'))
@ -250,9 +250,9 @@ def test_python_keep_namespace_init(
python_spec = spack.spec.Spec('python@2.7.12') python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec, ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
monkeypatch, py_namespace) monkeypatch, py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec, ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
monkeypatch, py_namespace) monkeypatch, py_namespace)
view_dir = str(tmpdir.join('view')) view_dir = str(tmpdir.join('view'))
@ -293,9 +293,9 @@ def test_python_namespace_conflict(tmpdir, namespace_extensions,
python_spec = spack.spec.Spec('python@2.7.12') python_spec = spack.spec.Spec('python@2.7.12')
python_spec._concrete = True python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1@1.0.0', ext1_prefix, python_spec, ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
monkeypatch, py_namespace) monkeypatch, py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2@1.0.0', ext2_prefix, python_spec, ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
monkeypatch, other_namespace) monkeypatch, other_namespace)
view_dir = str(tmpdir.join('view')) view_dir = str(tmpdir.join('view'))

View File

@ -96,6 +96,26 @@ def test_content_hash_all_same_but_patch_contents(mock_packages, config):
compare_hash_sans_name(False, spec1, spec2) compare_hash_sans_name(False, spec1, spec2)
def test_content_hash_not_concretized(mock_packages, config):
"""Check that Package.content_hash() works on abstract specs."""
# these are different due to the package hash
spec1 = Spec("hash-test1@1.1")
spec2 = Spec("hash-test2@1.3")
compare_hash_sans_name(False, spec1, spec2)
# at v1.1 these are actually the same package when @when's are removed
# and the name isn't considered
spec1 = Spec("hash-test1@1.1")
spec2 = Spec("hash-test2@1.1")
compare_hash_sans_name(True, spec1, spec2)
# these end up being different b/c we can't eliminate much of the package.py
# without a version.
spec1 = Spec("hash-test1")
spec2 = Spec("hash-test2")
compare_hash_sans_name(False, spec1, spec2)
def test_content_hash_different_variants(mock_packages, config): def test_content_hash_different_variants(mock_packages, config):
spec1 = Spec("hash-test1@1.2 +variantx").concretized() spec1 = Spec("hash-test1@1.2 +variantx").concretized()
spec2 = Spec("hash-test2@1.2 ~variantx").concretized() spec2 = Spec("hash-test2@1.2 ~variantx").concretized()

View File

@ -143,7 +143,7 @@ def test_check_prefix_manifest(tmpdir):
prefix_path = tmpdir.join('prefix') prefix_path = tmpdir.join('prefix')
prefix = str(prefix_path) prefix = str(prefix_path)
spec = spack.spec.Spec('libelf@0.8.13') spec = spack.spec.Spec('libelf')
spec._mark_concrete() spec._mark_concrete()
spec.prefix = prefix spec.prefix = prefix