zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixup binary cache reuse

Browse Source 
Signed-off-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>
This commit is contained in:
Massimiliano Culpo 2024-10-22 11:57:17 +02:00
parent 9ca0bd5d97
commit e6729320f7
No known key found for this signature in database
GPG Key ID: 3E52BB992233066C
4 changed files with 66 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
lib/spack/spack/binary_distribution.py
View File

@ -625,10 +625,11 @@ def tarball_directory_name(spec):
Return name of the tarball directory according to the convention
<os>-<architecture>/<compiler>/<package>-<version>/
"""
if not spec.compiler_as_nodes():
if spec.original_spec_format() < 5:
compiler = spec.annotations.compiler_node_attribute
assert compiler is not None, "a compiler spec is expected"
return spec.format_path(
f"{spec.architecture}/{spec.compiler_annotation.name}"
f"-{spec.compiler_annotation.version}/{spec.name}-{spec.version}"
f"{spec.architecture}/{compiler.name}-{compiler.version}/{spec.name}-{spec.version}"
)
return spec.format_path(f"{spec.architecture.platform}/{spec.name}-{spec.version}")
@ -639,10 +640,11 @@ def tarball_name(spec, ext):
Return the name of the tarfile according to the convention
<os>-<architecture>-<package>-<dag_hash><ext>
"""
if not spec.compiler_as_nodes():
if spec.original_spec_format() < 5:
compiler = spec.annotations.compiler_node_attribute
assert compiler is not None, "a compiler spec is expected"
spec_formatted = (
f"{spec.architecture}-{spec.compiler_annotation.name}"
f"-{spec.compiler_annotation.version}-{spec.name}"
f"{spec.architecture}-{compiler.name}-{compiler.version}-{spec.name}"
f"-{spec.version}-{spec.dag_hash()}"
)
else:

4
lib/spack/spack/environment/environment.py
View File

@ -23,7 +23,6 @@
import spack
import spack.caches
import spack.compilers.config
import spack.concretize
import spack.config
import spack.deptypes as dt
@ -134,7 +133,7 @@ def default_manifest_yaml():
valid_environment_name_re = r"^\w[\w-]*$"
#: version of the lockfile format. Must increase monotonically.
lockfile_format_version = 5
lockfile_format_version = 6
READER_CLS = {
@ -143,6 +142,7 @@ def default_manifest_yaml():
3: spack.spec.SpecfileV2,
4: spack.spec.SpecfileV3,
5: spack.spec.SpecfileV4,
6: spack.spec.SpecfileV5,
}

4
lib/spack/spack/provider_index.py
View File

@ -240,8 +240,8 @@ def from_json(stream, repository):
index.providers = _transform(
providers,
lambda vpkg, plist: (
spack.spec.SpecfileV4.from_node_dict(vpkg),
set(spack.spec.SpecfileV4.from_node_dict(p) for p in plist),
spack.spec.SpecfileV5.from_node_dict(vpkg),
set(spack.spec.SpecfileV5.from_node_dict(p) for p in plist),
),
)
return index

68
lib/spack/spack/spec.py
View File

@ -1397,6 +1397,26 @@ def tree(
return out
class SpecAnnotations:
def __init__(self) -> None:
self.original_spec_format = SPECFILE_FORMAT_VERSION
self.compiler_node_attribute: Optional["spack.spec.Spec"] = None
def with_spec_format(self, spec_format: int) -> "SpecAnnotations":
self.original_spec_format = spec_format
return self
def with_compiler(self, compiler: "spack.spec.Spec") -> "SpecAnnotations":
self.compiler_node_attribute = compiler
return self
def __repr__(self) -> str:
result = f"SpecAnnotations().with_spec_format({self.original_spec_format})"
if self.compiler_node_attribute:
result += f"with_compiler({str(self.compiler_node_attribute)})"
return result
@lang.lazy_lexicographic_ordering(set_hash=False)
class Spec:
compiler = DeprecatedCompilerSpec()
@ -1466,6 +1486,7 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None)
# is deployed "as built."
# Build spec should be the actual build spec unless marked dirty.
self._build_spec = None
self.annotations = SpecAnnotations()
if isinstance(spec_like, str):
spack.spec_parser.parse_one_or_raise(spec_like, self)
@ -2338,6 +2359,14 @@ def to_node_dict(self, hash=ht.dag_hash):
d["build_spec"] = syaml.syaml_dict(
[("name", self.build_spec.name), (hash.name, self.build_spec._cached_hash(hash))]
)
# Annotations
d["annotations"] = syaml.syaml_dict(
[("original_specfile_version", self.annotations.original_spec_format)]
)
if self.annotations.original_spec_format < 5:
d["annotations"]["compiler"] = str(self.annotations.compiler_node_attribute)
return d
def to_dict(self, hash=ht.dag_hash):
@ -3598,9 +3627,6 @@ def _dup(self, other, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, clearde
self._dependents = _EdgeMap(store_by_child=False)
self._dependencies = _EdgeMap(store_by_child=True)
if hasattr(other, "compiler_annotation"):
self.compiler_annotation = other.compiler_annotation
self.compiler_flags = other.compiler_flags.copy()
self.compiler_flags.spec = self
self.variants = other.variants.copy()
@ -3619,6 +3645,7 @@ def _dup(self, other, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, clearde
self.external_modules = other.external_modules
self.extra_attributes = other.extra_attributes
self.namespace = other.namespace
self.annotations = other.annotations
# If we copy dependencies, preserve DAG structure in the new spec
if deps:
@ -4550,9 +4577,9 @@ def attach_git_version_lookup(self):
if isinstance(v, vn.GitVersion) and v._ref_version is None:
v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname))
def compiler_as_nodes(self) -> bool:
"""Returns True if compiler are treated as nodes"""
return not hasattr(self, "compiler_annotation")
def original_spec_format(self) -> int:
"""Returns the spec format originally used for this spec."""
return self.annotations.original_spec_format
class VariantMap(lang.HashableMap):
@ -4867,10 +4894,6 @@ def from_node_dict(cls, node):
if "arch" in node:
spec.architecture = ArchSpec.from_dict(node)
if "compiler" in node:
# Annotate the compiler spec, might be used later
spec.compiler_annotation = cls.legacy_compiler(node)
propagated_names = node.get("propagate", [])
for name, values in node.get("parameters", {}).items():
propagate = name in propagated_names
@ -4911,6 +4934,17 @@ def from_node_dict(cls, node):
# FIXME: Monkey patches mvar to store patches order
mvar._patches_in_order_of_appearance = patches
# Annotate the compiler spec, might be used later
if "annotations" not in node:
# Specfile v4 and earlier
spec.annotations.with_spec_format(cls.SPEC_VERSION)
if "compiler" in node:
spec.annotations.with_compiler(cls.legacy_compiler(node))
else:
spec.annotations.with_spec_format(node["annotations"]["original_specfile_version"])
if "compiler" in node["annotations"]:
spec.annotations.with_compiler(Spec(f"{node['annotations']['compiler']}"))
# Don't read dependencies here; from_dict() is used by
# from_yaml() and from_json() to read the root *and* each dependency
# spec.
@ -4920,9 +4954,7 @@ def from_node_dict(cls, node):
@classmethod
def legacy_compiler(cls, node):
d = node["compiler"]
return Spec(
f"{d['name']}@{vn.VersionList.from_dict(d)}", external_path="/dev-null/", concrete=True
)
return Spec(f"{d['name']}@{vn.VersionList.from_dict(d)}")
@classmethod
def _load(cls, data):
@ -4990,6 +5022,8 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
class SpecfileV1(SpecfileReaderBase):
SPEC_VERSION = 1
@classmethod
def load(cls, data):
"""Construct a spec from JSON/YAML using the format version 1.
@ -5059,6 +5093,8 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
class SpecfileV2(SpecfileReaderBase):
SPEC_VERSION = 2
@classmethod
def load(cls, data):
result = cls._load(data)
@ -5113,10 +5149,12 @@ def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name
class SpecfileV3(SpecfileV2):
pass
SPEC_VERSION = 3
class SpecfileV4(SpecfileV2):
SPEC_VERSION = 4
@classmethod
def extract_info_from_dep(cls, elt, hash):
dep_hash = elt[hash.name]
@ -5131,6 +5169,8 @@ def load(cls, data):
class SpecfileV5(SpecfileV4):
SPEC_VERSION = 5
@classmethod
def legacy_compiler(cls, node):
raise RuntimeError("The 'compiler' option is unexpected in specfiles at v5 or greater")