Fixup binary cache reuse

Signed-off-by: Massimiliano Culpo <massimiliano.culpo@gmail.com>

lib/spack/spack/binary_distribution.py

@@ -625,10 +625,11 @@ def tarball_directory_name(spec):
     Return name of the tarball directory according to the convention
     <os>-<architecture>/<compiler>/<package>-<version>/
     """
-    if not spec.compiler_as_nodes():
+    if spec.original_spec_format() < 5:
+        compiler = spec.annotations.compiler_node_attribute
+        assert compiler is not None, "a compiler spec is expected"
         return spec.format_path(
-            f"{spec.architecture}/{spec.compiler_annotation.name}"
+            f"{spec.architecture}/{compiler.name}-{compiler.version}/{spec.name}-{spec.version}"
-            f"-{spec.compiler_annotation.version}/{spec.name}-{spec.version}"
         )
 
     return spec.format_path(f"{spec.architecture.platform}/{spec.name}-{spec.version}")
@@ -639,10 +640,11 @@ def tarball_name(spec, ext):
     Return the name of the tarfile according to the convention
     <os>-<architecture>-<package>-<dag_hash><ext>
     """
-    if not spec.compiler_as_nodes():
+    if spec.original_spec_format() < 5:
+        compiler = spec.annotations.compiler_node_attribute
+        assert compiler is not None, "a compiler spec is expected"
         spec_formatted = (
-            f"{spec.architecture}-{spec.compiler_annotation.name}"
+            f"{spec.architecture}-{compiler.name}-{compiler.version}-{spec.name}"
-            f"-{spec.compiler_annotation.version}-{spec.name}"
             f"-{spec.version}-{spec.dag_hash()}"
         )
     else:

lib/spack/spack/environment/environment.py

@@ -23,7 +23,6 @@
 
 import spack
 import spack.caches
-import spack.compilers.config
 import spack.concretize
 import spack.config
 import spack.deptypes as dt
@@ -134,7 +133,7 @@ def default_manifest_yaml():
 valid_environment_name_re = r"^\w[\w-]*$"
 
 #: version of the lockfile format. Must increase monotonically.
-lockfile_format_version = 5
+lockfile_format_version = 6
 
 
 READER_CLS = {
@@ -143,6 +142,7 @@ def default_manifest_yaml():
     3: spack.spec.SpecfileV2,
     4: spack.spec.SpecfileV3,
     5: spack.spec.SpecfileV4,
+    6: spack.spec.SpecfileV5,
 }
 
 

lib/spack/spack/provider_index.py

@@ -240,8 +240,8 @@ def from_json(stream, repository):
         index.providers = _transform(
             providers,
             lambda vpkg, plist: (
-                spack.spec.SpecfileV4.from_node_dict(vpkg),
+                spack.spec.SpecfileV5.from_node_dict(vpkg),
-                set(spack.spec.SpecfileV4.from_node_dict(p) for p in plist),
+                set(spack.spec.SpecfileV5.from_node_dict(p) for p in plist),
             ),
         )
         return index

lib/spack/spack/spec.py

@@ -1397,6 +1397,26 @@ def tree(
     return out
 
 
+class SpecAnnotations:
+    def __init__(self) -> None:
+        self.original_spec_format = SPECFILE_FORMAT_VERSION
+        self.compiler_node_attribute: Optional["spack.spec.Spec"] = None
+
+    def with_spec_format(self, spec_format: int) -> "SpecAnnotations":
+        self.original_spec_format = spec_format
+        return self
+
+    def with_compiler(self, compiler: "spack.spec.Spec") -> "SpecAnnotations":
+        self.compiler_node_attribute = compiler
+        return self
+
+    def __repr__(self) -> str:
+        result = f"SpecAnnotations().with_spec_format({self.original_spec_format})"
+        if self.compiler_node_attribute:
+            result += f"with_compiler({str(self.compiler_node_attribute)})"
+        return result
+
+
 @lang.lazy_lexicographic_ordering(set_hash=False)
 class Spec:
     compiler = DeprecatedCompilerSpec()
@@ -1466,6 +1486,7 @@ def __init__(self, spec_like=None, *, external_path=None, external_modules=None)
         # is deployed "as built."
         # Build spec should be the actual build spec unless marked dirty.
         self._build_spec = None
+        self.annotations = SpecAnnotations()
 
         if isinstance(spec_like, str):
             spack.spec_parser.parse_one_or_raise(spec_like, self)
@@ -2338,6 +2359,14 @@ def to_node_dict(self, hash=ht.dag_hash):
             d["build_spec"] = syaml.syaml_dict(
                 [("name", self.build_spec.name), (hash.name, self.build_spec._cached_hash(hash))]
             )
+
+        # Annotations
+        d["annotations"] = syaml.syaml_dict(
+            [("original_specfile_version", self.annotations.original_spec_format)]
+        )
+        if self.annotations.original_spec_format < 5:
+            d["annotations"]["compiler"] = str(self.annotations.compiler_node_attribute)
+
         return d
 
     def to_dict(self, hash=ht.dag_hash):
@@ -3598,9 +3627,6 @@ def _dup(self, other, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, clearde
             self._dependents = _EdgeMap(store_by_child=False)
             self._dependencies = _EdgeMap(store_by_child=True)
 
-        if hasattr(other, "compiler_annotation"):
-            self.compiler_annotation = other.compiler_annotation
-
         self.compiler_flags = other.compiler_flags.copy()
         self.compiler_flags.spec = self
         self.variants = other.variants.copy()
@@ -3619,6 +3645,7 @@ def _dup(self, other, deps: Union[bool, dt.DepTypes, dt.DepFlag] = True, clearde
         self.external_modules = other.external_modules
         self.extra_attributes = other.extra_attributes
         self.namespace = other.namespace
+        self.annotations = other.annotations
 
         # If we copy dependencies, preserve DAG structure in the new spec
         if deps:
@@ -4550,9 +4577,9 @@ def attach_git_version_lookup(self):
             if isinstance(v, vn.GitVersion) and v._ref_version is None:
                 v.attach_lookup(spack.version.git_ref_lookup.GitRefLookup(self.fullname))
 
-    def compiler_as_nodes(self) -> bool:
+    def original_spec_format(self) -> int:
-        """Returns True if compiler are treated as nodes"""
+        """Returns the spec format originally used for this spec."""
-        return not hasattr(self, "compiler_annotation")
+        return self.annotations.original_spec_format
 
 
 class VariantMap(lang.HashableMap):
@@ -4867,10 +4894,6 @@ def from_node_dict(cls, node):
         if "arch" in node:
             spec.architecture = ArchSpec.from_dict(node)
 
-        if "compiler" in node:
-            # Annotate the compiler spec, might be used later
-            spec.compiler_annotation = cls.legacy_compiler(node)
-
         propagated_names = node.get("propagate", [])
         for name, values in node.get("parameters", {}).items():
             propagate = name in propagated_names
@@ -4911,6 +4934,17 @@ def from_node_dict(cls, node):
                 # FIXME: Monkey patches mvar to store patches order
                 mvar._patches_in_order_of_appearance = patches
 
+        # Annotate the compiler spec, might be used later
+        if "annotations" not in node:
+            # Specfile v4 and earlier
+            spec.annotations.with_spec_format(cls.SPEC_VERSION)
+            if "compiler" in node:
+                spec.annotations.with_compiler(cls.legacy_compiler(node))
+        else:
+            spec.annotations.with_spec_format(node["annotations"]["original_specfile_version"])
+            if "compiler" in node["annotations"]:
+                spec.annotations.with_compiler(Spec(f"{node['annotations']['compiler']}"))
+
         # Don't read dependencies here; from_dict() is used by
         # from_yaml() and from_json() to read the root *and* each dependency
         # spec.
@@ -4920,9 +4954,7 @@ def from_node_dict(cls, node):
     @classmethod
     def legacy_compiler(cls, node):
         d = node["compiler"]
-        return Spec(
+        return Spec(f"{d['name']}@{vn.VersionList.from_dict(d)}")
-            f"{d['name']}@{vn.VersionList.from_dict(d)}", external_path="/dev-null/", concrete=True
-        )
 
     @classmethod
     def _load(cls, data):
@@ -4990,6 +5022,8 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
 
 
 class SpecfileV1(SpecfileReaderBase):
+    SPEC_VERSION = 1
+
     @classmethod
     def load(cls, data):
         """Construct a spec from JSON/YAML using the format version 1.
@@ -5059,6 +5093,8 @@ def read_specfile_dep_specs(cls, deps, hash_type=ht.dag_hash.name):
 
 
 class SpecfileV2(SpecfileReaderBase):
+    SPEC_VERSION = 2
+
     @classmethod
     def load(cls, data):
         result = cls._load(data)
@@ -5113,10 +5149,12 @@ def extract_build_spec_info_from_node_dict(cls, node, hash_type=ht.dag_hash.name
 
 
 class SpecfileV3(SpecfileV2):
-    pass
+    SPEC_VERSION = 3
 
 
 class SpecfileV4(SpecfileV2):
+    SPEC_VERSION = 4
+
     @classmethod
     def extract_info_from_dep(cls, elt, hash):
         dep_hash = elt[hash.name]
@@ -5131,6 +5169,8 @@ def load(cls, data):
 
 
 class SpecfileV5(SpecfileV4):
+    SPEC_VERSION = 5
+
     @classmethod
     def legacy_compiler(cls, node):
         raise RuntimeError("The 'compiler' option is unexpected in specfiles at v5 or greater")