oci: when base image uses Image Manifest Version 2, follow suit (#42777)

lib/spack/spack/binary_distribution.py

@@ -1541,7 +1541,7 @@ def fetch_url_to_mirror(url):
                     response = spack.oci.opener.urlopen(
                         urllib.request.Request(
                             url=ref.manifest_url(),
-                            headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
+                            headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)},
                         )
                     )
                 except Exception:

lib/spack/spack/cmd/buildcache.py

@@ -594,6 +594,15 @@ def _put_manifest(
     base_manifest, base_config = base_images[architecture]
     env = _retrieve_env_dict_from_config(base_config)
 
+    # If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too.
+    # This is because Singularity / Apptainer is very strict about not mixing them.
+    base_manifest_mediaType = base_manifest.get(
+        "mediaType", "application/vnd.oci.image.manifest.v1+json"
+    )
+    use_docker_format = (
+        base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json"
+    )
+
     spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)
 
     # Create an oci.image.config file
@@ -625,8 +634,8 @@ def _put_manifest(
     # Upload the config file
     upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum)
 
-    oci_manifest = {
+    manifest = {
-        "mediaType": "application/vnd.oci.image.manifest.v1+json",
+        "mediaType": base_manifest_mediaType,
         "schemaVersion": 2,
         "config": {
             "mediaType": base_manifest["config"]["mediaType"],
@@ -637,7 +646,11 @@ def _put_manifest(
             *(layer for layer in base_manifest["layers"]),
             *(
                 {
-                    "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
+                    "mediaType": (
+                        "application/vnd.docker.image.rootfs.diff.tar.gzip"
+                        if use_docker_format
+                        else "application/vnd.oci.image.layer.v1.tar+gzip"
+                    ),
                     "digest": str(checksums[s.dag_hash()].compressed_digest),
                     "size": checksums[s.dag_hash()].size,
                 }
@@ -646,11 +659,11 @@ def _put_manifest(
         ],
     }
 
-    if annotations:
+    if not use_docker_format and annotations:
-        oci_manifest["annotations"] = annotations
+        manifest["annotations"] = annotations
 
     # Finally upload the manifest
-    upload_manifest_with_retry(image_ref, oci_manifest=oci_manifest)
+    upload_manifest_with_retry(image_ref, manifest=manifest)
 
     # delete the config file
     os.unlink(config_file)

lib/spack/spack/oci/oci.py

@@ -161,7 +161,7 @@ def upload_blob(
 
 def upload_manifest(
     ref: ImageReference,
-    oci_manifest: dict,
+    manifest: dict,
     tag: bool = True,
     _urlopen: spack.oci.opener.MaybeOpen = None,
 ):
@@ -169,7 +169,7 @@ def upload_manifest(
 
     Args:
         ref: The image reference.
-        oci_manifest: The OCI manifest or index.
+        manifest: The manifest or index.
         tag: When true, use the tag, otherwise use the digest,
             this is relevant for multi-arch images, where the
             tag is an index, referencing the manifests by digest.
@@ -179,7 +179,7 @@ def upload_manifest(
     """
     _urlopen = _urlopen or spack.oci.opener.urlopen
 
-    data = json.dumps(oci_manifest, separators=(",", ":")).encode()
+    data = json.dumps(manifest, separators=(",", ":")).encode()
     digest = Digest.from_sha256(hashlib.sha256(data).hexdigest())
     size = len(data)
 
@@ -190,7 +190,7 @@ def upload_manifest(
         url=ref.manifest_url(),
         method="PUT",
         data=data,
-        headers={"Content-Type": oci_manifest["mediaType"]},
+        headers={"Content-Type": manifest["mediaType"]},
     )
 
     response = _urlopen(request)

lib/spack/spack/test/oci/integration_test.py

@@ -9,6 +9,7 @@
 import hashlib
 import json
 import os
+import pathlib
 from contextlib import contextmanager
 
 import spack.environment as ev
@@ -172,6 +173,12 @@ def test_buildcache_push_with_base_image_command(
         dst_image = ImageReference.from_string(f"dst.example.com/image:{tag}")
         retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image)
 
+        # Check that the media type is OCI
+        assert retrieved_manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json"
+        assert (
+            retrieved_manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json"
+        )
+
         # Check that the base image layer is first.
         assert retrieved_manifest["layers"][0]["digest"] == str(tar_gz_digest)
         assert retrieved_config["rootfs"]["diff_ids"][0] == str(tar_digest)
@@ -189,3 +196,93 @@ def test_buildcache_push_with_base_image_command(
         # And verify that all layers including the base layer are present
         for layer in retrieved_manifest["layers"]:
             assert blob_exists(dst_image, digest=Digest.from_string(layer["digest"]))
+            assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip"
+
+
+def test_uploading_with_base_image_in_docker_image_manifest_v2_format(
+    tmp_path: pathlib.Path, mutable_database, disable_parallel_buildcache_push
+):
+    """If the base image uses an old manifest schema, Spack should also use that.
+    That is necessary for container images to work with Apptainer, which is rather strict about
+    mismatching manifest/layer types."""
+
+    registry_src = InMemoryOCIRegistry("src.example.com")
+    registry_dst = InMemoryOCIRegistry("dst.example.com")
+
+    base_image = ImageReference.from_string("src.example.com/my-base-image:latest")
+
+    with oci_servers(registry_src, registry_dst):
+        mirror("add", "oci-test", "oci://dst.example.com/image")
+
+        # Create a dummy base image (blob, config, manifest) in registry A in the Docker Image
+        # Manifest V2 format.
+        rootfs = tmp_path / "rootfs"
+        (rootfs / "bin").mkdir(parents=True)
+        (rootfs / "bin" / "sh").write_text("hello world")
+        tarball = tmp_path / "base.tar.gz"
+        with gzip_compressed_tarfile(tarball) as (tar, tar_gz_checksum, tar_checksum):
+            tar.add(rootfs, arcname=".")
+        tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest())
+        tar_digest = Digest.from_sha256(tar_checksum.hexdigest())
+        upload_blob(base_image, str(tarball), tar_gz_digest)
+        config = {
+            "created": "2015-10-31T22:22:56.015925234Z",
+            "author": "Foo <example@example.com>",
+            "architecture": "amd64",
+            "os": "linux",
+            "config": {
+                "User": "foo",
+                "Memory": 2048,
+                "MemorySwap": 4096,
+                "CpuShares": 8,
+                "ExposedPorts": {"8080/tcp": {}},
+                "Env": ["PATH=/usr/bin:/bin"],
+                "Entrypoint": ["/bin/sh"],
+                "Cmd": ["-c", "'echo hello world'"],
+                "Volumes": {"/x": {}},
+                "WorkingDir": "/",
+            },
+            "rootfs": {"diff_ids": [str(tar_digest)], "type": "layers"},
+            "history": [
+                {
+                    "created": "2015-10-31T22:22:54.690851953Z",
+                    "created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /",
+                }
+            ],
+        }
+        config_file = tmp_path / "config.json"
+        config_file.write_text(json.dumps(config))
+        config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest())
+        upload_blob(base_image, str(config_file), config_digest)
+        manifest = {
+            "schemaVersion": 2,
+            "mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+            "config": {
+                "mediaType": "application/vnd.docker.container.image.v1+json",
+                "size": config_file.stat().st_size,
+                "digest": str(config_digest),
+            },
+            "layers": [
+                {
+                    "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+                    "size": tarball.stat().st_size,
+                    "digest": str(tar_gz_digest),
+                }
+            ],
+        }
+        upload_manifest(base_image, manifest)
+
+        # Finally upload some package to registry B with registry A's image as base
+        buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich")
+
+    # Should have some manifests uploaded to registry B now.
+    assert registry_dst.manifests
+
+    # Verify that all manifest are in the Docker Image Manifest V2 format, not OCI.
+    # And also check that we're not using annotations, which is an OCI-only "feature".
+    for m in registry_dst.manifests.values():
+        assert m["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json"
+        assert m["config"]["mediaType"] == "application/vnd.docker.container.image.v1+json"
+        for layer in m["layers"]:
+            assert layer["mediaType"] == "application/vnd.docker.image.rootfs.diff.tar.gzip"
+        assert "annotations" not in m

lib/spack/spack/test/oci/mock_registry.py

@@ -17,6 +17,7 @@
 from typing import Callable, Dict, List, Optional, Pattern, Tuple
 from urllib.request import Request
 
+import spack.oci.oci
 from spack.oci.image import Digest
 from spack.oci.opener import OCIAuthHandler
 
@@ -171,7 +172,7 @@ def __init__(self, domain: str, allow_single_post: bool = True) -> None:
         self.blobs: Dict[str, bytes] = {}
 
         # Map from (name, tag) to manifest
-        self.manifests: Dict[Tuple[str, str], Dict] = {}
+        self.manifests: Dict[Tuple[str, str], dict] = {}
 
     def index(self, req: Request):
         return MockHTTPResponse.with_json(200, "OK", body={})
@@ -225,15 +226,12 @@ def put_session(self, req: Request):
     def put_manifest(self, req: Request, name: str, ref: str):
         # In requests, Python runs header.capitalize().
         content_type = req.get_header("Content-type")
-        assert content_type in (
+        assert content_type in spack.oci.oci.all_content_type
-            "application/vnd.oci.image.manifest.v1+json",
-            "application/vnd.oci.image.index.v1+json",
-        )
 
         index_or_manifest = json.loads(self._require_data(req))
 
         # Verify that we have all blobs (layers for manifest, manifests for index)
-        if content_type == "application/vnd.oci.image.manifest.v1+json":
+        if content_type in spack.oci.oci.manifest_content_type:
             for layer in index_or_manifest["layers"]:
                 assert layer["digest"] in self.blobs, "Missing blob while uploading manifest"