zhangyiss/spack
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

oci: when base image uses Image Manifest Version 2, follow suit (#42777)

Browse Source
This commit is contained in:
Harmen Stoppels 2024-02-22 16:33:56 +01:00 committed by GitHub
parent 90778873d1
commit 3d1d5f755f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 125 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib/spack/spack/binary_distribution.py
View File

@ -1541,7 +1541,7 @@ def fetch_url_to_mirror(url):
response = spack.oci.opener.urlopen(
urllib.request.Request(
url=ref.manifest_url(),
headers={"Accept": "application/vnd.oci.image.manifest.v1+json"},
headers={"Accept": ", ".join(spack.oci.oci.manifest_content_type)},
)
)
except Exception:

25
lib/spack/spack/cmd/buildcache.py
View File

@ -594,6 +594,15 @@ def _put_manifest(
base_manifest, base_config = base_images[architecture]
env = _retrieve_env_dict_from_config(base_config)
# If the base image uses `vnd.docker.distribution.manifest.v2+json`, then we use that too.
# This is because Singularity / Apptainer is very strict about not mixing them.
base_manifest_mediaType = base_manifest.get(
"mediaType", "application/vnd.oci.image.manifest.v1+json"
)
use_docker_format = (
base_manifest_mediaType == "application/vnd.docker.distribution.manifest.v2+json"
)
spack.user_environment.environment_modifications_for_specs(*specs).apply_modifications(env)
# Create an oci.image.config file
@ -625,8 +634,8 @@ def _put_manifest(
# Upload the config file
upload_blob_with_retry(image_ref, file=config_file, digest=config_file_checksum)
oci_manifest = {
"mediaType": "application/vnd.oci.image.manifest.v1+json",
manifest = {
"mediaType": base_manifest_mediaType,
"schemaVersion": 2,
"config": {
"mediaType": base_manifest["config"]["mediaType"],
@ -637,7 +646,11 @@ def _put_manifest(
*(layer for layer in base_manifest["layers"]),
*(
{
"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip",
"mediaType": (
"application/vnd.docker.image.rootfs.diff.tar.gzip"
if use_docker_format
else "application/vnd.oci.image.layer.v1.tar+gzip"
),
"digest": str(checksums[s.dag_hash()].compressed_digest),
"size": checksums[s.dag_hash()].size,
}
@ -646,11 +659,11 @@ def _put_manifest(
],
}
if annotations:
oci_manifest["annotations"] = annotations
if not use_docker_format and annotations:
manifest["annotations"] = annotations
# Finally upload the manifest
upload_manifest_with_retry(image_ref, oci_manifest=oci_manifest)
upload_manifest_with_retry(image_ref, manifest=manifest)
# delete the config file
os.unlink(config_file)

8
lib/spack/spack/oci/oci.py
View File

@ -161,7 +161,7 @@ def upload_blob(
def upload_manifest(
ref: ImageReference,
oci_manifest: dict,
manifest: dict,
tag: bool = True,
_urlopen: spack.oci.opener.MaybeOpen = None,
):
@ -169,7 +169,7 @@ def upload_manifest(
Args:
ref: The image reference.
oci_manifest: The OCI manifest or index.
manifest: The manifest or index.
tag: When true, use the tag, otherwise use the digest,
this is relevant for multi-arch images, where the
tag is an index, referencing the manifests by digest.
@ -179,7 +179,7 @@ def upload_manifest(
"""
_urlopen = _urlopen or spack.oci.opener.urlopen
data = json.dumps(oci_manifest, separators=(",", ":")).encode()
data = json.dumps(manifest, separators=(",", ":")).encode()
digest = Digest.from_sha256(hashlib.sha256(data).hexdigest())
size = len(data)
@ -190,7 +190,7 @@ def upload_manifest(
url=ref.manifest_url(),
method="PUT",
data=data,
headers={"Content-Type": oci_manifest["mediaType"]},
headers={"Content-Type": manifest["mediaType"]},
)
response = _urlopen(request)

97
lib/spack/spack/test/oci/integration_test.py
View File

@ -9,6 +9,7 @@
import hashlib
import json
import os
import pathlib
from contextlib import contextmanager
import spack.environment as ev
@ -172,6 +173,12 @@ def test_buildcache_push_with_base_image_command(
dst_image = ImageReference.from_string(f"dst.example.com/image:{tag}")
retrieved_manifest, retrieved_config = get_manifest_and_config(dst_image)
# Check that the media type is OCI
assert retrieved_manifest["mediaType"] == "application/vnd.oci.image.manifest.v1+json"
assert (
retrieved_manifest["config"]["mediaType"] == "application/vnd.oci.image.config.v1+json"
)
# Check that the base image layer is first.
assert retrieved_manifest["layers"][0]["digest"] == str(tar_gz_digest)
assert retrieved_config["rootfs"]["diff_ids"][0] == str(tar_digest)
@ -189,3 +196,93 @@ def test_buildcache_push_with_base_image_command(
# And verify that all layers including the base layer are present
for layer in retrieved_manifest["layers"]:
assert blob_exists(dst_image, digest=Digest.from_string(layer["digest"]))
assert layer["mediaType"] == "application/vnd.oci.image.layer.v1.tar+gzip"
def test_uploading_with_base_image_in_docker_image_manifest_v2_format(
tmp_path: pathlib.Path, mutable_database, disable_parallel_buildcache_push
):
"""If the base image uses an old manifest schema, Spack should also use that.
That is necessary for container images to work with Apptainer, which is rather strict about
mismatching manifest/layer types."""
registry_src = InMemoryOCIRegistry("src.example.com")
registry_dst = InMemoryOCIRegistry("dst.example.com")
base_image = ImageReference.from_string("src.example.com/my-base-image:latest")
with oci_servers(registry_src, registry_dst):
mirror("add", "oci-test", "oci://dst.example.com/image")
# Create a dummy base image (blob, config, manifest) in registry A in the Docker Image
# Manifest V2 format.
rootfs = tmp_path / "rootfs"
(rootfs / "bin").mkdir(parents=True)
(rootfs / "bin" / "sh").write_text("hello world")
tarball = tmp_path / "base.tar.gz"
with gzip_compressed_tarfile(tarball) as (tar, tar_gz_checksum, tar_checksum):
tar.add(rootfs, arcname=".")
tar_gz_digest = Digest.from_sha256(tar_gz_checksum.hexdigest())
tar_digest = Digest.from_sha256(tar_checksum.hexdigest())
upload_blob(base_image, str(tarball), tar_gz_digest)
config = {
"created": "2015-10-31T22:22:56.015925234Z",
"author": "Foo <example@example.com>",
"architecture": "amd64",
"os": "linux",
"config": {
"User": "foo",
"Memory": 2048,
"MemorySwap": 4096,
"CpuShares": 8,
"ExposedPorts": {"8080/tcp": {}},
"Env": ["PATH=/usr/bin:/bin"],
"Entrypoint": ["/bin/sh"],
"Cmd": ["-c", "'echo hello world'"],
"Volumes": {"/x": {}},
"WorkingDir": "/",
},
"rootfs": {"diff_ids": [str(tar_digest)], "type": "layers"},
"history": [
{
"created": "2015-10-31T22:22:54.690851953Z",
"created_by": "/bin/sh -c #(nop) ADD file:a3bc1e842b69636f9df5256c49c5374fb4eef1e281fe3f282c65fb853ee171c5 in /",
}
],
}
config_file = tmp_path / "config.json"
config_file.write_text(json.dumps(config))
config_digest = Digest.from_sha256(hashlib.sha256(config_file.read_bytes()).hexdigest())
upload_blob(base_image, str(config_file), config_digest)
manifest = {
"schemaVersion": 2,
"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
"config": {
"mediaType": "application/vnd.docker.container.image.v1+json",
"size": config_file.stat().st_size,
"digest": str(config_digest),
},
"layers": [
{
"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
"size": tarball.stat().st_size,
"digest": str(tar_gz_digest),
}
],
}
upload_manifest(base_image, manifest)
# Finally upload some package to registry B with registry A's image as base
buildcache("push", "--base-image", str(base_image), "oci-test", "mpileaks^mpich")
# Should have some manifests uploaded to registry B now.
assert registry_dst.manifests
# Verify that all manifest are in the Docker Image Manifest V2 format, not OCI.
# And also check that we're not using annotations, which is an OCI-only "feature".
for m in registry_dst.manifests.values():
assert m["mediaType"] == "application/vnd.docker.distribution.manifest.v2+json"
assert m["config"]["mediaType"] == "application/vnd.docker.container.image.v1+json"
for layer in m["layers"]:
assert layer["mediaType"] == "application/vnd.docker.image.rootfs.diff.tar.gzip"
assert "annotations" not in m

10
lib/spack/spack/test/oci/mock_registry.py
View File

@ -17,6 +17,7 @@
from typing import Callable, Dict, List, Optional, Pattern, Tuple
from urllib.request import Request
import spack.oci.oci
from spack.oci.image import Digest
from spack.oci.opener import OCIAuthHandler
@ -171,7 +172,7 @@ def __init__(self, domain: str, allow_single_post: bool = True) -> None:
self.blobs: Dict[str, bytes] = {}
# Map from (name, tag) to manifest
self.manifests: Dict[Tuple[str, str], Dict] = {}
self.manifests: Dict[Tuple[str, str], dict] = {}
def index(self, req: Request):
return MockHTTPResponse.with_json(200, "OK", body={})
@ -225,15 +226,12 @@ def put_session(self, req: Request):
def put_manifest(self, req: Request, name: str, ref: str):
# In requests, Python runs header.capitalize().
content_type = req.get_header("Content-type")
assert content_type in (
"application/vnd.oci.image.manifest.v1+json",
"application/vnd.oci.image.index.v1+json",
)
assert content_type in spack.oci.oci.all_content_type
index_or_manifest = json.loads(self._require_data(req))
# Verify that we have all blobs (layers for manifest, manifests for index)
if content_type == "application/vnd.oci.image.manifest.v1+json":
if content_type in spack.oci.oci.manifest_content_type:
for layer in index_or_manifest["layers"]:
assert layer["digest"] in self.blobs, "Missing blob while uploading manifest"