From 89f860e83041665366e4ffea533392156ba22d11 Mon Sep 17 00:00:00 2001 From: tarilabs Date: Wed, 5 Mar 2025 11:27:48 +0100 Subject: [PATCH 1/2] core: allow oci_layers_on_top with single manifest Signed-off-by: tarilabs --- olot/basics.py | 26 +++++++++------ tests/basic_test.py | 32 ++++++++++++++++++- tests/data/ocilayout5/README.md | 4 +++ ...fff837162f5fd38c4e8daf8de1a120df9a7d79d633 | 1 + ...dd99fedad4445f3e835edb58760b2f83f2c0517878 | 1 + tests/data/ocilayout5/index.json | 1 + tests/data/ocilayout5/oci-layout | 1 + 7 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 tests/data/ocilayout5/README.md create mode 100644 tests/data/ocilayout5/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633 create mode 100644 tests/data/ocilayout5/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878 create mode 100644 tests/data/ocilayout5/index.json create mode 100644 tests/data/ocilayout5/oci-layout diff --git a/olot/basics.py b/olot/basics.py index e452534..37ad5fe 100644 --- a/olot/basics.py +++ b/olot/basics.py @@ -41,9 +41,9 @@ def oci_layers_on_top( logger.info("Invoked with 'remove' to delete original contents after adding as a blob layer.") verify_ocilayout(ocilayout) - ocilayout_root_index = read_ocilayout_root_index(ocilayout) + ocilayout_root_index: OCIImageIndex = read_ocilayout_root_index(ocilayout) ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout, ocilayout_root_index) - ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes) + ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes, ocilayout_root_index) new_layers = {} # layer digest : diff_id sha256_path = ocilayout / "blobs" / "sha256" @@ -137,15 +137,26 @@ def oci_layers_on_top( -def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex]) -> Dict[str, OCIImageManifest]: +def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex], ocilayout_root_index: OCIImageIndex = None) -> Dict[str, OCIImageManifest]: + """crawl Manifests from referred OCI Index(es) and Manifests in the root index of the oci-layout + """ ocilayout_manifests: Dict[str, OCIImageManifest] = {} for _, mi in ocilayout_indexes.items(): for m in mi.manifests: - print(m) + logger.debug("Parsing manifest %s", m) if m.mediaType != MediaTypes.manifest: raise ValueError("Did not expect something else than Image Manifest in a Index") target_hash = m.digest.removeprefix("sha256:") - print(target_hash) + logger.debug("target_hash %s", target_hash) + manifest_path = ocilayout / "blobs" / "sha256" / target_hash + with open(manifest_path, "r") as ip: + ocilayout_manifests[target_hash] = OCIImageManifest.model_validate_json(ip.read()) + if ocilayout_root_index is None: + return ocilayout_manifests # return early + for m in ocilayout_root_index.manifests: + if m.mediaType == MediaTypes.manifest: + target_hash = m.digest.removeprefix("sha256:") + logger.debug("Lookup remainder manifest from ocilayout_root_index having target_hash %s", target_hash) manifest_path = ocilayout / "blobs" / "sha256" / target_hash with open(manifest_path, "r") as ip: ocilayout_manifests[target_hash] = OCIImageManifest.model_validate_json(ip.read()) @@ -160,11 +171,6 @@ def crawl_ocilayout_indexes(ocilayout: Path, ocilayout_root_index: OCIImageIndex index_path = ocilayout / "blobs" / "sha256" / target_hash with open(index_path, "r") as ip: ocilayout_indexes[target_hash] = OCIImageIndex.model_validate_json(ip.read()) - else: - if len(ocilayout_indexes) == 0: - raise ValueError("TODO the root index has Image manifest") - else: - click.echo(f"Found Image Manifest {m.digest} in root index, TODO assuming these are referred through the other indexes") return ocilayout_indexes diff --git a/tests/basic_test.py b/tests/basic_test.py index 9c9ca64..86ab20e 100644 --- a/tests/basic_test.py +++ b/tests/basic_test.py @@ -37,7 +37,7 @@ def test_crawl_ocilayout_manifests(): ocilayout3_path = Path(__file__).parent / "data" / "ocilayout3" ocilayout_root_index = read_ocilayout_root_index(ocilayout3_path) ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout3_path, ocilayout_root_index) - mut: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout3_path, ocilayout_indexes) + mut: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout3_path, ocilayout_indexes, ocilayout_root_index) assert len(mut.keys()) == 2 assert "c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878" in mut.keys() @@ -93,3 +93,33 @@ def test_oci_layers_on_top_with_remove(tmp_path: Path): for model in models: assert not model.exists() assert not modelcard.exists() + + +def test_oci_layers_on_top_single_manifest(tmp_path: Path): + """check oci_layers_on_top with an oci-layout directory containing a single manifest + """ + test_sample_model = sample_model_path() + test_ocilayout5 = test_data_path() / "ocilayout5" + target_ocilayout = tmp_path / "myocilayout" + shutil.copytree(test_ocilayout5, target_ocilayout) + target_model = tmp_path / "models" + shutil.copytree(test_sample_model, target_model) + print(os.listdir(target_model)) + + models = [ + target_model / "model.joblib", + target_model / "hello.md" + ] + for model in models: + assert model.exists() + modelcard = target_model / "README.md" + assert modelcard.exists() + + oci_layers_on_top(target_ocilayout, models, modelcard) + + ocilayout_root_index: OCIImageIndex = read_ocilayout_root_index(target_ocilayout) + ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(target_ocilayout, ocilayout_root_index) + ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(target_ocilayout, ocilayout_indexes, ocilayout_root_index) + assert len(ocilayout_manifests) == 1 + manifest0: OCIImageManifest = next(iter(ocilayout_manifests.values())) + assert len(manifest0.layers) == 1 + len(models) + 1 # original value (only 1 layer in original oci-layout) + now added model files + now added modelcarD diff --git a/tests/data/ocilayout5/README.md b/tests/data/ocilayout5/README.md new file mode 100644 index 0000000..0227ca3 --- /dev/null +++ b/tests/data/ocilayout5/README.md @@ -0,0 +1,4 @@ +example only manifests (not layer tarballs) with: `oras copy --platform linux/arm64 --to-oci-layout quay.io/mmortari/hello-world-wait:latest ./tests/data/ocilayout5:latest` + +Q: "why you didn't use skopeo for this case?" +A: cannot supply a specific platform diff --git a/tests/data/ocilayout5/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633 b/tests/data/ocilayout5/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633 new file mode 100644 index 0000000..5f5b431 --- /dev/null +++ b/tests/data/ocilayout5/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633 @@ -0,0 +1 @@ +{"created":"2024-11-29T10:50:43.850369849Z","architecture":"arm64","os":"linux","variant":"v8","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:f21ad18174949794e810922c8ada6ff8416aabab8ef3fd3bd144e47058359f52"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:50:43.850412432Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]} \ No newline at end of file diff --git a/tests/data/ocilayout5/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878 b/tests/data/ocilayout5/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878 new file mode 100644 index 0000000..dc4ef28 --- /dev/null +++ b/tests/data/ocilayout5/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878 @@ -0,0 +1 @@ +{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633","size":778},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1933e30a3373776d5c7155591a6dacbc205cf6a2665b6dced682c6d2ea7b000f","size":1949749}],"annotations":{"org.opencontainers.image.base.digest":"sha256:6ca1ac3927a17445a61188b4f91af0bfb1e0b16757b07ec9f556e9e1e0851b15","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}} \ No newline at end of file diff --git a/tests/data/ocilayout5/index.json b/tests/data/ocilayout5/index.json new file mode 100644 index 0000000..cf9035e --- /dev/null +++ b/tests/data/ocilayout5/index.json @@ -0,0 +1 @@ +{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878","size":731,"annotations":{"org.opencontainers.image.ref.name":"latest"},"platform":{"architecture":"arm64","os":"linux"}}]} \ No newline at end of file diff --git a/tests/data/ocilayout5/oci-layout b/tests/data/ocilayout5/oci-layout new file mode 100644 index 0000000..1343d37 --- /dev/null +++ b/tests/data/ocilayout5/oci-layout @@ -0,0 +1 @@ +{"imageLayoutVersion":"1.0.0"} \ No newline at end of file From 752e107f310624abfc83389ddadd93f88dc2b888 Mon Sep 17 00:00:00 2001 From: tarilabs Date: Wed, 5 Mar 2025 11:44:29 +0100 Subject: [PATCH 2/2] linting Signed-off-by: tarilabs --- olot/basics.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/olot/basics.py b/olot/basics.py index 37ad5fe..d43c9d3 100644 --- a/olot/basics.py +++ b/olot/basics.py @@ -6,7 +6,6 @@ import tarfile from typing import Dict, List, Sequence import typing -import click from olot.oci.oci_config import OCIManifestConfig @@ -137,7 +136,7 @@ def oci_layers_on_top( -def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex], ocilayout_root_index: OCIImageIndex = None) -> Dict[str, OCIImageManifest]: +def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex], ocilayout_root_index: typing.Union[OCIImageIndex, None] = None) -> Dict[str, OCIImageManifest]: """crawl Manifests from referred OCI Index(es) and Manifests in the root index of the oci-layout """ ocilayout_manifests: Dict[str, OCIImageManifest] = {}