core: allow oci_layers_on_top with single manifest (#18)

* core: allow oci_layers_on_top with single manifest

Signed-off-by: tarilabs <[email protected]>

* linting

Signed-off-by: tarilabs <[email protected]>

---------

Signed-off-by: tarilabs <[email protected]>

olot/basics.py

@@ -6,7 +6,6 @@
 import tarfile
 from typing import Dict, List, Sequence
 import typing
-import click
 
 from olot.oci.oci_config import OCIManifestConfig
 
@@ -41,9 +40,9 @@ def oci_layers_on_top(
         logger.info("Invoked with 'remove' to delete original contents after adding as a blob layer.")
 
     verify_ocilayout(ocilayout)
-    ocilayout_root_index = read_ocilayout_root_index(ocilayout)
+    ocilayout_root_index: OCIImageIndex = read_ocilayout_root_index(ocilayout)
     ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout, ocilayout_root_index)
-    ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes)
+    ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes, ocilayout_root_index)
     new_layers = {} # layer digest : diff_id
 
     sha256_path = ocilayout / "blobs" / "sha256"
@@ -137,15 +136,26 @@ def oci_layers_on_top(
 
 
 
-def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex]) -> Dict[str, OCIImageManifest]:
+def crawl_ocilayout_manifests(ocilayout: Path, ocilayout_indexes: Dict[str, OCIImageIndex], ocilayout_root_index: typing.Union[OCIImageIndex, None] = None) -> Dict[str, OCIImageManifest]:
+    """crawl Manifests from referred OCI Index(es) and Manifests in the root index of the oci-layout
+    """
     ocilayout_manifests: Dict[str, OCIImageManifest]  = {}
     for _, mi in ocilayout_indexes.items():
         for m in mi.manifests:
-            print(m)
+            logger.debug("Parsing manifest %s", m)
             if m.mediaType != MediaTypes.manifest:
                 raise ValueError("Did not expect something else than Image Manifest in a Index")
             target_hash = m.digest.removeprefix("sha256:")
-            print(target_hash)
+            logger.debug("target_hash %s", target_hash)
+            manifest_path = ocilayout / "blobs" / "sha256" / target_hash
+            with open(manifest_path, "r") as ip:
+                ocilayout_manifests[target_hash] = OCIImageManifest.model_validate_json(ip.read())
+    if ocilayout_root_index is None:
+        return ocilayout_manifests # return early
+    for m in ocilayout_root_index.manifests:
+        if m.mediaType == MediaTypes.manifest:
+            target_hash = m.digest.removeprefix("sha256:")
+            logger.debug("Lookup remainder manifest from ocilayout_root_index having target_hash %s", target_hash)
             manifest_path = ocilayout / "blobs" / "sha256" / target_hash
             with open(manifest_path, "r") as ip:
                 ocilayout_manifests[target_hash] = OCIImageManifest.model_validate_json(ip.read())
@@ -160,11 +170,6 @@ def crawl_ocilayout_indexes(ocilayout: Path, ocilayout_root_index: OCIImageIndex
             index_path = ocilayout / "blobs" / "sha256" / target_hash
             with open(index_path, "r") as ip:
                 ocilayout_indexes[target_hash] = OCIImageIndex.model_validate_json(ip.read())
-        else:
-            if len(ocilayout_indexes) == 0:
-                raise ValueError("TODO the root index has Image manifest")
-            else:
-                click.echo(f"Found Image Manifest {m.digest} in root index, TODO assuming these are referred through the other indexes")
     return ocilayout_indexes
 
 

tests/basic_test.py

@@ -37,7 +37,7 @@ def test_crawl_ocilayout_manifests():
     ocilayout3_path = Path(__file__).parent / "data" / "ocilayout3"
     ocilayout_root_index = read_ocilayout_root_index(ocilayout3_path)
     ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout3_path, ocilayout_root_index)
-    mut: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout3_path, ocilayout_indexes)
+    mut: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout3_path, ocilayout_indexes, ocilayout_root_index)
 
     assert len(mut.keys()) == 2
     assert "c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878" in mut.keys()
@@ -93,3 +93,33 @@ def test_oci_layers_on_top_with_remove(tmp_path: Path):
     for model in models:
         assert not model.exists()
     assert not modelcard.exists()
+
+
+def test_oci_layers_on_top_single_manifest(tmp_path: Path):
+    """check oci_layers_on_top with an oci-layout directory containing a single manifest
+    """
+    test_sample_model = sample_model_path()
+    test_ocilayout5 = test_data_path() / "ocilayout5"
+    target_ocilayout = tmp_path / "myocilayout"
+    shutil.copytree(test_ocilayout5, target_ocilayout)
+    target_model = tmp_path / "models"
+    shutil.copytree(test_sample_model, target_model)
+    print(os.listdir(target_model))
+
+    models = [
+        target_model / "model.joblib",
+        target_model / "hello.md"
+    ]
+    for model in models:
+        assert model.exists()
+    modelcard = target_model / "README.md"
+    assert modelcard.exists()
+
+    oci_layers_on_top(target_ocilayout, models, modelcard)
+
+    ocilayout_root_index: OCIImageIndex = read_ocilayout_root_index(target_ocilayout)
+    ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(target_ocilayout, ocilayout_root_index)
+    ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(target_ocilayout, ocilayout_indexes, ocilayout_root_index)
+    assert len(ocilayout_manifests) == 1
+    manifest0: OCIImageManifest = next(iter(ocilayout_manifests.values()))
+    assert len(manifest0.layers) == 1 + len(models) + 1 # original value (only 1 layer in original oci-layout) + now added model files + now added modelcarD

tests/data/ocilayout5/README.md

@@ -0,0 +1,4 @@
+example only manifests (not layer tarballs) with: `oras copy --platform linux/arm64 --to-oci-layout quay.io/mmortari/hello-world-wait:latest ./tests/data/ocilayout5:latest`
+
+Q: "why you didn't use skopeo for this case?"
+A: cannot supply a specific platform

tests/data/ocilayout5/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633

@@ -0,0 +1 @@
+{"created":"2024-11-29T10:50:43.850369849Z","architecture":"arm64","os":"linux","variant":"v8","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:f21ad18174949794e810922c8ada6ff8416aabab8ef3fd3bd144e47058359f52"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:50:43.850412432Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]}

tests/data/ocilayout5/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633","size":778},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1933e30a3373776d5c7155591a6dacbc205cf6a2665b6dced682c6d2ea7b000f","size":1949749}],"annotations":{"org.opencontainers.image.base.digest":"sha256:6ca1ac3927a17445a61188b4f91af0bfb1e0b16757b07ec9f556e9e1e0851b15","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}}

tests/data/ocilayout5/index.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878","size":731,"annotations":{"org.opencontainers.image.ref.name":"latest"},"platform":{"architecture":"arm64","os":"linux"}}]}

tests/data/ocilayout5/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}