test: add tests for crawl functions

Signed-off-by: tarilabs <[email protected]>

olot/basics.py

@@ -40,14 +40,12 @@ def get_file_hash(path) -> str:
 
 def oci_layers_on_top(ocilayout: Path, model_files):
     check_ocilayout(ocilayout)
+    ocilayout_root_index = read_ocilayout_root_index(ocilayout)
     new_layers = []
     for model in model_files:
         model = Path(model)
         new_layer = tar_into_ocilayout(ocilayout, model)
         new_layers.append(new_layer)
-    ocilayout_root_index = None
-    with open(ocilayout / "index.json", "r") as f:
-        ocilayout_root_index = OCIImageIndex.model_validate_json(f.read())
     ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout, ocilayout_root_index)
     ocilayout_manifests: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout, ocilayout_indexes)
     new_ocilayout_manifests: Dict[str, str] = {}
@@ -188,5 +186,12 @@ def tar_filter_fn(input: tarfile.TarInfo) -> tarfile.TarInfo :
     return input
 
 
+def read_ocilayout_root_index(ocilayout: Path) -> OCIImageIndex:
+    ocilayout_root_index = None
+    with open(ocilayout / "index.json", "r") as f:
+        ocilayout_root_index = OCIImageIndex.model_validate_json(f.read())
+    return ocilayout_root_index
+
+
 if __name__ == "__main__":
     print("?")

tests/basic_test.py

@@ -2,8 +2,13 @@
 import os
 from pathlib import Path
 import tarfile
+from typing import Dict
 
-from olot.basics import HashingWriter, get_file_hash
+import pytest
+
+from olot.basics import HashingWriter, get_file_hash, check_ocilayout, read_ocilayout_root_index, crawl_ocilayout_indexes, crawl_ocilayout_manifests
+from olot.oci.oci_image_index import OCIImageIndex
+from olot.oci.oci_image_manifest import OCIImageManifest
 
 
 def test_get_file_hash():
@@ -81,3 +86,67 @@ def test_bespoke_single_file_gz(tmp_path):
     for file in tmp_path.rglob('*'):
         if file.is_file():
             print(file)
+
+
+def test_check_ocilayout():
+    """Verify check_ocilayout() fn on known oci-layout and not
+    """
+    data_path = Path(__file__).parent / "data"
+    check_ocilayout(data_path / "ocilayout1")
+    check_ocilayout(data_path / "ocilayout2")
+    check_ocilayout(data_path / "ocilayout3")
+    with pytest.raises(Exception):
+        check_ocilayout(data_path)
+
+
+def test_read_ocilayout_root_index():
+    """Read correctly the ocilayout_root_index in a given oci-layout
+    """
+    ocilayout3_path = Path(__file__).parent / "data" / "ocilayout3"
+    mut = read_ocilayout_root_index(ocilayout3_path)
+    assert mut.schemaVersion == 2
+    assert len(mut.manifests) == 3
+    manifest0 = mut.manifests[0]
+    assert manifest0.mediaType == "application/vnd.oci.image.index.v1+json"
+    assert manifest0.digest == "sha256:d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b"
+    assert manifest0.size == 491
+
+
+def test_crawl_ocilayout_indexes():
+    """Crawl for indexes models (the index content itself, not a manifest ref) in given oci-layout
+    """
+    ocilayout3_path = Path(__file__).parent / "data" / "ocilayout3"
+    mut: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout3_path, read_ocilayout_root_index(ocilayout3_path))
+    assert len(mut.keys()) == 1
+    assert "d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b" in mut.keys()
+    index0 = mut["d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b"]
+    assert index0.mediaType == "application/vnd.oci.image.index.v1+json"
+    assert len(index0.manifests) == 2
+
+    # I will redo the same fo ocilayout2 which is simplified from ocilayout3 as a sanity check
+    ocilayout2_path = Path(__file__).parent / "data" / "ocilayout2"
+    mut: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout2_path, read_ocilayout_root_index(ocilayout2_path))
+    assert len(mut.keys()) == 1
+    assert "d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b" in mut.keys()
+    index0 = mut["d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b"]
+    assert index0.mediaType == "application/vnd.oci.image.index.v1+json"
+    assert len(index0.manifests) == 2
+
+
+def test_crawl_ocilayout_manifests():
+    """Crawl for image manifest models (the image manifest content itself, not a manifest ref) in given oci-layout
+    """
+    ocilayout3_path = Path(__file__).parent / "data" / "ocilayout3"
+    ocilayout_root_index = read_ocilayout_root_index(ocilayout3_path)
+    ocilayout_indexes: Dict[str, OCIImageIndex] = crawl_ocilayout_indexes(ocilayout3_path, ocilayout_root_index)
+    mut: Dict[str, OCIImageManifest] = crawl_ocilayout_manifests(ocilayout3_path, ocilayout_indexes)
+
+    assert len(mut.keys()) == 2
+    assert "c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878" in mut.keys()
+    image0 = mut["c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878"]
+    assert image0.mediaType == "application/vnd.oci.image.manifest.v1+json"
+    assert len(image0.layers) == 1
+    layer0 = image0.layers[0]
+    assert layer0.digest == "sha256:1933e30a3373776d5c7155591a6dacbc205cf6a2665b6dced682c6d2ea7b000f"
+    assert layer0.size == 1949749
+    assert layer0.mediaType == "application/vnd.oci.image.layer.v1.tar+gzip"

tests/data/ocilayout2/README.md

@@ -0,0 +1 @@
+example only manifests (not layer tarballs) with skopeo copy --multi-arch all docker://quay.io/mmortari/hello-world-wait:latest oci:download:latest

tests/data/ocilayout2/blobs/sha256/4ac1d9b9fd75b80ef0763b0d17bff622b6b7483493c4406577b4d5c816e109d6

@@ -0,0 +1 @@
+{"created":"2024-11-29T10:57:59.359186315Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:dab88e68d7096681d624de8bd542df9d91bae06c739a73483b6fb0e6869421ea"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:57:59.359261356Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]}

tests/data/ocilayout2/blobs/sha256/5f6f1ff3a7dcccc23fc02beec84a75950c989e76a2debb58ec281f46c28a1159

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4ac1d9b9fd75b80ef0763b0d17bff622b6b7483493c4406577b4d5c816e109d6","size":763},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:efd59a45b2fb54508f226cbe5e9b86c0b8cbf3bc6ba9488cc68acc5c862d2d52","size":2280493}],"annotations":{"org.opencontainers.image.base.digest":"sha256:a3e1b257b47c09c9997212e53a0b570c1666501ad26e5bf33461304babab47c7","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}}

tests/data/ocilayout2/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633

@@ -0,0 +1 @@
+{"created":"2024-11-29T10:50:43.850369849Z","architecture":"arm64","os":"linux","variant":"v8","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:f21ad18174949794e810922c8ada6ff8416aabab8ef3fd3bd144e47058359f52"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:50:43.850412432Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]}

tests/data/ocilayout2/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633","size":778},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1933e30a3373776d5c7155591a6dacbc205cf6a2665b6dced682c6d2ea7b000f","size":1949749}],"annotations":{"org.opencontainers.image.base.digest":"sha256:6ca1ac3927a17445a61188b4f91af0bfb1e0b16757b07ec9f556e9e1e0851b15","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}}

tests/data/ocilayout2/blobs/sha256/d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878","size":731,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:5f6f1ff3a7dcccc23fc02beec84a75950c989e76a2debb58ec281f46c28a1159","size":731,"platform":{"architecture":"amd64","os":"linux"}}]}

tests/data/ocilayout2/index.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b","size":491,"annotations":{"org.opencontainers.image.ref.name":"latest"}}]}

tests/data/ocilayout2/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}

tests/data/ocilayout3/README.md

@@ -0,0 +1 @@
+example only manifests (not layer tarballs) with oras copy --to-oci-layout quay.io/mmortari/hello-world-wait:latest ./download:latest

tests/data/ocilayout3/blobs/sha256/4ac1d9b9fd75b80ef0763b0d17bff622b6b7483493c4406577b4d5c816e109d6

@@ -0,0 +1 @@
+{"created":"2024-11-29T10:57:59.359186315Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:dab88e68d7096681d624de8bd542df9d91bae06c739a73483b6fb0e6869421ea"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:57:59.359261356Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]}

tests/data/ocilayout3/blobs/sha256/5f6f1ff3a7dcccc23fc02beec84a75950c989e76a2debb58ec281f46c28a1159

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:4ac1d9b9fd75b80ef0763b0d17bff622b6b7483493c4406577b4d5c816e109d6","size":763},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:efd59a45b2fb54508f226cbe5e9b86c0b8cbf3bc6ba9488cc68acc5c862d2d52","size":2280493}],"annotations":{"org.opencontainers.image.base.digest":"sha256:a3e1b257b47c09c9997212e53a0b570c1666501ad26e5bf33461304babab47c7","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}}

tests/data/ocilayout3/blobs/sha256/70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633

@@ -0,0 +1 @@
+{"created":"2024-11-29T10:50:43.850369849Z","architecture":"arm64","os":"linux","variant":"v8","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/sh","-c","echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity"],"Labels":{"io.buildah.version":"1.37.1"}},"rootfs":{"type":"layers","diff_ids":["sha256:f21ad18174949794e810922c8ada6ff8416aabab8ef3fd3bd144e47058359f52"]},"history":[{"created":"2024-09-26T21:31:42Z","created_by":"BusyBox 1.37.0 (glibc), Debian 12"},{"created":"2024-11-29T10:50:43.850412432Z","created_by":"/bin/sh -c #(nop) CMD [\"/bin/sh\", \"-c\", \"echo 'Hello, World! and will wait forever' \u0026\u0026 sleep infinity\"]","comment":"FROM docker.io/library/busybox:latest","empty_layer":true}]}

tests/data/ocilayout3/blobs/sha256/c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.manifest.v1+json","config":{"mediaType":"application/vnd.oci.image.config.v1+json","digest":"sha256:70378ac6bb0f7e7e39ce50fff837162f5fd38c4e8daf8de1a120df9a7d79d633","size":778},"layers":[{"mediaType":"application/vnd.oci.image.layer.v1.tar+gzip","digest":"sha256:1933e30a3373776d5c7155591a6dacbc205cf6a2665b6dced682c6d2ea7b000f","size":1949749}],"annotations":{"org.opencontainers.image.base.digest":"sha256:6ca1ac3927a17445a61188b4f91af0bfb1e0b16757b07ec9f556e9e1e0851b15","org.opencontainers.image.base.name":"docker.io/library/busybox:latest","org.opencontainers.image.url":"https://github.com/docker-library/busybox","org.opencontainers.image.version":"1.37.0-glibc"}}

tests/data/ocilayout3/blobs/sha256/d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.oci.image.index.v1+json","manifests":[{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878","size":731,"platform":{"architecture":"arm64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:5f6f1ff3a7dcccc23fc02beec84a75950c989e76a2debb58ec281f46c28a1159","size":731,"platform":{"architecture":"amd64","os":"linux"}}]}

tests/data/ocilayout3/index.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"manifests":[{"mediaType":"application/vnd.oci.image.index.v1+json","digest":"sha256:d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b","size":491,"annotations":{"org.opencontainers.image.ref.name":"latest"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:5f6f1ff3a7dcccc23fc02beec84a75950c989e76a2debb58ec281f46c28a1159","size":731,"platform":{"architecture":"amd64","os":"linux"}},{"mediaType":"application/vnd.oci.image.manifest.v1+json","digest":"sha256:c23ed8b7e30f5edd2417e1dd99fedad4445f3e835edb58760b2f83f2c0517878","size":731,"platform":{"architecture":"arm64","os":"linux"}}]}

tests/data/ocilayout3/oci-layout

@@ -0,0 +1 @@
+{"imageLayoutVersion":"1.0.0"}

tests/test_oci.py

@@ -12,6 +12,8 @@
 
 
 def test_non_regression():
+    """Given a known oci-layout, smoke test deserialization of OCI "models" 
+    """
     base_path = Path(__file__).parent / "data" / "ocilayout1"
     with open(base_path / "oci-layout", "r") as f:
         m = OCIImageLayout.model_validate_json(f.read())
@@ -42,4 +44,4 @@ def test_non_regression():
         m = OCIManifestConfig.model_validate_json(f.read())
         print(m)
         print(m.rootfs)
-        print(m.history)    
+        print(m.history)