From 1efd485368628b7e125f4fefac5102b60f4828b9 Mon Sep 17 00:00:00 2001 From: Matteo Mortari Date: Thu, 19 Dec 2024 10:37:44 +0100 Subject: [PATCH] tests: end-to-end with Oras CP (#3) Signed-off-by: tarilabs --- .github/workflows/e2e.yaml | 29 +++++++++++++ Makefile | 4 ++ olot/backend/oras_cp.py | 24 +++++++++++ pyproject.toml | 1 + tests/backend/test_oras_cp.py | 79 +++++++++++++++++++++++++++++++++++ tests/backend/test_skopeo.py | 21 +++++++++- tests/conftest.py | 21 +++++++--- 7 files changed, 172 insertions(+), 7 deletions(-) create mode 100644 olot/backend/oras_cp.py create mode 100644 tests/backend/test_oras_cp.py diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f76a646..791e5a7 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -37,3 +37,32 @@ jobs: - name: Run E2E tests run: | make test-e2e-skopeo + e2e-oras: + name: E2E using Oras CP + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Install oras + uses: oras-project/setup-oras@v1 + - run: oras version + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + - name: Install Poetry + run: | + pipx install poetry + - name: Install dependencies + run: | + make install + - name: Start Kind Cluster + uses: helm/kind-action@v1 + with: + cluster_name: kind + - name: Start distribution-registry + run: | + ./e2e/deploy_distribution_registry.sh + - name: Run E2E tests + run: | + make test-e2e-oras diff --git a/Makefile b/Makefile index 8f1475f..78568da 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,10 @@ test: test-e2e-skopeo: poetry run pytest --e2e-skopeo -s -x -rA +.PHONY: test-e2e-oras +test-e2e-oras: + poetry run pytest --e2e-oras -s -x -rA + .PHONY: lint lint: install poetry run ruff check --fix diff --git a/olot/backend/oras_cp.py b/olot/backend/oras_cp.py new file mode 100644 index 0000000..191e8a4 --- /dev/null +++ b/olot/backend/oras_cp.py @@ -0,0 +1,24 @@ +import os +import shutil +import subprocess +import typing + +def is_oras() -> bool : + return shutil.which("oras") is not None + + +def oras_pull(base_image: str, dest: typing.Union[str, os.PathLike]): + if isinstance(dest, os.PathLike): + dest = str(dest) + subprocess.run(["oras", "copy", "--to-oci-layout", base_image, dest+":latest"], check=True) + blobs_dir = os.path.join(dest, "blobs", "sha256") + for _, _, files in os.walk(blobs_dir): + for file in files: + os.chmod(os.path.join(blobs_dir, file), 0o664) # TODO eventually avoid this by refactor manifest change logic + + + +def oras_push(src: typing.Union[str, os.PathLike], oci_ref: str): + if isinstance(src, os.PathLike): + src = str(src) + return subprocess.run(["oras", "copy", "--from-oci-layout", "--to-plain-http", src+":latest", oci_ref], check=True) diff --git a/pyproject.toml b/pyproject.toml index db87330..27d8114 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ pytest = "^8.3.4" [tool.pytest.ini_options] markers = [ "e2e_skopeo: end-to-end testing with skopeo", + "e2e_oras: end-to-end testing with oras", ] diff --git a/tests/backend/test_oras_cp.py b/tests/backend/test_oras_cp.py new file mode 100644 index 0000000..5486a4d --- /dev/null +++ b/tests/backend/test_oras_cp.py @@ -0,0 +1,79 @@ +import os +import subprocess +import time +import docker # type: ignore +from pathlib import Path +import pytest +from olot.backend.oras_cp import is_oras, oras_pull, oras_push +from olot.basics import check_ocilayout, oci_layers_on_top, read_ocilayout_root_index + +@pytest.mark.e2e_oras +def test_is_oras(): + assert is_oras() + + +@pytest.mark.e2e_oras +def test_oras_pull(tmp_path): + """Test oras to pull/dl a known base-image to an oci-layout + """ + oras_pull("quay.io/mmortari/hello-world-wait:latest", tmp_path) + + assert check_ocilayout(tmp_path) + + mut = read_ocilayout_root_index(tmp_path) + assert mut.schemaVersion == 2 + assert len(mut.manifests) == 3 + manifest0 = mut.manifests[0] + assert manifest0.mediaType == "application/vnd.oci.image.index.v1+json" + assert manifest0.digest == "sha256:d437889e826ecce2116ac711469bd09b1bb3c64d45055cbf23a6f8f3db223b8b" + assert manifest0.size == 491 + + +@pytest.mark.e2e_oras +def test_oras_scenario(tmp_path): + """Test oras with an end-to-end scenario + """ + oras_pull("quay.io/mmortari/hello-world-wait:latest", tmp_path) + model_joblib = Path(__file__).parent / ".." / "data" / "model.joblib" + model_files = [ + model_joblib, + Path(__file__).parent / ".." / "data" / "hello.md", + ] + oci_layers_on_top(tmp_path, model_files) + oras_push(tmp_path, "localhost:5001/nstestorg/modelcar:latest") + + # show what has been copied in Container Registry + subprocess.run(["skopeo","list-tags","--tls-verify=false","docker://localhost:5001/nstestorg/modelcar"], check=True) + + # copy from Container Registry to Docker daemon for local running the modelcar as-is + result = subprocess.run("skopeo inspect --tls-verify=false --raw docker://localhost:5001/nstestorg/modelcar | jq -r '.manifests[] | select(.platform.architecture == \"amd64\") | .digest'", shell=True, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + assert result.returncode == 0 + digest = result.stdout.strip() + print(digest) + # use by convention the linux/amd64 + subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True) + client = docker.from_env() + container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True) + print(container.logs()) + _, stat = container.get_archive('/models/model.joblib') + print(str(stat["size"])) + # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size + assert stat["size"] == os.stat(model_joblib).st_size + container.kill() + max_attempts = 5 + attempt = 0 + while attempt < max_attempts: + try: + if client.containers.get(container.id): + container.kill() + time.sleep(2**attempt) + else: + break + except docker.errors.NotFound: + print("test container terminated") + except Exception as e: + print(f"Attempt to terminate {attempt + 1} failed: {e}") + attempt += 1 + if attempt == max_attempts: + print("Was unable to terminate the test container") + client.images.remove("localhost:5001/nstestorg/modelcar") diff --git a/tests/backend/test_skopeo.py b/tests/backend/test_skopeo.py index fdb2e54..391c8b7 100644 --- a/tests/backend/test_skopeo.py +++ b/tests/backend/test_skopeo.py @@ -1,5 +1,6 @@ import os import subprocess +import time import docker # type: ignore from pathlib import Path import pytest @@ -52,9 +53,27 @@ def test_skopeo_scenario(tmp_path): # use by convention the linux/amd64 subprocess.run(["skopeo", "copy", "--src-tls-verify=false", f"docker://localhost:5001/nstestorg/modelcar@{digest}", "docker-daemon:localhost:5001/nstestorg/modelcar:latest"], check=True) client = docker.from_env() - container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True) + container = client.containers.run("localhost:5001/nstestorg/modelcar", detach=True, remove=True) print(container.logs()) _, stat = container.get_archive('/models/model.joblib') print(str(stat["size"])) # assert the model.joblib from the KServe modelcar is in expected location (above) and expected size assert stat["size"] == os.stat(model_joblib).st_size + container.kill() + max_attempts = 5 + attempt = 0 + while attempt < max_attempts: + try: + if client.containers.get(container.id): + container.kill() + time.sleep(2**attempt) + else: + break + except docker.errors.NotFound: + print("test container terminated") + except Exception as e: + print(f"Attempt to terminate {attempt + 1} failed: {e}") + attempt += 1 + if attempt == max_attempts: + print("Was unable to terminate the test container") + client.images.remove("localhost:5001/nstestorg/modelcar") diff --git a/tests/conftest.py b/tests/conftest.py index 8eea23f..bf74803 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,10 +2,13 @@ def pytest_collection_modifyitems(config, items): - for item in items: + for item in items: skip_e2e_skopeo = pytest.mark.skip( reason="this is an end-to-end test, requires explicit opt-in --e2e-skopeo option to run." ) + skip_e2e_oras = pytest.mark.skip( + reason="this is an end-to-end test, requires explicit opt-in --e2e-oras option to run." + ) skip_not_e2e = pytest.mark.skip( reason="skipping non-e2e tests; opt-out of --e2e -like options to run." ) @@ -13,12 +16,12 @@ def pytest_collection_modifyitems(config, items): if not config.getoption("--e2e-skopeo"): item.add_marker(skip_e2e_skopeo) continue - # elif "e2e_model_registry" in item.keywords: - # if not config.getoption("--e2e-model-registry"): - # item.add_marker(skip_e2e_model_registry) - # continue + elif "e2e_oras" in item.keywords: + if not config.getoption("--e2e-oras"): + item.add_marker(skip_e2e_oras) + continue - if config.getoption("--e2e-skopeo"): # or config.getoption("--e2e-model-registry"): + if config.getoption("--e2e-skopeo") or config.getoption("--e2e-oras"): item.add_marker(skip_not_e2e) @@ -29,3 +32,9 @@ def pytest_addoption(parser): default=False, help="opt-in to run tests marked with e2e_skopeo", ) + parser.addoption( + "--e2e-oras", + action="store_true", + default=False, + help="opt-in to run tests marked with e2e_oras", + )