From 8d547791b1c49bcb76e3357cf2314c21c9fac463 Mon Sep 17 00:00:00 2001 From: Jaideep Rao Date: Wed, 8 Jan 2025 02:00:51 +0530 Subject: [PATCH] create empty config entry Signed-off-by: Jaideep Rao --- olot/oci/oci_common.py | 1 + olot/oci/oci_image_manifest.py | 33 +++++++++------- olot/oci_artifact.py | 24 ++++++------ ...6d6c1e7401cc0128c21439456eaf964180b00dfda1 | Bin 0 -> 10240 bytes ...16f7e8649e94fb4fc21fe77e8310c060f61caaff8a | 1 + ...72f78032ce593d73797845751debed90558638d5a1 | Bin 0 -> 2218 bytes ...310700749d8e630a68e4c91f42173ec597e137c470 | 37 ++++++++++++++++++ ...8b495381463f1f2302425a96033dc51a81e17cb0bb | Bin 0 -> 10240 bytes tests/data/sample-model/oci/index.json | 11 ++++++ tests/data/sample-model/oci/oci-layout | 3 ++ 10 files changed, 83 insertions(+), 27 deletions(-) create mode 100644 tests/data/sample-model/oci/blobs/sha256/06cd6f2c31be8e4d5a1cea6d6c1e7401cc0128c21439456eaf964180b00dfda1 create mode 100644 tests/data/sample-model/oci/blobs/sha256/44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a create mode 100644 tests/data/sample-model/oci/blobs/sha256/69c637f302502a950e7a5072f78032ce593d73797845751debed90558638d5a1 create mode 100644 tests/data/sample-model/oci/blobs/sha256/70202a75d040e8a232a904310700749d8e630a68e4c91f42173ec597e137c470 create mode 100644 tests/data/sample-model/oci/blobs/sha256/80da85cc1eb84676a6adbc8b495381463f1f2302425a96033dc51a81e17cb0bb create mode 100644 tests/data/sample-model/oci/index.json create mode 100644 tests/data/sample-model/oci/oci-layout diff --git a/olot/oci/oci_common.py b/olot/oci/oci_common.py index 6a624a9..33f4d03 100644 --- a/olot/oci/oci_common.py +++ b/olot/oci/oci_common.py @@ -27,6 +27,7 @@ class MediaTypes: layer: MediaType = "application/vnd.oci.image.layer.v1.tar" layer_gzip: MediaType = "application/vnd.oci.image.layer.v1.tar+gzip" empty: MediaType = "application/vnd.oci.empty.v1+json" + config: MediaType = "application/vnd.oci.image.config.v1+json" Digest = Annotated[str, Field( ..., diff --git a/olot/oci/oci_image_manifest.py b/olot/oci/oci_image_manifest.py index 525009d..7b36bae 100644 --- a/olot/oci/oci_image_manifest.py +++ b/olot/oci/oci_image_manifest.py @@ -4,14 +4,14 @@ from __future__ import annotations -from typing import Annotated, List, Optional +from typing import Annotated, List, Optional, Dict import os import subprocess from pathlib import Path from pydantic import BaseModel, Field -from olot.oci.oci_common import Digest, Urls, Keys, Values, MediaTypes, MediaType +from olot.oci.oci_common import Urls, Keys, Values, MediaTypes, MediaType from olot.utils.types import Int64, Base64, Annotations from olot.utils.files import MIMETypes @@ -96,24 +96,27 @@ class ContentDescriptor(BaseModel): mediaType: MediaType = Field( - ..., description='the mediatype of the referenced object' + ..., description="The media type of the referenced object" ) - size: Int64 = Field(..., description='the size in bytes of the referenced object') - digest: Digest = Field( - ..., - description="the cryptographic checksum digest of the object, in the pattern ':'", + size: Int64 = Field( + ..., description="The size in bytes of the referenced object" + ) + digest: str = Field( + ..., description="The cryptographic checksum digest of the object, in the pattern ':'" ) urls: Optional[Urls] = Field( - None, description='a list of urls from which this object may be downloaded' + None, description="A list of URLs from which this object may be downloaded" ) data: Optional[Base64] = Field( - None, description='an embedding of the targeted content (base64 encoded)' + None, description="An embedding of the targeted content (base64 encoded)" ) artifactType: Optional[MediaType] = Field( - None, description='the IANA media type of this artifact' + None, description="The IANA media type of this artifact" ) - annotations: Optional[Annotations] = None + annotations: Optional[Dict[str, str]] = None + class Config: + exclude_none = True class OCIImageManifest(BaseModel): schemaVersion: Annotated[int, Field(ge=2, le=2)] = Field( @@ -193,12 +196,12 @@ def create_manifest_layers(files: List[Path], blob_layers: dict) -> List[Content mediaType=get_file_media_type(file), size=os.stat(file).st_size, digest=f"sha256:{file_digest}", - urls=None, - data=None, - artifactType=None, annotations= { Keys.image_title_annotation: os.path.basename(file) - } + }, + urls = None, + data = None, + artifactType = None, ) layers.append(layer) return layers \ No newline at end of file diff --git a/olot/oci_artifact.py b/olot/oci_artifact.py index 709dded..5d810c2 100644 --- a/olot/oci_artifact.py +++ b/olot/oci_artifact.py @@ -1,13 +1,12 @@ from pathlib import Path import os -import datetime import json import argparse from typing import List from olot.oci.oci_image_manifest import create_oci_image_manifest, create_manifest_layers from olot.oci.oci_image_layout import create_ocilayout -from olot.oci.oci_common import Keys, MediaTypes +from olot.oci.oci_common import MediaTypes, Values from olot.oci.oci_image_index import Manifest, create_oci_image_index from olot.utils.files import MIMETypes, tarball_from_file, targz_from_file from olot.utils.types import compute_hash_of_str @@ -39,16 +38,12 @@ def create_oci_artifact_from_model(source_dir: Path, dest_dir: Path): # Create the OCI image manifest manifest_layers = create_manifest_layers(model_files, layers) - annotations = { - Keys.image_created_annotation: datetime.datetime.now().isoformat() - } artifactType = MIMETypes.mlmodel manifest = create_oci_image_manifest( artifactType=artifactType, layers=manifest_layers, - annotations=annotations ) - manifest_json = json.dumps(manifest.model_dump(), indent=4, sort_keys=True) + manifest_json = json.dumps(manifest.dict(exclude_none=True), indent=4, sort_keys=True) manifest_SHA = compute_hash_of_str(manifest_json) with open(sha256_path / manifest_SHA, "w") as f: f.write(manifest_json) @@ -60,22 +55,27 @@ def create_oci_artifact_from_model(source_dir: Path, dest_dir: Path): mediaType=MediaTypes.manifest, size=os.stat(sha256_path / manifest_SHA).st_size, digest=f"sha256:{manifest_SHA}", - urls=None, - platform=None, - annotations=None + urls = None, ) ] ) - index_json = json.dumps(index.model_dump(), indent=4, sort_keys=True) + index_json = json.dumps(index.dict(exclude_none=True), indent=4, sort_keys=True) with open(dest_dir / "index.json", "w") as f: f.write(index_json) - # Create the OCI-layout file oci_layout = create_ocilayout() with open(dest_dir / "oci-layout", "w") as f: f.write(json.dumps(oci_layout.model_dump(), indent=4, sort_keys=True)) + # Create empty config file with digest as name + empty_config: dict[str, str] = {} + empty_digest_split = Values.empty_digest.split(":") + if len(empty_digest_split) == 2: + with open(dest_dir / "blobs" / "sha256" / empty_digest_split[1], "w") as f: + f.write(json.dumps(empty_config)) + else: + raise ValueError(f"Invalid empty_digest format: {Values.empty_digest}") def create_blobs(model_files: List[Path], dest_dir: Path): """ diff --git a/tests/data/sample-model/oci/blobs/sha256/06cd6f2c31be8e4d5a1cea6d6c1e7401cc0128c21439456eaf964180b00dfda1 b/tests/data/sample-model/oci/blobs/sha256/06cd6f2c31be8e4d5a1cea6d6c1e7401cc0128c21439456eaf964180b00dfda1 new file mode 100644 index 0000000000000000000000000000000000000000..48da1bba55e4c0bb5cfdaa4931a1c65af364154a GIT binary patch literal 10240 zcmeIy%W8x$6b4||eTvg%K{U5Xq)=#gv+NDJ4kY5V#z1uF%-dHfb}`V|S(y4~bJT>G zeBn<~7Ulh;J#4*oK4ptE_T92yDq~X3ebbPPaVEIP=&;CYPA1*=gH7Y_Pd=~ZRa6v) zUGR6@NTaH1BUK^wMhj`K)?%>ceTL|K+n0@R+gJp*2*ng?t$#8;XML`WbCz*slr%;O zp~w6SJr??tOwRd#Pd1+0{di5(^|=9;*|(hzKBu>sT6ayajqN^qdgxs8pE*6p)^;wZ zm!o$yKGD?GJ-tZTb<0J|}4cdXZzUN-jmP?@NAvCU`9Lv#&Cy zemwL!o$l?o-)Vk#I@_H+WA|T2oYheo%Qe`43toT0F?-2!D|7MB-NU`Z{g*Fa?Cp>D z4qm*NO=p8YTehe1^+&E^k(tTW>Cwr>={S$S1`kT=gM#aDVq1{l9m3xIdd6 zApd9khlkJDH_6E+|37H1{%9Yz&F9}jpl>@yqtPIfktmFK&3MV9RE%ch>A>>DydI30 zxaWD934Du1t_HMy@SI&x)Uac&V>VZs{YU5pyOy+(oz@2iH>orX|31LNj58}R&8=WA z;#h>UV~}cSoe0ZiW@x6JGLe?0m8uvF{_(fbh+S%RD`Np_nJN;YnW^%egSVQqyOdjI ziKk~E&6tfw{~8Q_`Nhu!aZ8W=3U%hZ%Iwy6-wm17mDpy#{WgG_J3A-hR%EIaF$)*3 zcXk*E#PyUm-IMPwlosqw0Tblhk^M=CSj6uelAuTgf2fMsQDg6vW6!hqHQ~8i3b53J z?sdxb19XjB>Pgp|-!4i~Q}gzti9Y1cd685+5nE=PZT)n9xm{Du>iJ$Y>yWJ}`Cq+! zBMV`x0%n|RmDjPeVa|{n5%{%7^613#wW@R^Oo#efSld((CmBJZ;-hh?MRSgE9nB>3 zl_-^wR_P`@H75Eff5{Qjo7$Fcx(B@zK*T0F0*)SLV?P^VY6~63u@=U#Pn9rq!Z5BC zQirALt~C*%jV!Ei*=7|B1*?pRhnS4ADwaj!_JXf4Lhvwi>&R)qM@xi*tW6cd34kvm z>oaWK59HPpsj*IS)?zxyGngZ#9p9jOe8hXP)m>q%7Ca}>H^cD@v`9shxAq8`3z-Y*F4(O1oGR*xX*H^S|eKBctzLQzT z3T4@y(jVEKz%K~YIs_gK)I2c6yr%~2d&$j^T}kuNctOr(F0DIZ9&k)9cws5fBURjr z1q+o;QD>vHkP-KRY!uXMr?Qr2D}4ygfhNX5XT6sfuHfH1M*==dZoPWSB?%@?N!f*M z4i}wyNJ?*=FVbm^Zbze^UEXgwi}`eNRNFw@MhR6_788i%oO+HsRMZA3w09G6LA21x zuI0LLGG@+pvSYi#+0G@+ywcdI*DWw!i;XivImLbymR+M;B}j5MA}+4B3x6J9Q7Caeebdf-m>2;3XZrNgn67OtJ;maoKtH!D>*=8%Cl7|2Q89vOjknH3uIZG<&E19!SlvMfz>No?avuYi`9&#+YRTbV7IJh)lL z{Wa1<&hfPl{bNO?&*A{suUXmQS(AYdo~31;iMXwkamA2jBT`%Lo*21|>aq1DtS*E|=nyYX$$a#f2&=1uRn^n;&$BxgnjpO<^HA)A-G2Ph2phzaNt zNrMAO@%e+Zg(^lHD~`$oD=*gxb!-c7iWrb`E&nI*aRXI8(TQbl zC`rr84t@Sl1YstK4jFEE(Z^DuE;9pWOwN~KFl53;ZHOBuQ$V{+V4Xxg@Zt29XBF?< z;#QRKm$J~+T*1V0FKK60q%se$OI;oM3IhpDrxUxOwy*}9A^U+xXpRm0JG&58%g8k0 z)!FsMkVN)DyKN-e#ZLBnQoWP^c_ZXf4nYm4_8^XHY>{kO1d_(m@FYQ`bEinSfXBwk z7*4LjQk7NKR3Hts%aIsnQ{;JVonBHe4f+|WJEG=(8wwGLSvTRa04y&WWbfX_-SvI3 zDQZKeUn}B*#L=q*rk~2!{*9qaC}`dt9(qPt-udV?L>8XGMmAqH=u4Lt4OFl^l$o>( z-)FA&-HCF=079?6YKRVpKG|8^O05c47c=%IFA*J$cNI+u^vcEs)s`$+0!VfZ&hgus z__XbEvr{M0&>8qy`!J-ij8ZDSP&6V}l5Q137%-G5r_Gq1;Z-Nf)H zQkt%Ah#aRlax#QAtymE+s7a%c$z4USOmowFcflj1l3?e8YganFf;f{!lq^cW-%715 z&Fje|&muJ&Cy34}97{EkZtHV2W`I%nwWM2PVcnI6y*uZ)$yl&Oi-9I@Yc}0|wcQfq z`d;3Sl};u+G?U#|yVLRX)!}ryJ@&Qi49Mad%A59;C9UUrKeU{wO1qm(+)$(mx{$Z9TmO}JgE4}S*2&-7|+A855! z61jceInszVTuUsvCYKj)!w*#CZbc_s=}yxR&lH`_*CJRL@3c0Mb+Fc>6-dSAFmwO*Q`xAedDfOblF1BP(NY;Lc>fYRKXq0Y86*e z0}sw6L-+OQ^2qt@dl>?JRyryV!_+_fHUYo|0I;1VrY>ANJ2-tK{)_1IZmA^=LQnLu z%I@qystsbi?9*_tA2akYRI)y2wzmKCFJ-9bLg&8oYM|0Z{+i<6E(E}-$V!l*zaayG z3i`1AA+4!juwv)se_6zmMy?(@ZERb+lH^zqx~}ZSWBHekqspe#IBjsD#3ro*Uhnw- s_|QepRpC|Qk)M7n@jv?yOi$x!JdLOEG@i!O_{)vo0d1vS6#ysz0ON&J$^ZZW literal 0 HcmV?d00001 diff --git a/tests/data/sample-model/oci/blobs/sha256/70202a75d040e8a232a904310700749d8e630a68e4c91f42173ec597e137c470 b/tests/data/sample-model/oci/blobs/sha256/70202a75d040e8a232a904310700749d8e630a68e4c91f42173ec597e137c470 new file mode 100644 index 0000000..13376f8 --- /dev/null +++ b/tests/data/sample-model/oci/blobs/sha256/70202a75d040e8a232a904310700749d8e630a68e4c91f42173ec597e137c470 @@ -0,0 +1,37 @@ +{ + "artifactType": "application/x-mlmodel", + "config": { + "data": "e30=", + "digest": "sha256:44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a", + "mediaType": "application/vnd.oci.empty.v1+json", + "size": 2 + }, + "layers": [ + { + "annotations": { + "org.opencontainers.image.title": "model.joblib" + }, + "digest": "sha256:80da85cc1eb84676a6adbc8b495381463f1f2302425a96033dc51a81e17cb0bb", + "mediaType": "application/octet-stream", + "size": 3299 + }, + { + "annotations": { + "org.opencontainers.image.title": "README.md" + }, + "digest": "sha256:69c637f302502a950e7a5072f78032ce593d73797845751debed90558638d5a1", + "mediaType": "text/plain", + "size": 6625 + }, + { + "annotations": { + "org.opencontainers.image.title": "hello.md" + }, + "digest": "sha256:06cd6f2c31be8e4d5a1cea6d6c1e7401cc0128c21439456eaf964180b00dfda1", + "mediaType": "text/plain", + "size": 72 + } + ], + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "schemaVersion": 2 +} \ No newline at end of file diff --git a/tests/data/sample-model/oci/blobs/sha256/80da85cc1eb84676a6adbc8b495381463f1f2302425a96033dc51a81e17cb0bb b/tests/data/sample-model/oci/blobs/sha256/80da85cc1eb84676a6adbc8b495381463f1f2302425a96033dc51a81e17cb0bb new file mode 100644 index 0000000000000000000000000000000000000000..4ee65f9bfb60a62fb23c41c5f91bcb05ad9f743d GIT binary patch literal 10240 zcmeHKZ)_Ar6yILk>-CEK>tS0+2oRA9^tkI?drj2bm48J~+WIGG^uxM$H`iUdw|ngF zm6k*UFyqUT1hVX{epfI`(2|1*3_oYN=;by5LkyzXwGgY`qG}0OE;*qZ4T;X6Y$5~KwNY*!4 zRd}bGk)_N%r^8I@?rtZ??OeC(?pQp=MdOKhERl$Jc6LMl$GRer{|_W5?fl=V3F_S< zhhrk2znjAUb8K&2IqIXX!^E)5{7O`I3?~%nw}@iU{=75slzrMnl3dZ%8opAodR2TMxLIAg6C<1l2Z#jYzG6Gs=QtlG)r|pP=W-bq#>P`6rQB(@Iy0!d4ij1 zhJ}?@U#t<$K7%@i)nHq#dZQd4lx!_gf)ZKWWY&85|U3S>4UU zq7;FoWzL7cxt)CHm|4C~(sqdT&#xH2P%$24 zI5*k;_?gv<**stB>^Ek)o}~89v2h#U$Mk|JD=u!5;kPo}1~%WuyTNeUn0{S~`rXXh zQ%P!XDVrB?=2QGB7Uwc&ymCHmw=mpB=0`Krzs2S+VfF&7UhLSXc3Ro|dKO=Zwf}JZ zr#$;u|00He-SMB|*!UrbXBuBC!}VABM|pPnp*S|*w;BJf%-=xOd>Vh7!w;SRuoJ(H zPxb1V|1LlFIE~Nm#7}XiSo}7>E&-|;X-20*M z{d@baT)xEg-Tqe&uHSNa#TBRj^Dn>sY|YqDPMZQwPE^(>pZZf54qdA1zj#&6=1V_T z_0L#;Nr7uUi;C5UyL&UaXCqtxPhR@r^joKYPm1sN?mj%|OYImwbZvF>@>F>}3mN-) z^R3Z4`<`+!;k=c{-#D}Fs_)cqeZ!;E1NEmizq{_4>)U;&D)Bb{L%dJh&c`o=&QG)? zM~xH5&rPf!*i-JP{b-`hnB;I%UNWrbO9`%pG>02`xEJXYXVweaO@jsY#$|N?!uEq4 z1u1W~IS5+M65g~#(vo@C;3jwz-Ale>wyh@q-zHzvj2OK9GkKg$KCt{VA3A1%XRNj9 zst!yy`P^OC=&aZs`)xUeg7*&3{<`q+ayS2+z-u@f?g~dpDHCQ&Uga_1G2k)aG2k)a jG2k)aG2k)aG2k)aG2k)aG2k)aG2k)aG2k&Uw;1>f`ZP{w literal 0 HcmV?d00001 diff --git a/tests/data/sample-model/oci/index.json b/tests/data/sample-model/oci/index.json new file mode 100644 index 0000000..ef24c0a --- /dev/null +++ b/tests/data/sample-model/oci/index.json @@ -0,0 +1,11 @@ +{ + "manifests": [ + { + "digest": "sha256:70202a75d040e8a232a904310700749d8e630a68e4c91f42173ec597e137c470", + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "size": 1250 + } + ], + "mediaType": "application/vnd.oci.image.index.v1+json", + "schemaVersion": 2 +} \ No newline at end of file diff --git a/tests/data/sample-model/oci/oci-layout b/tests/data/sample-model/oci/oci-layout new file mode 100644 index 0000000..224a869 --- /dev/null +++ b/tests/data/sample-model/oci/oci-layout @@ -0,0 +1,3 @@ +{ + "imageLayoutVersion": "1.0.0" +} \ No newline at end of file