From 88190758f0df11c4dd9857271bfd00cd24b8b258 Mon Sep 17 00:00:00 2001 From: BenB Date: Wed, 30 Oct 2024 11:17:13 +0100 Subject: [PATCH 1/4] feat: extend the write()-method of File to accept a rootnode and foldername --- src/viur/core/modules/file.py | 101 +++++++++++++++++++++++++--------- 1 file changed, 74 insertions(+), 27 deletions(-) diff --git a/src/viur/core/modules/file.py b/src/viur/core/modules/file.py index 7de504911..64d6522bc 100644 --- a/src/viur/core/modules/file.py +++ b/src/viur/core/modules/file.py @@ -29,7 +29,6 @@ from viur.core.skeleton import SkeletonInstance, skeletonByKind from viur.core.tasks import CallDeferred, DeleteEntitiesIter, PeriodicTask - # Globals for connectivity VALID_FILENAME_REGEX = re.compile( @@ -531,12 +530,12 @@ def create_internal_serving_url( # Append additional parameters if params := { - k: v for k, v in { - "download": download, - "filename": filename, - "options": options, - "size": size, - }.items() if v + k: v for k, v in { + "download": download, + "filename": filename, + "options": options, + "size": size, + }.items() if v }: serving_url += f"?{urlencode(params)}" @@ -708,9 +707,16 @@ def write( width: int = None, height: int = None, public: bool = False, + foldername: t.Optional[str | t.Iterable[str]] = None, + rootnode: t.Optional[db.Key] = None, ) -> db.Key: """ - Write a file from any buffer into the file module. + Write a file from any bytes-like object into the file module. + + If foldername and rootnode are both set, the file is added to the repository in that folder. + If only foldername is set, the file is added to the default repository in that folder. + If only rootnode is set, the file is added to that repository in the root folder. + If both are not set, the file is added without a path or repository. It will not be visible in admin. :param filename: Filename to be written. :param content: The file content to be written, as bytes-like object. @@ -718,35 +724,76 @@ def write( :param width: Optional width information for the file. :param height: Optional height information for the file. :param public: True if the file should be publicly accessible. + :param foldername: Optional folder the file should be written into. + :param rootnode: Optional root-node of the repository to add the file to :return: Returns the key of the file object written. This can be associated e.g. with a FileBone. """ - if not File.is_valid_filename(filename): + if not self.is_valid_filename(filename): raise ValueError(f"{filename=} is invalid") + # Check for foldername + if rootnode is None: + rootnode = self.ensureOwnModuleRootNode() + elif not foldername: + # if rootnode is set and foldername is not, save the file in the root of the rootnode + foldername = [] + + if foldername is not None: + foldernames = foldername + if isinstance(foldername, str): + foldernames = foldernames.replace('\\', '/') + foldernames = (i for i in foldernames.split('/')) + foldernames = (i for i in foldernames if i) + + parentrepokey = rootnode.key + parentfolderkey = rootnode.key + + for fname in foldernames: + currentfolder = (self.addSkel("node").all() + .filter("parentrepo", parentrepokey) + .filter("parententry", parentfolderkey) + .filter("name", fname) + .getSkel()) + if currentfolder: + parentfolderkey = currentfolder.key + else: + newfolderskel = self.addSkel("node") + newfolderskel["name"] = fname + newfolderskel["parentrepo"] = parentrepokey + newfolderskel["parententry"] = parentfolderkey + newfolderskel.write() + parentfolderkey = newfolderskel["key"] + + # Save the file into the folder. dl_key = utils.string.random() if public: dl_key += PUBLIC_DLKEY_SUFFIX # mark file as public - bucket = File.get_bucket(dl_key) + bucket = self.get_bucket(dl_key) blob = bucket.blob(f"{dl_key}/source/{filename}") blob.upload_from_file(io.BytesIO(content), content_type=mimetype) - skel = self.addSkel("leaf") - skel["name"] = filename - skel["size"] = blob.size - skel["mimetype"] = mimetype - skel["dlkey"] = dl_key - skel["weak"] = True - skel["public"] = public - skel["width"] = width - skel["height"] = height - skel["crc32c_checksum"] = base64.b64decode(blob.crc32c).hex() - skel["md5_checksum"] = base64.b64decode(blob.md5_hash).hex() - - skel.write() - return skel["key"] + fileskel = self.addSkel("leaf") + fileskel["name"] = filename + fileskel["size"] = blob.size + fileskel["mimetype"] = mimetype + fileskel["dlkey"] = dl_key + fileskel["weak"] = foldername is None + fileskel["public"] = public + fileskel["width"] = width + fileskel["height"] = height + fileskel["crc32c_checksum"] = base64.b64decode(blob.crc32c).hex() + fileskel["md5_checksum"] = base64.b64decode(blob.md5_hash).hex() + + if foldername is not None: + fileskel["parentrepo"] = parentrepokey + fileskel["parententry"] = parentfolderkey + fileskel["pending"] = False + + fileskel.write() + return fileskel["key"] def read( self, @@ -1210,7 +1257,7 @@ def inject_serving_url(self, skel: SkeletonInstance) -> None: """Inject the serving url for public image files into a FileSkel""" # try to create a servingurl for images if not conf.instance.is_dev_server and skel["public"] and skel["mimetype"] \ - and skel["mimetype"].startswith("image/") and not skel["serving_url"]: + and skel["mimetype"].startswith("image/") and not skel["serving_url"]: try: bucket = File.get_bucket(skel['dlkey']) @@ -1332,8 +1379,8 @@ def start_delete_pending_files(): def __getattr__(attr: str) -> object: if entry := { - # stuff prior viur-core < 3.7 - "GOOGLE_STORAGE_BUCKET": ("File.get_bucket()", _private_bucket), + # stuff prior viur-core < 3.7 + "GOOGLE_STORAGE_BUCKET": ("File.get_bucket()", _private_bucket), }.get(attr): msg = f"{attr} was replaced by {entry[0]}" warnings.warn(msg, DeprecationWarning, stacklevel=2) From d9f7d5edc85d247218651aa0bfe6d7b238354da3 Mon Sep 17 00:00:00 2001 From: BenB Date: Tue, 5 Nov 2024 11:31:06 +0100 Subject: [PATCH 2/4] refactor: files created with File.write are never pending code-formatting --- src/viur/core/modules/file.py | 128 +++++++++++++++++----------------- 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/src/viur/core/modules/file.py b/src/viur/core/modules/file.py index 64d6522bc..391241171 100644 --- a/src/viur/core/modules/file.py +++ b/src/viur/core/modules/file.py @@ -70,7 +70,7 @@ def importBlobFromViur2(dlKey, fileName): marker["error"] = "Failed URL-FETCH 1" db.Put(marker) return False - if importDataReq.status != 200: + if importDataReq.status!=200: marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey)) marker["success"] = False marker["error"] = "Failed URL-FETCH 2" @@ -79,7 +79,7 @@ def importBlobFromViur2(dlKey, fileName): importData = json.loads(importDataReq.read()) oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + importData["key"] srcBlob = storage.Blob(bucket=bucket, - name=conf.viur2import_blobsource["gsdir"] + "/" + importData["key"]) + name=conf.viur2import_blobsource["gsdir"] + "/" + importData["key"]) else: oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + dlKey srcBlob = storage.Blob(bucket=bucket, name=conf.viur2import_blobsource["gsdir"] + "/" + dlKey) @@ -211,7 +211,7 @@ def make_request(): sig = File.hmac_sign(data_str) datadump = json.dumps({"dataStr": data_str.decode('ASCII'), "sign": sig}) resp = requests.post(conf.file_thumbnailer_url, data=datadump, headers=headers, allow_redirects=False) - if resp.status_code != 200: # Error Handling + if resp.status_code!=200: # Error Handling match resp.status_code: case 302: # The problem is Google resposen 302 to an auth Site when the cloudfunction was not found @@ -258,7 +258,7 @@ def make_request(): fileName = File.sanitize_filename(data["name"]) blob = bucket.blob(f"""{fileSkel["dlkey"]}/derived/{fileName}""") uploadUrls[fileSkel["dlkey"] + fileName] = blob.create_resumable_upload_session(timeout=60, - content_type=data["mimeType"]) + content_type=data["mimeType"]) if not (url := getsignedurl()): return @@ -497,11 +497,11 @@ def hmac_verify(data: t.Any, signature: str) -> bool: @staticmethod def create_internal_serving_url( - serving_url: str, - size: int = 0, - filename: str = "", - options: str = "", - download: bool = False + serving_url: str, + size: int = 0, + filename: str = "", + options: str = "", + download: bool = False ) -> str: """ Helper function to generate an internal serving url (endpoint: /file/serve) from a Google serving url. @@ -543,11 +543,11 @@ def create_internal_serving_url( @staticmethod def create_download_url( - dlkey: str, - filename: str, - derived: bool = False, - expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1), - download_filename: t.Optional[str] = None + dlkey: str, + filename: str, + derived: bool = False, + expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1), + download_filename: t.Optional[str] = None ) -> str: """ Utility function that creates a signed download-url for the given folder/filename combination @@ -615,24 +615,24 @@ def parse_download_url(url) -> t.Optional[FilePath]: # Invalid path return None - if valid_until != "0" and datetime.strptime(valid_until, "%Y%m%d%H%M") < datetime.now(): + if valid_until!="0" and datetime.strptime(valid_until, "%Y%m%d%H%M") < datetime.now(): # Signature expired return None - if dlpath.count("/") != 3: + if dlpath.count("/")!=3: # Invalid path return None dlkey, derived, filename = dlpath.split("/", 3) - return FilePath(dlkey, derived != "source", filename) + return FilePath(dlkey, derived!="source", filename) @staticmethod def create_src_set( - file: t.Union["SkeletonInstance", dict, str], - expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1), - width: t.Optional[int] = None, - height: t.Optional[int] = None, - language: t.Optional[str] = None, + file: t.Union["SkeletonInstance", dict, str], + expires: t.Optional[datetime.timedelta | int] = datetime.timedelta(hours=1), + width: t.Optional[int] = None, + height: t.Optional[int] = None, + language: t.Optional[str] = None, ) -> str: """ Generates a string suitable for use as the srcset tag in html. This functionality provides the browser @@ -672,9 +672,9 @@ def create_src_set( from viur.core.skeleton import SkeletonInstance # avoid circular imports if not ( - isinstance(file, (SkeletonInstance, dict)) - and "dlkey" in file - and "derived" in file + isinstance(file, (SkeletonInstance, dict)) + and "dlkey" in file + and "derived" in file ): logging.error("Invalid file supplied") return "" @@ -700,15 +700,15 @@ def create_src_set( return ", ".join(src_set) def write( - self, - filename: str, - content: t.Any, - mimetype: str = "text/plain", - width: int = None, - height: int = None, - public: bool = False, - foldername: t.Optional[str | t.Iterable[str]] = None, - rootnode: t.Optional[db.Key] = None, + self, + filename: str, + content: t.Any, + mimetype: str = "text/plain", + width: int = None, + height: int = None, + public: bool = False, + foldername: t.Optional[str | t.Iterable[str]] = None, + rootnode: t.Optional[db.Key] = None, ) -> db.Key: """ Write a file from any bytes-like object into the file module. @@ -786,19 +786,19 @@ def write( fileskel["height"] = height fileskel["crc32c_checksum"] = base64.b64decode(blob.crc32c).hex() fileskel["md5_checksum"] = base64.b64decode(blob.md5_hash).hex() + fileskel["pending"] = False if foldername is not None: fileskel["parentrepo"] = parentrepokey fileskel["parententry"] = parentfolderkey - fileskel["pending"] = False fileskel.write() return fileskel["key"] def read( - self, - key: db.Key | int | str | None = None, - path: str | None = None, + self, + key: db.Key | int | str | None = None, + path: str | None = None, ) -> tuple[io.BytesIO, str]: """ Read a file from the Cloud Storage. @@ -848,14 +848,14 @@ def deleteRecursive(self, parentKey): @exposed @skey def getUploadURL( - self, - fileName: str, - mimeType: str, - size: t.Optional[int] = None, - node: t.Optional[str | db.Key] = None, - authData: t.Optional[str] = None, - authSig: t.Optional[str] = None, - public: bool = False, + self, + fileName: str, + mimeType: str, + size: t.Optional[int] = None, + node: t.Optional[str | db.Key] = None, + authData: t.Optional[str] = None, + authSig: t.Optional[str] = None, + public: bool = False, ): filename = fileName.strip() # VIUR4 FIXME: just for compatiblity of the parameter names @@ -865,9 +865,9 @@ def getUploadURL( # Validate the mimetype from the client seems legit mimetype = mimeType.strip().lower() if not ( - mimetype - and mimetype.count("/") == 1 - and all(ch in string.ascii_letters + string.digits + "/-.+" for ch in mimetype) + mimetype + and mimetype.count("/")==1 + and all(ch in string.ascii_letters + string.digits + "/-.+" for ch in mimetype) ): raise errors.UnprocessableEntity(f"Invalid mime-type {mimetype!r} provided") @@ -885,8 +885,8 @@ def getUploadURL( if authData["validMimeTypes"]: for validMimeType in authData["validMimeTypes"]: if ( - validMimeType == mimetype - or (validMimeType.endswith("*") and mimetype.startswith(validMimeType[:-1])) + validMimeType==mimetype + or (validMimeType.endswith("*") and mimetype.startswith(validMimeType[:-1])) ): break else: @@ -999,7 +999,7 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig if not self.hmac_verify(blobKey, sig): raise errors.Forbidden() - if validUntil != "0" and datetime.datetime.strptime(validUntil, "%Y%m%d%H%M") < datetime.datetime.now(): + if validUntil!="0" and datetime.datetime.strptime(validUntil, "%Y%m%d%H%M") < datetime.datetime.now(): blob = None else: blob = bucket.get_blob(dlPath) @@ -1029,7 +1029,7 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig response.headers["Content-Disposition"] = content_disposition return blob.download_as_bytes() - if validUntil == "0" or blobKey.endswith(PUBLIC_DLKEY_SUFFIX): # Its an indefinitely valid URL + if validUntil=="0" or blobKey.endswith(PUBLIC_DLKEY_SUFFIX): # Its an indefinitely valid URL if blob.size < 5 * 1024 * 1024: # Less than 5 MB - Serve directly and push it into the ede caches response = current.request.get().response response.headers["Content-Type"] = blob.content_type @@ -1078,13 +1078,13 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig @exposed def serve( - self, - host: str, - key: str, - size: t.Optional[int] = None, - filename: t.Optional[str] = None, - options: str = "", - download: bool = False, + self, + host: str, + key: str, + size: t.Optional[int] = None, + filename: t.Optional[str] = None, + options: str = "", + download: bool = False, ): """ Requests an image using the serving url to bypass direct Google requests. @@ -1151,7 +1151,7 @@ def serve( @skey(allow_empty=True) def add(self, skelType: SkelType, node: db.Key | int | str | None = None, *args, **kwargs): # We can't add files directly (they need to be uploaded - if skelType == "leaf": # We need to handle leafs separately here + if skelType=="leaf": # We need to handle leafs separately here targetKey = kwargs.get("key") skel = self.addSkel("leaf") @@ -1181,7 +1181,7 @@ def add(self, skelType: SkelType, node: db.Key | int | str | None = None, *args, bucket = File.get_bucket(skel["dlkey"]) blobs = list(bucket.list_blobs(prefix=f"""{skel["dlkey"]}/""")) - if len(blobs) != 1: + if len(blobs)!=1: logging.error("Invalid number of blobs in folder") logging.error(targetKey) raise errors.PreconditionFailed() @@ -1213,7 +1213,7 @@ def onEdit(self, skelType: SkelType, skel: SkeletonInstance): old_skel = self.editSkel(skelType) old_skel.setEntity(skel.dbEntity) - if old_skel["name"] == skel["name"]: # name not changed we can return + if old_skel["name"]==skel["name"]: # name not changed we can return return # Move Blob to new name @@ -1257,7 +1257,7 @@ def inject_serving_url(self, skel: SkeletonInstance) -> None: """Inject the serving url for public image files into a FileSkel""" # try to create a servingurl for images if not conf.instance.is_dev_server and skel["public"] and skel["mimetype"] \ - and skel["mimetype"].startswith("image/") and not skel["serving_url"]: + and skel["mimetype"].startswith("image/") and not skel["serving_url"]: try: bucket = File.get_bucket(skel['dlkey']) From 0a5485b71d4a7ec4eaadf798ad7f6a36c11638ab Mon Sep 17 00:00:00 2001 From: BenB Date: Tue, 5 Nov 2024 13:45:06 +0100 Subject: [PATCH 3/4] refactor: fix spaces around operators --- src/viur/core/modules/file.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/viur/core/modules/file.py b/src/viur/core/modules/file.py index 391241171..41ee4eb31 100644 --- a/src/viur/core/modules/file.py +++ b/src/viur/core/modules/file.py @@ -70,7 +70,7 @@ def importBlobFromViur2(dlKey, fileName): marker["error"] = "Failed URL-FETCH 1" db.Put(marker) return False - if importDataReq.status!=200: + if importDataReq.status != 200: marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey)) marker["success"] = False marker["error"] = "Failed URL-FETCH 2" @@ -211,7 +211,7 @@ def make_request(): sig = File.hmac_sign(data_str) datadump = json.dumps({"dataStr": data_str.decode('ASCII'), "sign": sig}) resp = requests.post(conf.file_thumbnailer_url, data=datadump, headers=headers, allow_redirects=False) - if resp.status_code!=200: # Error Handling + if resp.status_code != 200: # Error Handling match resp.status_code: case 302: # The problem is Google resposen 302 to an auth Site when the cloudfunction was not found @@ -615,16 +615,16 @@ def parse_download_url(url) -> t.Optional[FilePath]: # Invalid path return None - if valid_until!="0" and datetime.strptime(valid_until, "%Y%m%d%H%M") < datetime.now(): + if valid_until != "0" and datetime.strptime(valid_until, "%Y%m%d%H%M") < datetime.now(): # Signature expired return None - if dlpath.count("/")!=3: + if dlpath.count("/") != 3: # Invalid path return None dlkey, derived, filename = dlpath.split("/", 3) - return FilePath(dlkey, derived!="source", filename) + return FilePath(dlkey, derived != "source", filename) @staticmethod def create_src_set( @@ -866,7 +866,7 @@ def getUploadURL( mimetype = mimeType.strip().lower() if not ( mimetype - and mimetype.count("/")==1 + and mimetype.count("/") == 1 and all(ch in string.ascii_letters + string.digits + "/-.+" for ch in mimetype) ): raise errors.UnprocessableEntity(f"Invalid mime-type {mimetype!r} provided") @@ -885,7 +885,7 @@ def getUploadURL( if authData["validMimeTypes"]: for validMimeType in authData["validMimeTypes"]: if ( - validMimeType==mimetype + validMimeType == mimetype or (validMimeType.endswith("*") and mimetype.startswith(validMimeType[:-1])) ): break @@ -999,7 +999,7 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig if not self.hmac_verify(blobKey, sig): raise errors.Forbidden() - if validUntil!="0" and datetime.datetime.strptime(validUntil, "%Y%m%d%H%M") < datetime.datetime.now(): + if validUntil != "0" and datetime.datetime.strptime(validUntil, "%Y%m%d%H%M") < datetime.datetime.now(): blob = None else: blob = bucket.get_blob(dlPath) @@ -1029,7 +1029,7 @@ def download(self, blobKey: str, fileName: str = "", download: bool = False, sig response.headers["Content-Disposition"] = content_disposition return blob.download_as_bytes() - if validUntil=="0" or blobKey.endswith(PUBLIC_DLKEY_SUFFIX): # Its an indefinitely valid URL + if validUntil == "0" or blobKey.endswith(PUBLIC_DLKEY_SUFFIX): # Its an indefinitely valid URL if blob.size < 5 * 1024 * 1024: # Less than 5 MB - Serve directly and push it into the ede caches response = current.request.get().response response.headers["Content-Type"] = blob.content_type @@ -1151,7 +1151,7 @@ def serve( @skey(allow_empty=True) def add(self, skelType: SkelType, node: db.Key | int | str | None = None, *args, **kwargs): # We can't add files directly (they need to be uploaded - if skelType=="leaf": # We need to handle leafs separately here + if skelType == "leaf": # We need to handle leafs separately here targetKey = kwargs.get("key") skel = self.addSkel("leaf") @@ -1181,7 +1181,7 @@ def add(self, skelType: SkelType, node: db.Key | int | str | None = None, *args, bucket = File.get_bucket(skel["dlkey"]) blobs = list(bucket.list_blobs(prefix=f"""{skel["dlkey"]}/""")) - if len(blobs)!=1: + if len(blobs) != 1: logging.error("Invalid number of blobs in folder") logging.error(targetKey) raise errors.PreconditionFailed() @@ -1213,7 +1213,7 @@ def onEdit(self, skelType: SkelType, skel: SkeletonInstance): old_skel = self.editSkel(skelType) old_skel.setEntity(skel.dbEntity) - if old_skel["name"]==skel["name"]: # name not changed we can return + if old_skel["name"] == skel["name"]: # name not changed we can return return # Move Blob to new name From 1182667e2855cbd49985d3fb961be23bcd741afc Mon Sep 17 00:00:00 2001 From: BenB Date: Tue, 5 Nov 2024 13:50:35 +0100 Subject: [PATCH 4/4] refactor: fix parameter alignment --- src/viur/core/modules/file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/viur/core/modules/file.py b/src/viur/core/modules/file.py index 41ee4eb31..2f09ddf67 100644 --- a/src/viur/core/modules/file.py +++ b/src/viur/core/modules/file.py @@ -79,7 +79,7 @@ def importBlobFromViur2(dlKey, fileName): importData = json.loads(importDataReq.read()) oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + importData["key"] srcBlob = storage.Blob(bucket=bucket, - name=conf.viur2import_blobsource["gsdir"] + "/" + importData["key"]) + name=conf.viur2import_blobsource["gsdir"] + "/" + importData["key"]) else: oldBlobName = conf.viur2import_blobsource["gsdir"] + "/" + dlKey srcBlob = storage.Blob(bucket=bucket, name=conf.viur2import_blobsource["gsdir"] + "/" + dlKey) @@ -258,7 +258,7 @@ def make_request(): fileName = File.sanitize_filename(data["name"]) blob = bucket.blob(f"""{fileSkel["dlkey"]}/derived/{fileName}""") uploadUrls[fileSkel["dlkey"] + fileName] = blob.create_resumable_upload_session(timeout=60, - content_type=data["mimeType"]) + content_type=data["mimeType"]) if not (url := getsignedurl()): return