Merge pull request #1311 from keboola/zajca-ct-745-file-delete-async

CT-745 file delete async
keboola · Apr 4, 2024 · 141a7df · 141a7df
2 parents f6b348d + 8a625b5
commit 141a7df
Show file tree

Hide file tree

Showing 7 changed files with 214 additions and 81 deletions.
diff --git a/apiary.apib b/apiary.apib
@@ -6578,7 +6578,7 @@ And then, upload that manifest file to the key `exp-180/11/files/2014/02/25/1108
 [Developers documentation](https://developers.keboola.com/integrate/storage/api/import-export/#working-with-sliced-files)
 for a more detailed guide.
 
-## Upload File [/v2/storage/files/prepare]
+## Upload File [/v2/storage/files/prepare?async={async}]
 ### Create File Resource [POST]
 This method allows you to upload a file to KBC File Storage. The upload should be done in two steps:
 1. Create a new file resource (use this API call).
@@ -6587,6 +6587,9 @@ This method allows you to upload a file to KBC File Storage. The upload should b
 For a more detailed guide on the file upload, see the corresponding part of the
 [Developers documentation](https://developers.keboola.com/integrate/storage/api/import-export/#manually-uploading-a-file).
 
++ Parameters
+    + async (optional, boolean) - Prepare file in a background job. Calling the parameter as `async=false` is deprecated.
+
 + Attributes
     + name (required) - File name
     + sizeBytes (optional, number) - File size in bytes
@@ -6733,6 +6736,50 @@ For a more detailed guide on the file upload, see the corresponding part of the
                 }
             }
 
++ Response 202 (application/json)
+    Asynchronous call response (async=true) that creates a new job. To see job detail use [Job Detail API endpoint](#reference/jobs/manage-jobs/job-detail)
+    + Body
+
+           {
+               "id": 11,
+               "status": "waiting",
+               "url": "https://connection.keboola.com/v2/storage/jobs/11",
+               "operationName": "workspaceCreate",
+               "operationParams": {
+                   "name": "filePrepare",
+                   "isPermanent": true,
+                   "isSliced": false,
+                   "contentType": "text/csv",
+                   "tags": [
+                       "tag1",
+                       "tag2"
+                   ],
+                   "sizeBytes": 123456,
+                   "isPublic": false,
+                   "notify": true,
+                   "isEncrypted": false,
+                   "federationToken": false,
+                   "branchId": 1
+               },
+               "createdTime": "2013-05-31T16:11:05+0200",
+               "startTime": null,
+               "endTime": null,
+               "runId": null,
+               "results": null,
+               "creatorToken": {
+                   "id": "27978",
+                   "description": "[email protected]"
+               },
+               "metrics": {
+                   "inCompressed": false,
+                   "inBytes": 0,
+                   "inBytesUncompressed": 0,
+                   "outCompressed": false,
+                   "outBytes": 0,
+                   "outBytesUncompressed": 0
+               }
+           }
+
 ## Refresh File Credentials [/v2/storage/files/{file_id}/refresh]
 ### Refresh File Credentials [PUT]
 **Endpoint is used to generate new credentials only for files stored in GCS.**
@@ -6890,14 +6937,49 @@ Deletes a file from storage.
 
 + Parameters
     + file_id (required, number) - File Id
+    + async (optional, boolean) - Prepare file in a background job. Calling the parameter as `async=false` is deprecated.
+
 
 + Request
     + Headers
 
             X-StorageApi-Token: your_token
 
 + Response 204
+    Synchronous call response without content.
+
 
++ Response 202 (application/json)
+    Asynchronous call response (async=true) that creates a new job. To see job detail use [Job Detail API endpoint](#reference/jobs/manage-jobs/job-detail)
+    + Body
+
+           {
+               "id": 11,
+               "status": "waiting",
+               "url": "https://connection.keboola.com/v2/storage/jobs/11",
+               "operationName": "workspaceCreate",
+               "operationParams": {
+                   "fileId": 1,
+                   "branchId": 1
+               },
+               "createdTime": "2013-05-31T16:11:05+0200",
+               "startTime": null,
+               "endTime": null,
+               "runId": null,
+               "results": null,
+               "creatorToken": {
+                   "id": "27978",
+                   "description": "[email protected]"
+               },
+               "metrics": {
+                   "inCompressed": false,
+                   "inBytes": 0,
+                   "inBytesUncompressed": 0,
+                   "outCompressed": false,
+                   "outBytes": 0,
+                   "outBytesUncompressed": 0
+               }
+           }
 
 ## Create Tag [/v2/storage/files/{file_id}/tags]
 ### Add Tag [POST]

diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon
@@ -255,11 +255,6 @@ parameters:
 			count: 1
 			path: src/Keboola/StorageApi/Client.php
 
-		-
-			message: "#^Method Keboola\\\\StorageApi\\\\Client\\:\\:deleteFile\\(\\) has parameter \\$fileId with no type specified\\.$#"
-			count: 1
-			path: src/Keboola/StorageApi/Client.php
-
 		-
 			message: "#^Method Keboola\\\\StorageApi\\\\Client\\:\\:deleteFileTag\\(\\) has no return type specified\\.$#"
 			count: 1
@@ -4975,11 +4970,6 @@ parameters:
 			count: 1
 			path: tests/Common/TokensTest.php
 
-		-
-			message: "#^Method Keboola\\\\Test\\\\File\\\\AwsFileTest\\:\\:encryptedData\\(\\) has no return type specified\\.$#"
-			count: 1
-			path: tests/File/AwsFileTest.php
-
 		-
 			message: "#^Method Keboola\\\\Test\\\\File\\\\AwsFileTest\\:\\:testFileUpload\\(\\) has parameter \\$filePath with no type specified\\.$#"
 			count: 1
@@ -5015,16 +5005,6 @@ parameters:
 			count: 1
 			path: tests/File/AzureFileTest.php
 
-		-
-			message: "#^Method Keboola\\\\Test\\\\File\\\\AzureFileTest\\:\\:uploadData\\(\\) has no return type specified\\.$#"
-			count: 1
-			path: tests/File/AzureFileTest.php
-
-		-
-			message: "#^Method Keboola\\\\Test\\\\File\\\\AzureFileTest\\:\\:uploadSlicedData\\(\\) has no return type specified\\.$#"
-			count: 1
-			path: tests/File/AzureFileTest.php
-
 		-
 			message: "#^Parameter \\#1 \\$json of function json_decode expects string, string\\|false given\\.$#"
 			count: 1

diff --git a/src/Keboola/StorageApi/Client.php b/src/Keboola/StorageApi/Client.php
@@ -1774,7 +1774,7 @@ public function deleteTableRowsAsQuery($tableId, $options = [])
      * @return int - created file id
      * @throws ClientException
      */
-    public function uploadFile($filePath, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null)
+    public function uploadFile($filePath, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null, bool $async = true)
     {
         if (!is_readable($filePath)) {
             throw new ClientException('File is not readable: ' . $filePath, null, null, 'fileNotReadable');
@@ -1815,7 +1815,7 @@ public function uploadFile($filePath, FileUploadOptions $options, FileUploadTran
             ->setSizeBytes($sizeBytes)
             ->setFederationToken(true);
 
-        $prepareResult = $this->prepareFileUpload($newOptions);
+        $prepareResult = $this->prepareFileUpload($newOptions, $async);
 
         switch ($prepareResult['provider']) {
             case self::FILE_PROVIDER_AZURE:
@@ -1973,7 +1973,7 @@ private function uploadFileToS3(
      * @return int created file id
      * @throws ClientException
      */
-    public function uploadSlicedFile(array $slices, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null)
+    public function uploadSlicedFile(array $slices, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null, bool $async = true)
     {
         if (!$options->getIsSliced()) {
             throw new ClientException('File is not sliced.');
@@ -2030,7 +2030,7 @@ public function uploadSlicedFile(array $slices, FileUploadOptions $options, File
             ->setIsSliced(true);
 
         // 1. prepare resource
-        $prepareResult = $this->prepareFileUpload($newOptions);
+        $prepareResult = $this->prepareFileUpload($newOptions, $async);
 
         switch ($prepareResult['provider']) {
             case self::FILE_PROVIDER_AZURE:
@@ -2475,9 +2475,14 @@ private function downloadGcsSlicedFile(array $fileInfo, string $destinationFolde
      * @param FileUploadOptions $options
      * @return array file info
      */
-    public function prepareFileUpload(FileUploadOptions $options)
+    public function prepareFileUpload(FileUploadOptions $options, bool $async = true)
     {
-        return $this->apiPostJson('files/prepare', [
+        $url = 'files/prepare';
+        if ($async) {
+            $url .= '?' . http_build_query(['async' => $async]);
+        }
+
+        return $this->apiPostJson($url, [
             'isPublic' => $options->getIsPublic(),
             'isPermanent' => $options->getIsPermanent(),
             'isEncrypted' => $options->getIsEncrypted(),
@@ -2500,15 +2505,16 @@ public function refreshFileCredentials(int $fileId): array
 
     /**
      * Delete a single file
-     * @param $fileId
-     * @return mixed|string
      */
-    public function deleteFile($fileId)
+    public function deleteFile(int $fileId, bool $async = true): void
     {
-        return $this->apiDelete("files/$fileId");
+        $url = sprintf('files/%s', $fileId);
+        if ($async) {
+            $url .= '?' . http_build_query(['async' => $async]);
+        }
+        $this->apiDelete($url);
     }
 
-
     /**
      * Get a single file
      * @param string|int $fileId

diff --git a/tests/File/AwsFileTest.php b/tests/File/AwsFileTest.php
@@ -3,6 +3,7 @@
 namespace Keboola\Test\File;
 
 use Aws\S3\S3Client;
+use Generator;
 use GuzzleHttp\Client;
 use Keboola\StorageApi\BranchAwareClient;
 use Keboola\StorageApi\Client as StorageApiClient;
@@ -50,9 +51,14 @@ public function setUp(): void
     /**
      * @dataProvider uploadData
      */
-    public function testFileUpload(string $devBranchType, string $userRole, $filePath, FileUploadOptions $options): void
-    {
-        $fileId = $this->_testClient->uploadFile($filePath, $options);
+    public function testFileUpload(
+        string $devBranchType,
+        string $userRole,
+        $filePath,
+        FileUploadOptions $options,
+        bool $isAsync
+    ): void {
+        $fileId = $this->_testClient->uploadFile($filePath, $options, null, $isAsync);
         $file = $this->_testClient->getFile($fileId);
 
         $this->assertEquals($options->getIsPublic(), $file['isPublic']);
@@ -126,7 +132,17 @@ public function uploadData()
 
         $clientProvider = $this->provideComponentsClientTypeBasedOnSuite();
 
-        return $this->combineProviders($uploadData, $clientProvider);
+        foreach ([true, false] as $async) {
+            $asyncName = $async ? 'async' : 'sync';
+            /**
+             * @var string $testName
+             * @var array<mixed> $data
+             */
+            foreach ($this->combineProviders($uploadData, $clientProvider) as $testName => $data) {
+                $data['isAsync'] = $async;
+                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
+            }
+        }
     }
 
     public function provideComponentsClientTypeBasedOnSuite(): array
@@ -138,7 +154,7 @@ public function provideComponentsClientTypeBasedOnSuite(): array
      * @dataProvider encryptedData
      * @param $encrypted
      */
-    public function testFileUploadUsingFederationToken(string $devBranchType, string $userRole, $encrypted): void
+    public function testFileUploadUsingFederationToken(string $devBranchType, string $userRole, $encrypted, bool $isAsync): void
     {
         $pathToFile = __DIR__ . '/../_data/files.upload.txt';
         $options = new FileUploadOptions();
@@ -152,7 +168,7 @@ public function testFileUploadUsingFederationToken(string $devBranchType, string
         $uploadParams = $result['uploadParams'];
         $this->assertArrayHasKey('credentials', $uploadParams);
 
-        $fileId = $this->_testClient->uploadFile($pathToFile, $options);
+        $fileId = $this->_testClient->uploadFile($pathToFile, $options, null, $isAsync);
 
         $file = $this->_testClient->getFile($fileId);
 
@@ -181,7 +197,7 @@ public function testFileUploadUsingFederationToken(string $devBranchType, string
         }
     }
 
-    public function encryptedData()
+    public function encryptedData(): Generator
     {
         $encryptedData = [
             'encrypted: false' => [false],
@@ -190,7 +206,17 @@ public function encryptedData()
 
         $clientProvider = $this->provideComponentsClientTypeBasedOnSuite();
 
-        return $this->combineProviders($encryptedData, $clientProvider);
+        foreach ([true, false] as $async) {
+            $asyncName = $async ? 'async' : 'sync';
+            /**
+             * @var string $testName
+             * @var array<mixed> $data
+             */
+            foreach ($this->combineProviders($encryptedData, $clientProvider) as $testName => $data) {
+                $data['isAsync'] = $async;
+                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
+            }
+        }
     }
 
     /**