Merge pull request #1315 from keboola/zajca-ct-1363

CT-1363 remove file prepare async
keboola · Apr 8, 2024 · fb1401d · fb1401d
2 parents 141a7df + 62510cf
commit fb1401d
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 155 deletions.
diff --git a/apiary.apib b/apiary.apib
@@ -6578,7 +6578,7 @@ And then, upload that manifest file to the key `exp-180/11/files/2014/02/25/1108
 [Developers documentation](https://developers.keboola.com/integrate/storage/api/import-export/#working-with-sliced-files)
 for a more detailed guide.
 
-## Upload File [/v2/storage/files/prepare?async={async}]
+## Upload File [/v2/storage/files/prepare]
 ### Create File Resource [POST]
 This method allows you to upload a file to KBC File Storage. The upload should be done in two steps:
 1. Create a new file resource (use this API call).
@@ -6587,9 +6587,6 @@ This method allows you to upload a file to KBC File Storage. The upload should b
 For a more detailed guide on the file upload, see the corresponding part of the
 [Developers documentation](https://developers.keboola.com/integrate/storage/api/import-export/#manually-uploading-a-file).
 
-+ Parameters
-    + async (optional, boolean) - Prepare file in a background job. Calling the parameter as `async=false` is deprecated.
-
 + Attributes
     + name (required) - File name
     + sizeBytes (optional, number) - File size in bytes
@@ -6736,50 +6733,6 @@ For a more detailed guide on the file upload, see the corresponding part of the
                 }
             }
 
-+ Response 202 (application/json)
-    Asynchronous call response (async=true) that creates a new job. To see job detail use [Job Detail API endpoint](#reference/jobs/manage-jobs/job-detail)
-    + Body
-
-           {
-               "id": 11,
-               "status": "waiting",
-               "url": "https://connection.keboola.com/v2/storage/jobs/11",
-               "operationName": "workspaceCreate",
-               "operationParams": {
-                   "name": "filePrepare",
-                   "isPermanent": true,
-                   "isSliced": false,
-                   "contentType": "text/csv",
-                   "tags": [
-                       "tag1",
-                       "tag2"
-                   ],
-                   "sizeBytes": 123456,
-                   "isPublic": false,
-                   "notify": true,
-                   "isEncrypted": false,
-                   "federationToken": false,
-                   "branchId": 1
-               },
-               "createdTime": "2013-05-31T16:11:05+0200",
-               "startTime": null,
-               "endTime": null,
-               "runId": null,
-               "results": null,
-               "creatorToken": {
-                   "id": "27978",
-                   "description": "[email protected]"
-               },
-               "metrics": {
-                   "inCompressed": false,
-                   "inBytes": 0,
-                   "inBytesUncompressed": 0,
-                   "outCompressed": false,
-                   "outBytes": 0,
-                   "outBytesUncompressed": 0
-               }
-           }
-
 ## Refresh File Credentials [/v2/storage/files/{file_id}/refresh]
 ### Refresh File Credentials [PUT]
 **Endpoint is used to generate new credentials only for files stored in GCS.**

diff --git a/phpstan-baseline.neon b/phpstan-baseline.neon
@@ -4970,6 +4970,11 @@ parameters:
 			count: 1
 			path: tests/Common/TokensTest.php
 
+		-
+			message: "#^Method Keboola\\\\Test\\\\File\\\\AwsFileTest\\:\\:encryptedData\\(\\) has no return type specified\\.$#"
+			count: 1
+			path: tests/File/AwsFileTest.php
+
 		-
 			message: "#^Method Keboola\\\\Test\\\\File\\\\AwsFileTest\\:\\:testFileUpload\\(\\) has parameter \\$filePath with no type specified\\.$#"
 			count: 1
@@ -5005,6 +5010,16 @@ parameters:
 			count: 1
 			path: tests/File/AzureFileTest.php
 
+		-
+			message: "#^Method Keboola\\\\Test\\\\File\\\\AzureFileTest\\:\\:uploadData\\(\\) has no return type specified\\.$#"
+			count: 1
+			path: tests/File/AzureFileTest.php
+
+		-
+			message: "#^Method Keboola\\\\Test\\\\File\\\\AzureFileTest\\:\\:uploadSlicedData\\(\\) has no return type specified\\.$#"
+			count: 1
+			path: tests/File/AzureFileTest.php
+
 		-
 			message: "#^Parameter \\#1 \\$json of function json_decode expects string, string\\|false given\\.$#"
 			count: 1

diff --git a/src/Keboola/StorageApi/Client.php b/src/Keboola/StorageApi/Client.php
@@ -1774,7 +1774,7 @@ public function deleteTableRowsAsQuery($tableId, $options = [])
      * @return int - created file id
      * @throws ClientException
      */
-    public function uploadFile($filePath, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null, bool $async = true)
+    public function uploadFile($filePath, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null)
     {
         if (!is_readable($filePath)) {
             throw new ClientException('File is not readable: ' . $filePath, null, null, 'fileNotReadable');
@@ -1815,7 +1815,7 @@ public function uploadFile($filePath, FileUploadOptions $options, FileUploadTran
             ->setSizeBytes($sizeBytes)
             ->setFederationToken(true);
 
-        $prepareResult = $this->prepareFileUpload($newOptions, $async);
+        $prepareResult = $this->prepareFileUpload($newOptions);
 
         switch ($prepareResult['provider']) {
             case self::FILE_PROVIDER_AZURE:
@@ -1973,7 +1973,7 @@ private function uploadFileToS3(
      * @return int created file id
      * @throws ClientException
      */
-    public function uploadSlicedFile(array $slices, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null, bool $async = true)
+    public function uploadSlicedFile(array $slices, FileUploadOptions $options, FileUploadTransferOptions $transferOptions = null)
     {
         if (!$options->getIsSliced()) {
             throw new ClientException('File is not sliced.');
@@ -2030,7 +2030,7 @@ public function uploadSlicedFile(array $slices, FileUploadOptions $options, File
             ->setIsSliced(true);
 
         // 1. prepare resource
-        $prepareResult = $this->prepareFileUpload($newOptions, $async);
+        $prepareResult = $this->prepareFileUpload($newOptions);
 
         switch ($prepareResult['provider']) {
             case self::FILE_PROVIDER_AZURE:
@@ -2475,14 +2475,9 @@ private function downloadGcsSlicedFile(array $fileInfo, string $destinationFolde
      * @param FileUploadOptions $options
      * @return array file info
      */
-    public function prepareFileUpload(FileUploadOptions $options, bool $async = true)
+    public function prepareFileUpload(FileUploadOptions $options)
     {
-        $url = 'files/prepare';
-        if ($async) {
-            $url .= '?' . http_build_query(['async' => $async]);
-        }
-
-        return $this->apiPostJson($url, [
+        return $this->apiPostJson('files/prepare', [
             'isPublic' => $options->getIsPublic(),
             'isPermanent' => $options->getIsPermanent(),
             'isEncrypted' => $options->getIsEncrypted(),

diff --git a/tests/File/AwsFileTest.php b/tests/File/AwsFileTest.php
@@ -3,7 +3,6 @@
 namespace Keboola\Test\File;
 
 use Aws\S3\S3Client;
-use Generator;
 use GuzzleHttp\Client;
 use Keboola\StorageApi\BranchAwareClient;
 use Keboola\StorageApi\Client as StorageApiClient;
@@ -51,14 +50,9 @@ public function setUp(): void
     /**
      * @dataProvider uploadData
      */
-    public function testFileUpload(
-        string $devBranchType,
-        string $userRole,
-        $filePath,
-        FileUploadOptions $options,
-        bool $isAsync
-    ): void {
-        $fileId = $this->_testClient->uploadFile($filePath, $options, null, $isAsync);
+    public function testFileUpload(string $devBranchType, string $userRole, $filePath, FileUploadOptions $options): void
+    {
+        $fileId = $this->_testClient->uploadFile($filePath, $options);
         $file = $this->_testClient->getFile($fileId);
 
         $this->assertEquals($options->getIsPublic(), $file['isPublic']);
@@ -132,17 +126,7 @@ public function uploadData()
 
         $clientProvider = $this->provideComponentsClientTypeBasedOnSuite();
 
-        foreach ([true, false] as $async) {
-            $asyncName = $async ? 'async' : 'sync';
-            /**
-             * @var string $testName
-             * @var array<mixed> $data
-             */
-            foreach ($this->combineProviders($uploadData, $clientProvider) as $testName => $data) {
-                $data['isAsync'] = $async;
-                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
-            }
-        }
+        return $this->combineProviders($uploadData, $clientProvider);
     }
 
     public function provideComponentsClientTypeBasedOnSuite(): array
@@ -154,7 +138,7 @@ public function provideComponentsClientTypeBasedOnSuite(): array
      * @dataProvider encryptedData
      * @param $encrypted
      */
-    public function testFileUploadUsingFederationToken(string $devBranchType, string $userRole, $encrypted, bool $isAsync): void
+    public function testFileUploadUsingFederationToken(string $devBranchType, string $userRole, $encrypted): void
     {
         $pathToFile = __DIR__ . '/../_data/files.upload.txt';
         $options = new FileUploadOptions();
@@ -168,7 +152,7 @@ public function testFileUploadUsingFederationToken(string $devBranchType, string
         $uploadParams = $result['uploadParams'];
         $this->assertArrayHasKey('credentials', $uploadParams);
 
-        $fileId = $this->_testClient->uploadFile($pathToFile, $options, null, $isAsync);
+        $fileId = $this->_testClient->uploadFile($pathToFile, $options);
 
         $file = $this->_testClient->getFile($fileId);
 
@@ -197,7 +181,7 @@ public function testFileUploadUsingFederationToken(string $devBranchType, string
         }
     }
 
-    public function encryptedData(): Generator
+    public function encryptedData()
     {
         $encryptedData = [
             'encrypted: false' => [false],
@@ -206,17 +190,7 @@ public function encryptedData(): Generator
 
         $clientProvider = $this->provideComponentsClientTypeBasedOnSuite();
 
-        foreach ([true, false] as $async) {
-            $asyncName = $async ? 'async' : 'sync';
-            /**
-             * @var string $testName
-             * @var array<mixed> $data
-             */
-            foreach ($this->combineProviders($encryptedData, $clientProvider) as $testName => $data) {
-                $data['isAsync'] = $async;
-                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
-            }
-        }
+        return $this->combineProviders($encryptedData, $clientProvider);
     }
 
     /**

diff --git a/tests/File/AzureFileTest.php b/tests/File/AzureFileTest.php
@@ -3,13 +3,11 @@
 namespace Keboola\Test\File;
 
 use Exception;
-use Generator;
 use GuzzleHttp\Client;
 use Keboola\StorageApi\ABSUploader;
 use Keboola\StorageApi\Downloader\BlobClientFactory;
 use Keboola\StorageApi\Options\FileUploadOptions;
 use Keboola\StorageApi\Options\GetFileOptions;
-use Keboola\Test\ClientProvider\TestSetupHelper;
 use Keboola\Test\StorageApiTestCase;
 use MicrosoftAzure\Storage\Blob\Models\CommitBlobBlocksOptions;
 use MicrosoftAzure\Storage\Blob\Models\CreateBlockBlobOptions;
@@ -30,9 +28,9 @@ public function setUp(): void
     /**
      * @dataProvider uploadData
      */
-    public function testFileUpload($filePath, FileUploadOptions $options, bool $isAsync): void
+    public function testFileUpload($filePath, FileUploadOptions $options): void
     {
-        $fileId = $this->_client->uploadFile($filePath, $options, null, $isAsync);
+        $fileId = $this->_client->uploadFile($filePath, $options);
         $file = $this->_client->getFile($fileId, (new GetFileOptions())->setFederationToken(true));
 
         //azure storage is always encrypted and private. Request params 'isEncrypted' and 'isPublic' is ignored
@@ -68,40 +66,40 @@ public function testFileUpload($filePath, FileUploadOptions $options, bool $isAs
         $client->get($file['url']);
     }
 
-    public function provideComponentsClientTypeBasedOnSuite(): array
-    {
-        return (new TestSetupHelper())->provideComponentsClientTypeBasedOnSuite($this);
-    }
-
-    public function uploadData(): Generator
+    public function uploadData()
     {
         $path = __DIR__ . '/../_data/files.upload.txt';
         $largeFilePath = sys_get_temp_dir() . '/large_abs_upload.txt';
         $this->generateFile($largeFilePath, 16);
 
-        $uploadData = [
-            'isPublic: true' => [
+        return [
+            [
                 $path,
                 (new FileUploadOptions())->setIsPublic(true),
             ],
-            'isEncrypted: false' => [
+            [
+                $path,
+                (new FileUploadOptions())
+                    ->setIsPublic(true),
+            ],
+            [
                 $path,
                 (new FileUploadOptions())
                     ->setIsEncrypted(false),
             ],
-            'isEncrypted: true' => [
+            [
                 $path,
                 (new FileUploadOptions())
                     ->setIsEncrypted(true),
             ],
-            'notify: false, compress: false, isPublic:false' => [
+            [
                 $path,
                 (new FileUploadOptions())
                     ->setNotify(false)
                     ->setCompress(false)
                     ->setIsPublic(false),
             ],
-            'isPublic: false, isPermanent: true, tags: \'sapi-import\', \'martin\'' => [
+            [
                 $path,
                 (new FileUploadOptions())
                     ->setIsPublic(true)
@@ -116,40 +114,24 @@ public function uploadData(): Generator
                     ->setTags(['sapi-import', 'martin']),
             ],
         ];
-
-        foreach ([true, false] as $async) {
-            $asyncName = $async ? 'async' : 'sync';
-            foreach ($uploadData as $testName => $data) {
-                $data['isAsync'] = $async;
-                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
-            }
-        }
     }
 
-    public function uploadSlicedData(): Generator
+    public function uploadSlicedData()
     {
         $part1 = sys_get_temp_dir() . '/slice.csv.part_1';
         $part2 = sys_get_temp_dir() . '/slice.csv.part_2';
         $parts = [$part1, $part2];
         $this->generateFile($part1, 16);
         $this->generateFile($part2, 16);
 
-        $uploadData = [
-            'slice' => [
+        return [
+            [
                 $parts,
                 (new FileUploadOptions())
                     ->setIsSliced(true)
                     ->setFileName('slice.csv'),
             ],
         ];
-
-        foreach ([true, false] as $async) {
-            $asyncName = $async ? 'async' : 'sync';
-            foreach ($uploadData as $testName => $data) {
-                $data['isAsync'] = $async;
-                yield sprintf('%s -> %s', $testName, $asyncName) => $data;
-            }
-        }
     }
 
     public function testReUpload(): void
@@ -200,9 +182,9 @@ public function testReUpload(): void
     /**
      * @dataProvider uploadSlicedData
      */
-    public function testUploadSlicedFile(array $slices, FileUploadOptions $options, bool $isAsync): void
+    public function testUploadSlicedFile(array $slices, FileUploadOptions $options): void
     {
-        $fileId = $this->_client->uploadSlicedFile($slices, $options, null, $isAsync);
+        $fileId = $this->_client->uploadSlicedFile($slices, $options);
         $file = $this->_client->getFile($fileId, (new GetFileOptions())->setFederationToken(true));
 
         $this->assertEquals($options->getIsPublic(), $file['isPublic']);