Merge pull request #1317 from keboola/martinj-ct-1340-table-imort-ord…

…ered-columns

CT-1340 table import ordered columns

apiary.apib

@@ -3097,7 +3097,7 @@ Further information can be found in the [Developers Documentation](https://devel
     + enclosure (optional, string) - Field enclosure used in the CSV file. The default value is `"`. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.)
     + escapedBy (optional, string) - Escape character used in the CSV file. The default value is an empty value - no escape character is used. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.)
     + columns[] (optional, string) - List of columns present in the CSV file; the first line of the file will not be treated as a header!
-    + withoutHeaders (optional, boolean) - The CSV file doesn't contain headers, columns are matched by their order. If this option is used, columns option is ignored.
+    + withoutHeaders (optional, boolean) - Indicates that the CSV file does not have header row. The columns in the CSV file must be in the same order as the columns in the table in storage. Use the 'columns' option to specify the column order.
     + treatValuesAsNull (optional, array[string]) - (Snowflake and BigQuery only) Array of string values that should be imported as NULL values. The default value is `['']`. Some other common values you might consider are `'\N'` or `'null'`. **Currently only single value is allowed**, but it still must be sent as array to allow future expansion.
 
 + Request (application/json)

tests/Backend/CommonPart1/ImportExportCommonTest.php

@@ -441,6 +441,167 @@ public function testImportWithColumnsList(): void
         $this->assertNotEmpty($result['totalDataSizeBytes']);
     }
 
+    /**
+     * @dataProvider tableImportReorderedData
+     */
+    public function testImportWithReorderedColumnsList(
+        string $headersFile,
+        array $importedFiles,
+        array $columns,
+        bool $incremental,
+        array $expectedColumns,
+        string $expectedData,
+        array $skipBackends = [],
+        string $skipMessage = ''
+    ): void {
+        if (!empty($skipBackends)) {
+            $this->skipTestForBackend($skipBackends, $skipMessage);
+        }
+        $headersCsv = new CsvFile($headersFile);
+        $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', $headersCsv);
+
+        // import reordered csv file
+        foreach ($importedFiles as $importedFile) {
+            /** @var array $result */
+            $result = $this->_client->writeTableAsync($tableId, new CsvFile($importedFile), [
+                'columns' => $columns,
+                'withoutHeaders' => true,
+                'incremental' => $incremental,
+            ]);
+            $this->assertEmpty($result['warnings']);
+            $this->assertEmpty($result['transaction']);
+            $this->assertNotEmpty($result['totalDataSizeBytes']);
+        }
+        $table = $this->_client->getTable($tableId);
+        $this->assertNotEmpty($table['dataSizeBytes']);
+        $this->assertArrayHasKey('columns', $table);
+        $this->assertIsArray($table['columns']);
+        $this->assertTrue($expectedColumns === $table['columns']);
+
+        // check that table columns aren't reordered
+        $this->assertLinesEqualsSorted(
+            $expectedData,
+            $this->_client->getTableDataPreview($tableId, [
+                'format' => 'rfc',
+            ]),
+            'imported data comparison',
+        );
+    }
+
+    /**
+     * @return \Generator<string, array{headersFile: string, importedFiles: array<string>, columns: array<string>, incremental: bool, expectedData: string, expectedColumns: array<string>}>
+     */
+    public function tableImportReorderedData(): \Generator
+    {
+        yield 'new-column' => [
+            'headersFile' => __DIR__ . '/../../_data/languages-headers.csv',
+            'importedFiles' => [
+                __DIR__ . '/../../_data/languages-without-headers-reordered-new-column.csv',
+            ],
+            'columns' => [
+                'name',
+                'id',
+                'code',
+            ],
+            'incremental' => false,
+            'expectedColumns' => [
+                'id',
+                'name',
+                'code',
+            ],
+            'expectedData' => <<<END
+
+"0","- unchecked -",""
+"1","english","en"
+"11","finnish","fi"
+"24","french","fr"
+"26","czech","cz"
+"id","name","code"
+END,
+            'skipBackends' => [
+                self::BACKEND_BIGQUERY,
+            ],
+            'skipMessage' => 'Don\'t test new columns on BigQuery.',
+        ];
+        yield 'full' => [
+            'headersFile' => __DIR__ . '/../../_data/languages-headers.csv',
+            'importedFiles' => [
+                __DIR__ . '/../../_data/languages-without-headers-reordered.csv',
+            ],
+            'columns' => [
+                'name',
+                'id',
+            ],
+            'incremental' => false,
+            'expectedColumns' => [
+                'id',
+                'name',
+            ],
+            'expectedData' => <<<END
+
+"0","- unchecked -"
+"1","english"
+"11","finnish"
+"24","french"
+"26","czech"
+"id","name"
+END,
+        ];
+        yield 'incremental' => [
+            'headersFile' => __DIR__ . '/../../_data/languages.csv',
+            'importedFiles' => [
+                __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv',
+            ],
+            'columns' => [
+                'name',
+                'id',
+            ],
+            'incremental' => true,
+            'expectedColumns' => [
+                'id',
+                'name',
+            ],
+            'expectedData' => <<<END
+
+"0","- unchecked -"
+"1","english"
+"11","finnish"
+"24","french"
+"26","czech"
+"27","spanish"
+"28","greek"
+"id","name"
+END,
+        ];
+        yield 'incremental-to-empty-table' => [
+            'headersFile' => __DIR__ . '/../../_data/languages-headers.csv',
+            'importedFiles' => [
+                __DIR__ . '/../../_data/languages-without-headers-reordered.csv',
+                __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv',
+            ],
+            'columns' => [
+                'name',
+                'id',
+            ],
+            'incremental' => true,
+            'expectedColumns' => [
+                'id',
+                'name',
+            ],
+            'expectedData' => <<<END
+
+"0","- unchecked -"
+"1","english"
+"11","finnish"
+"24","french"
+"26","czech"
+"27","spanish"
+"28","greek"
+"id","name"
+END,
+        ];
+    }
+
     public function testTableImportFromString(): void
     {
         $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', new CsvFile(__DIR__ . '/../../_data/languages-headers.csv'));

tests/_data/languages-without-headers-reordered-incremental.csv

@@ -0,0 +1,2 @@
+"spanish","27"
+"greek","28"

tests/_data/languages-without-headers-reordered-new-column.csv

@@ -0,0 +1,5 @@
+"- unchecked -","0",""
+"czech","26","cz"
+"english","1","en"
+"finnish","11","fi"
+"french","24","fr"

tests/_data/languages-without-headers-reordered.csv

@@ -0,0 +1,5 @@
+"- unchecked -","0"
+"czech","26"
+"english","1"
+"finnish","11"
+"french","24"