From 6d535996e9e931c24809d4fdcc18beb2b6399547 Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Sun, 14 Apr 2024 18:29:51 +0200 Subject: [PATCH 1/9] Import table async test with ordered columns without headers --- .../CommonPart1/ImportExportCommonTest.php | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 515463d56..908070c0e 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -441,6 +441,29 @@ public function testImportWithColumnsList(): void $this->assertNotEmpty($result['totalDataSizeBytes']); } + public function testImportWithOrderedColumnsListWithoutHeaders(): void + { + $headersCsv = new CsvFile(__DIR__ . '/../../_data/languages-headers.csv'); + $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', $headersCsv); + + $importedFile = __DIR__ . '/../../_data/languages-without-headers.csv'; + /** @var array $result */ + $result = $this->_client->writeTableAsync($tableId, new CsvFile($importedFile), [ + 'columns' => array_reverse($headersCsv->getHeader()), + 'withoutHeaders' => true, + ]); + $table = $this->_client->getTable($tableId); + + $this->assertEmpty($result['warnings']); + $this->assertEmpty($result['transaction']); + $this->assertNotEmpty($table['dataSizeBytes']); + $this->assertNotEmpty($result['totalDataSizeBytes']); + $this->assertArrayHasKey('columns', $table); + $this->assertIsArray($table['columns']); + $this->assertEquals('name', reset($table['columns'])); + $this->assertEquals('id', next($table['columns'])); + } + public function testTableImportFromString(): void { $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', new CsvFile(__DIR__ . '/../../_data/languages-headers.csv')); From 9eefe2320836e8d79f0405d72ad951f9526a4cd3 Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Sun, 14 Apr 2024 18:39:12 +0200 Subject: [PATCH 2/9] Fix rebase --- apiary.apib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apiary.apib b/apiary.apib index 28f38eb23..49fe8b65c 100644 --- a/apiary.apib +++ b/apiary.apib @@ -3097,7 +3097,7 @@ Further information can be found in the [Developers Documentation](https://devel + enclosure (optional, string) - Field enclosure used in the CSV file. The default value is `"`. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.) + escapedBy (optional, string) - Escape character used in the CSV file. The default value is an empty value - no escape character is used. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.) + columns[] (optional, string) - List of columns present in the CSV file; the first line of the file will not be treated as a header! - + withoutHeaders (optional, boolean) - The CSV file doesn't contain headers, columns are matched by their order. If this option is used, columns option is ignored. + + withoutHeaders (optional, boolean) - The CSV file doesn't contain headers, columns are matched by their order. The columns option could be used to specify the order of columns. + treatValuesAsNull (optional, array[string]) - (Snowflake and BigQuery only) Array of string values that should be imported as NULL values. The default value is `['']`. Some other common values you might consider are `'\N'` or `'null'`. **Currently only single value is allowed**, but it still must be sent as array to allow future expansion. + Request (application/json) From 6cfb387cc2f7d713d8411b18c96a1493b7ec2ef6 Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Thu, 18 Apr 2024 13:27:10 +0200 Subject: [PATCH 3/9] Fix rebase --- apiary.apib | 2 +- .../CommonPart1/ImportExportCommonTest.php | 86 ++++++++++++++++--- ...-without-headers-reordered-incremental.csv | 2 + .../languages-without-headers-reordered.csv | 5 ++ 4 files changed, 81 insertions(+), 14 deletions(-) create mode 100644 tests/_data/languages-without-headers-reordered-incremental.csv create mode 100644 tests/_data/languages-without-headers-reordered.csv diff --git a/apiary.apib b/apiary.apib index 49fe8b65c..115fc27ec 100644 --- a/apiary.apib +++ b/apiary.apib @@ -3097,7 +3097,7 @@ Further information can be found in the [Developers Documentation](https://devel + enclosure (optional, string) - Field enclosure used in the CSV file. The default value is `"`. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.) + escapedBy (optional, string) - Escape character used in the CSV file. The default value is an empty value - no escape character is used. (Note: you can specify either `enclosure` or `escapedBy` parameter, not both.) + columns[] (optional, string) - List of columns present in the CSV file; the first line of the file will not be treated as a header! - + withoutHeaders (optional, boolean) - The CSV file doesn't contain headers, columns are matched by their order. The columns option could be used to specify the order of columns. + + withoutHeaders (optional, boolean) - Indicates that the CSV file does not have header row. The columns in the CSV file must be in the same order as the columns in the table in storage. Use the 'columns' option to specify the column order. + treatValuesAsNull (optional, array[string]) - (Snowflake and BigQuery only) Array of string values that should be imported as NULL values. The default value is `['']`. Some other common values you might consider are `'\N'` or `'null'`. **Currently only single value is allowed**, but it still must be sent as array to allow future expansion. + Request (application/json) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 908070c0e..c796b1122 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -441,27 +441,87 @@ public function testImportWithColumnsList(): void $this->assertNotEmpty($result['totalDataSizeBytes']); } - public function testImportWithOrderedColumnsListWithoutHeaders(): void - { - $headersCsv = new CsvFile(__DIR__ . '/../../_data/languages-headers.csv'); + /** + * @dataProvider tableImportReorderedData + */ + public function testImportWithReorderedColumnsList( + string $headersFile, + array $importedFiles, + array $columns, + bool $incremental, + string $expectedData + ): void { + $headersCsv = new CsvFile($headersFile); $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', $headersCsv); - $importedFile = __DIR__ . '/../../_data/languages-without-headers.csv'; - /** @var array $result */ - $result = $this->_client->writeTableAsync($tableId, new CsvFile($importedFile), [ - 'columns' => array_reverse($headersCsv->getHeader()), - 'withoutHeaders' => true, - ]); + // import reordered csv file + foreach ($importedFiles as $importedFile) { + /** @var array $result */ + $result = $this->_client->writeTableAsync($tableId, new CsvFile($importedFile), [ + 'columns' => $columns, + 'withoutHeaders' => true, + 'incremental' => $incremental, + ]); + } $table = $this->_client->getTable($tableId); $this->assertEmpty($result['warnings']); $this->assertEmpty($result['transaction']); $this->assertNotEmpty($table['dataSizeBytes']); $this->assertNotEmpty($result['totalDataSizeBytes']); - $this->assertArrayHasKey('columns', $table); - $this->assertIsArray($table['columns']); - $this->assertEquals('name', reset($table['columns'])); - $this->assertEquals('id', next($table['columns'])); + + // check that table columns aren't reordered + $this->assertLinesEqualsSorted( + $expectedData, + $this->_client->getTableDataPreview($tableId, [ + 'format' => 'rfc', + ]), + 'imported data comparison', + ); + } + + /** + * @return \Generator, columns: array, incremental: bool, expectedData: string}> + */ + public function tableImportReorderedData(): \Generator + { + yield 'full' => [ + __DIR__ . '/../../_data/languages-headers.csv', + [ + __DIR__ . '/../../_data/languages-without-headers-reordered.csv', + ], + [ + 'name', + 'id', + ], + false, + '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"id","name"' . PHP_EOL, + ]; + yield 'incremental' => [ + __DIR__ . '/../../_data/languages.csv', + [ + __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv', + ], + [ + 'name', + 'id', + ], + true, + '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + ]; + yield 'incremental-to-empty-table' => [ + __DIR__ . '/../../_data/languages-headers.csv', + [ + __DIR__ . '/../../_data/languages-without-headers-reordered.csv', + __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv', + ], + [ + 'name', + 'id', + ], + true, + '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + ]; } public function testTableImportFromString(): void diff --git a/tests/_data/languages-without-headers-reordered-incremental.csv b/tests/_data/languages-without-headers-reordered-incremental.csv new file mode 100644 index 000000000..2fec7a3a1 --- /dev/null +++ b/tests/_data/languages-without-headers-reordered-incremental.csv @@ -0,0 +1,2 @@ +"spanish","27" +"greek","28" diff --git a/tests/_data/languages-without-headers-reordered.csv b/tests/_data/languages-without-headers-reordered.csv new file mode 100644 index 000000000..62c90616b --- /dev/null +++ b/tests/_data/languages-without-headers-reordered.csv @@ -0,0 +1,5 @@ +"- unchecked -","0" +"czech","26" +"english","1" +"finnish","11" +"french","24" From fe8f48dbe295182fd1eb22f832b14b3d3cf75b9b Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Thu, 18 Apr 2024 13:31:58 +0200 Subject: [PATCH 4/9] Fix testImportWithReorderedColumnsList --- tests/Backend/CommonPart1/ImportExportCommonTest.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index c796b1122..54165f5b8 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -462,13 +462,12 @@ public function testImportWithReorderedColumnsList( 'withoutHeaders' => true, 'incremental' => $incremental, ]); + $this->assertEmpty($result['warnings']); + $this->assertEmpty($result['transaction']); + $this->assertNotEmpty($result['totalDataSizeBytes']); } $table = $this->_client->getTable($tableId); - - $this->assertEmpty($result['warnings']); - $this->assertEmpty($result['transaction']); $this->assertNotEmpty($table['dataSizeBytes']); - $this->assertNotEmpty($result['totalDataSizeBytes']); // check that table columns aren't reordered $this->assertLinesEqualsSorted( From 7825a65ae03c24d37163ff3207cd3f75aea881fd Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Thu, 18 Apr 2024 13:43:18 +0200 Subject: [PATCH 5/9] Fix provider data offset --- .../CommonPart1/ImportExportCommonTest.php | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 54165f5b8..02971aec2 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -485,41 +485,41 @@ public function testImportWithReorderedColumnsList( public function tableImportReorderedData(): \Generator { yield 'full' => [ - __DIR__ . '/../../_data/languages-headers.csv', - [ + 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv', + 'importedFiles' => [ __DIR__ . '/../../_data/languages-without-headers-reordered.csv', ], - [ + 'columns' => [ 'name', 'id', ], - false, - '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'incremental' => false, + 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"id","name"' . PHP_EOL, ]; yield 'incremental' => [ - __DIR__ . '/../../_data/languages.csv', - [ + 'headersFile' => __DIR__ . '/../../_data/languages.csv', + 'importedFiles' => [ __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv', ], - [ + 'columns' => [ 'name', 'id', ], - true, - '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'incremental' => true, + 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, ]; yield 'incremental-to-empty-table' => [ - __DIR__ . '/../../_data/languages-headers.csv', - [ + 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv', + 'importedFiles' => [ __DIR__ . '/../../_data/languages-without-headers-reordered.csv', __DIR__ . '/../../_data/languages-without-headers-reordered-incremental.csv', ], - [ + 'columns' => [ 'name', 'id', ], - true, - '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'incremental' => true, + 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, ]; } From 5353f2253714e5c663886c9bcdc010a7df21f1e3 Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Thu, 18 Apr 2024 19:06:49 +0200 Subject: [PATCH 6/9] Add assert columns order --- tests/Backend/CommonPart1/ImportExportCommonTest.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 02971aec2..e9a510159 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -468,6 +468,10 @@ public function testImportWithReorderedColumnsList( } $table = $this->_client->getTable($tableId); $this->assertNotEmpty($table['dataSizeBytes']); + $this->assertArrayHasKey('columns', $table); + $this->assertIsArray($table['columns']); + $this->assertEquals('id', reset($table['columns'])); + $this->assertEquals('name', next($table['columns'])); // check that table columns aren't reordered $this->assertLinesEqualsSorted( From aecc4b1b8c4c684a01e08854a2399d5319f517e4 Mon Sep 17 00:00:00 2001 From: Martin Jandl Date: Thu, 18 Apr 2024 19:07:22 +0200 Subject: [PATCH 7/9] Use heredoc for expected data --- .../CommonPart1/ImportExportCommonTest.php | 34 +++++++++++++++++-- 1 file changed, 31 insertions(+), 3 deletions(-) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index e9a510159..7cdd80002 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -498,7 +498,15 @@ public function tableImportReorderedData(): \Generator 'id', ], 'incremental' => false, - 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'expectedData' => << [ 'headersFile' => __DIR__ . '/../../_data/languages.csv', @@ -510,7 +518,17 @@ public function tableImportReorderedData(): \Generator 'id', ], 'incremental' => true, - 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'expectedData' => << [ 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv', @@ -523,7 +541,17 @@ public function tableImportReorderedData(): \Generator 'id', ], 'incremental' => true, - 'expectedData' => '"0","- unchecked -"' . PHP_EOL . '"1","english"' . PHP_EOL . '"11","finnish"' . PHP_EOL . '"24","french"' . PHP_EOL . '"26","czech"' . PHP_EOL . '"27","spanish"' . PHP_EOL . '"28","greek"' . PHP_EOL . '"id","name"' . PHP_EOL, + 'expectedData' => << Date: Fri, 19 Apr 2024 10:22:46 +0200 Subject: [PATCH 8/9] Fix new column test case --- .../CommonPart1/ImportExportCommonTest.php | 44 +++++++++++++++++-- ...s-without-headers-reordered-new-column.csv | 5 +++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 tests/_data/languages-without-headers-reordered-new-column.csv diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 7cdd80002..4be4cbafd 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -449,6 +449,7 @@ public function testImportWithReorderedColumnsList( array $importedFiles, array $columns, bool $incremental, + array $expectedColumns, string $expectedData ): void { $headersCsv = new CsvFile($headersFile); @@ -470,8 +471,7 @@ public function testImportWithReorderedColumnsList( $this->assertNotEmpty($table['dataSizeBytes']); $this->assertArrayHasKey('columns', $table); $this->assertIsArray($table['columns']); - $this->assertEquals('id', reset($table['columns'])); - $this->assertEquals('name', next($table['columns'])); + $this->assertTrue($expectedColumns === $table['columns']); // check that table columns aren't reordered $this->assertLinesEqualsSorted( @@ -484,10 +484,36 @@ public function testImportWithReorderedColumnsList( } /** - * @return \Generator, columns: array, incremental: bool, expectedData: string}> + * @return \Generator, columns: array, incremental: bool, expectedData: string, expectedColumns: array}> */ public function tableImportReorderedData(): \Generator { + yield 'new-column' => [ + 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv', + 'importedFiles' => [ + __DIR__ . '/../../_data/languages-without-headers-reordered-new-column.csv', + ], + 'columns' => [ + 'name', + 'id', + 'code', + ], + 'incremental' => false, + 'expectedColumns' => [ + 'id', + 'name', + 'code', + ], + 'expectedData' => << [ 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv', 'importedFiles' => [ @@ -498,6 +524,10 @@ public function tableImportReorderedData(): \Generator 'id', ], 'incremental' => false, + 'expectedColumns' => [ + 'id', + 'name', + ], 'expectedData' => << true, + 'expectedColumns' => [ + 'id', + 'name', + ], 'expectedData' => << true, + 'expectedColumns' => [ + 'id', + 'name', + ], 'expectedData' => << Date: Fri, 19 Apr 2024 11:45:25 +0200 Subject: [PATCH 9/9] Skip test new columns import on BQ --- tests/Backend/CommonPart1/ImportExportCommonTest.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/Backend/CommonPart1/ImportExportCommonTest.php b/tests/Backend/CommonPart1/ImportExportCommonTest.php index 4be4cbafd..a0feb01f9 100644 --- a/tests/Backend/CommonPart1/ImportExportCommonTest.php +++ b/tests/Backend/CommonPart1/ImportExportCommonTest.php @@ -450,8 +450,13 @@ public function testImportWithReorderedColumnsList( array $columns, bool $incremental, array $expectedColumns, - string $expectedData + string $expectedData, + array $skipBackends = [], + string $skipMessage = '' ): void { + if (!empty($skipBackends)) { + $this->skipTestForBackend($skipBackends, $skipMessage); + } $headersCsv = new CsvFile($headersFile); $tableId = $this->_client->createTableAsync($this->getTestBucketId(), 'languages', $headersCsv); @@ -513,6 +518,10 @@ public function tableImportReorderedData(): \Generator "26","czech","cz" "id","name","code" END, + 'skipBackends' => [ + self::BACKEND_BIGQUERY, + ], + 'skipMessage' => 'Don\'t test new columns on BigQuery.', ]; yield 'full' => [ 'headersFile' => __DIR__ . '/../../_data/languages-headers.csv',