Skip to content

Commit

Permalink
Merge pull request #1324 from keboola/martinj-ct-1361-default-value-f…
Browse files Browse the repository at this point in the history 
…or-null-conversion-in-bq

CT-1361 default value for null conversion in bq
  • Loading branch information
@martinjandl
martinjandl authored May 3, 2024
2 parents a7e249b + 7770325 commit 93227d2
Show file tree
Hide file tree
Showing 4 changed files with 314 additions and 1 deletion.
154 changes: 154 additions & 0 deletions tests/Backend/Bigquery/ImportTreatValuesAsNullTest.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,154 @@
<?php

namespace Keboola\Test\Backend\Bigquery;

use Generator;
use Keboola\Csv\CsvFile;
use Keboola\StorageApi\Workspaces;
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;
use Keboola\Test\Backend\Workspaces\ParallelWorkspacesTestCase;
use PDO;

class ImportTreatValuesAsNullTest extends ParallelWorkspacesTestCase
{
/**
* @dataProvider treatValuesAsNullData
*/
public function testImportTreatValuesAsNull(
string $importedFile,
array $expectedData,
?array $treatValuesAsNull = null
): void {
$this->allowTestForBackendsOnly([
self::BACKEND_BIGQUERY,
]);
$tableName = 'testImportTreatValuesAsNull';
$tableId = $this->createTable($tableName);

$params = [];
if ($treatValuesAsNull !== null) {
$params = ['treatValuesAsNull' => $treatValuesAsNull,];
}
$this->_client->writeTableAsync($tableId, new CsvFile($importedFile), $params);

$workspaces = new Workspaces($this->workspaceSapiClient);
$workspace = $workspaces->createWorkspace([], true);
$backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);

$options = [
'input' => [
[
'source' => $tableId,
'destination' => $tableName,
'useView' => true,
],
],
'preserve' => false,
];
$workspaces->loadWorkspaceData($workspace['id'], $options);
$data = $backend->fetchAll($tableName, PDO::FETCH_NUM, 'Id');
$data = array_map(function ($row) {
unset($row[4]); // remove timestamp
return $row;
}, $data);

$this->assertEqualsCanonicalizing($expectedData, $data);
}

/**
* @return Generator<string, array{importedFile: string, expectedData: array{array{mixed}}, treatValuesAsNull?: array<string>|null}>
*/
public function treatValuesAsNullData(): Generator
{
yield 'empty-array' => [
'importedFile' => __DIR__ . '/../../_data/languages-empty-string.csv',
'expectedData' => [
0 => [
0 => 30,
1 => 'armenia',
2 => '',
3 => 'b',
],
1 => [
0 => 31,
1 => 'belarus',
2 => '',
3 => 'b',
],
2 => [
0 => 32,
1 => 'malta',
2 => 'a',
3 => 'b',
],
],
'treatValuesAsNull' => [],
];
yield 'default' => [
'importedFile' => __DIR__ . '/../../_data/languages-empty-string.csv',
'expectedData' => [
0 => [
0 => 30,
1 => 'armenia',
2 => null,
3 => 'b',
],
1 => [
0 => 31,
1 => 'belarus',
2 => '',
3 => 'b',
],
2 => [
0 => 32,
1 => 'malta',
2 => 'a',
3 => 'b',
],
],
];
}

private function createTable(string $tableName): string
{
$bucketId = $this->getTestBucketId();
$data = [
'name' => $tableName,
'primaryKeysNames' => ['Id'],
'columns' => [
[
'name' => 'Id',
'definition' => [
'type' => 'INT64',
'nullable' => false,
],
],
[
'name' => 'Name',
'definition' => [
'type' => 'STRING',
],
],
[
'name' => 'iso',
'definition' => [
'type' => 'STRING',
'nullable' => true,
],
],
[
'name' => 'Something',
'definition' => [
'type' => 'STRING',
'nullable' => true,
],
],
],
];

$runId = $this->_client->generateRunId();
$this->_client->setRunId($runId);

return $this->_client->createTableDefinition($bucketId, $data);
}
}
2 changes: 1 addition & 1 deletion tests/Backend/CommonPart1/ImportExportCommonTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -761,7 +761,7 @@ public function testImportTreatValuesAsNull(): void
} catch (ClientException $e) {
$this->assertEquals('storage.tables.validation', $e->getStringCode());
$this->assertStringContainsString(
'This collection should contain exactly 1 element.',
'This collection should contain 1 element or less.',
$e->getMessage(),
);
}
Expand Down
155 changes: 155 additions & 0 deletions tests/Backend/Snowflake/ImportTreatValuesAsNullTest.php
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
<?php

namespace Keboola\Test\Backend\Snowflake;

use Generator;
use Keboola\Csv\CsvFile;
use Keboola\StorageApi\Workspaces;
use Keboola\Test\Backend\Workspaces\Backend\WorkspaceBackendFactory;
use Keboola\Test\Backend\Workspaces\ParallelWorkspacesTestCase;
use PDO;

class ImportTreatValuesAsNullTest extends ParallelWorkspacesTestCase
{
/**
* @dataProvider treatValuesAsNullData
*/
public function testImportTreatValuesAsNull(
string $importedFile,
array $expectedData,
?array $treatValuesAsNull = null
): void {
$this->allowTestForBackendsOnly([
self::BACKEND_SNOWFLAKE,
]);
$tableName = 'testImportTreatValuesAsNull';
$tableId = $this->createTable($tableName);

$params = [];
if ($treatValuesAsNull !== null) {
$params = ['treatValuesAsNull' => $treatValuesAsNull,];
}
$this->_client->writeTableAsync($tableId, new CsvFile($importedFile), $params);

$workspaces = new Workspaces($this->workspaceSapiClient);
$workspace = $workspaces->createWorkspace([], true);
$backend = WorkspaceBackendFactory::createWorkspaceBackend($workspace);

$options = [
'input' => [
[
'source' => $tableId,
'destination' => $tableName,
'useView' => true,
],
],
'preserve' => false,
];
$workspaces->loadWorkspaceData($workspace['id'], $options);

$data = $backend->fetchAll($tableName);
$data = array_map(function ($row) {
unset($row[4]); // remove timestamp
return $row;
}, $data);

$this->assertEqualsCanonicalizing($expectedData, $data);
}

/**
* @return Generator<string, array{importedFile: string, expectedData: array{array{mixed}}, treatValuesAsNull?: array<string>|null}>
*/
public function treatValuesAsNullData(): Generator
{
yield 'empty-array' => [
'importedFile' => __DIR__ . '/../../_data/languages-empty-string.csv',
'expectedData' => [
0 => [
0 => '30',
1 => 'armenia',
2 => null,
3 => 'b',
],
1 => [
0 => '31',
1 => 'belarus',
2 => '',
3 => 'b',
],
2 => [
0 => '32',
1 => 'malta',
2 => 'a',
3 => 'b',
],
],
'treatValuesAsNull' => [],
];
yield 'default' => [
'importedFile' => __DIR__ . '/../../_data/languages-empty-string.csv',
'expectedData' => [
0 => [
0 => '30',
1 => 'armenia',
2 => null,
3 => 'b',
],
1 => [
0 => '31',
1 => 'belarus',
2 => null,
3 => 'b',
],
2 => [
0 => '32',
1 => 'malta',
2 => 'a',
3 => 'b',
],
],
];
}

private function createTable(string $tableName): string
{
$bucketId = $this->getTestBucketId(self::STAGE_IN);
$data = [
'name' => $tableName,
'primaryKeysNames' => ['Id'],
'columns' => [
[
'name' => 'Id',
'definition' => [
'type' => 'INT',
'nullable' => false,
],
],
[
'name' => 'Name',
'definition' => [
'type' => 'STRING',
],
],
[
'name' => 'iso',
'definition' => [
'type' => 'STRING',
'nullable' => true,
],
],
[
'name' => 'Something',
'definition' => [
'type' => 'STRING',
'nullable' => true,
],
],
],
];

$runId = $this->_client->generateRunId();
$this->_client->setRunId($runId);

return $this->_client->createTableDefinition($bucketId, $data);
}
}
4 changes: 4 additions & 0 deletions tests/_data/languages-empty-string.csv
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
"Id","Name","iso","Something"
"30","armenia",,"b"
"31","belarus","","b"
"32","malta","a","b"

0 comments on commit 93227d2

Please sign in to comment.