Skip to content

Commit

Permalink
Merge pull request #447 from keboola/erik-KBC-535-table-exporter
Browse files Browse the repository at this point in the history 
TableExporter: auto columns add to export params
  • Loading branch information
@ErikZigo
ErikZigo authored Jul 20, 2020
2 parents 7eb4389 + 66d9038 commit 1eccd06
Show file tree
Hide file tree
Showing 2 changed files with 108 additions and 11 deletions.
5 changes: 5 additions & 0 deletions src/Keboola/StorageApi/TableExporter.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,11 @@ public function exportTables(array $tables = array())
if (!isset($table['exportOptions'])) {
$table['exportOptions'] = array();
}
if (empty($table['exportOptions']['columns'])) {
$tableDetail = $this->client->getTable($table['tableId']);
$table['exportOptions']['columns'] = $tableDetail['columns'];
}

$jobId = $this->client->queueTableExport($table['tableId'], $table['exportOptions']);
$exportJobs[$jobId] = $table;
}
Expand Down
114 changes: 103 additions & 11 deletions tests/Backend/CommonPart1/TableExporterTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public function setUp()
}

/**
* @dataProvider tableImportData
* @dataProvider tableExportData
* @param $importFileName
*/
public function testTableAsyncExport(array $supportedBackends, CsvFile $importFile, $expectationsFileName, $exportOptions = array())
Expand All @@ -49,8 +49,19 @@ public function testTableAsyncExport(array $supportedBackends, CsvFile $importFi
}

$tableId = $this->_client->createTableAsync($this->getTestBucketId(self::STAGE_IN), 'languages', $importFile);

$runId = $this->_client->generateRunId();
$this->_client->setRunId($runId);

$exporter = new TableExporter($this->_client);

if (isset($exportOptions['columns'])) {
$expectedColumns = $exportOptions['columns'];
} else {
$table = $this->_client->getTable($tableId);
$expectedColumns = $table['columns'];
}

if ($exportOptions['gzip'] === true) {
$exporter->exportTable($tableId, $this->downloadPathGZip, $exportOptions);
if (file_exists($this->downloadPath)) {
Expand All @@ -68,6 +79,16 @@ public function testTableAsyncExport(array $supportedBackends, CsvFile $importFi
// compare data
$this->assertTrue(file_exists($this->downloadPath));
$this->assertLinesEqualsSorted(file_get_contents($expectationsFile), file_get_contents($this->downloadPath), 'imported data comparison');

// check that columns has been set in export job params
$jobs = $this->_client->listJobs(['limit' => 1]);
$job = reset($jobs);

$this->assertSame($runId, $job['runId']);
$this->assertSame('tableExport', $job['operationName']);
$this->assertSame($tableId, $job['tableId']);
$this->assertNotEmpty($job['operationParams']['export']['columns']);
$this->assertSame($expectedColumns, $job['operationParams']['export']['columns']);
}

public function testLimitParameter()
Expand All @@ -87,31 +108,102 @@ public function testLimitParameter()
}


public function testExportTables()
public function testExportTablesEmptyColumns()
{
$filesBasePath = __DIR__ . '/../../_data/';
$table1Id = $this->_client->createTableAsync($this->getTestBucketId(self::STAGE_IN), 'languages1', new CsvFile($filesBasePath . 'languages.csv'));
$table2Id = $this->_client->createTableAsync($this->getTestBucketId(self::STAGE_IN), 'languages2', new CsvFile($filesBasePath . 'languages.csv'));

$table1 = $this->_client->getTable($table1Id);
$table2 = $this->_client->getTable($table2Id);

$runId = $this->_client->generateRunId();
$this->_client->setRunId($runId);

$exporter = new TableExporter($this->_client);
$file1 = __DIR__ . '/../../_tmp/languages1.csv';
$file2 = __DIR__ . '/../../_tmp/languages2.csv';
$exports = array(
array(
$exports = [
[
'tableId' => $table1Id,
'destination' => $file1
),
array(
'destination' => $file1,
],
[
'tableId' => $table2Id,
'destination' => $file2
)
'destination' => $file2,
'columns' => [],
]

);
];
$exporter->exportTables($exports);
// compare data
$this->assertTrue(file_exists($file1));
$this->assertTrue(file_exists($file2));
$this->assertLinesEqualsSorted(file_get_contents($filesBasePath . 'languages.csv'), file_get_contents($file1), 'imported data comparison');
$this->assertLinesEqualsSorted(file_get_contents($filesBasePath . 'languages.csv'), file_get_contents($file2), 'imported data comparison');

// check that columns has been set in export job params
$table1Job = null;
$table2Job = null;
foreach ($this->_client->listJobs(['limit' => 2]) as $job) {
$this->assertSame($runId, $job['runId']);
$this->assertSame('tableExport', $job['operationName']);

if ($job['tableId'] === $table1Id) {
$table1Job = $job;
}
if ($job['tableId'] === $table2Id) {
$table2Job = $job;
}
}

$this->assertNotEmpty($table1Job['operationParams']['export']['columns']);
$this->assertSame($table1['columns'], $table1Job['operationParams']['export']['columns']);

$this->assertNotEmpty($table2Job['operationParams']['export']['columns']);
$this->assertSame($table2['columns'], $table2Job['operationParams']['export']['columns']);
}

public function testExportTablesWithColumns()
{
$filesBasePath = __DIR__ . '/../../_data/';
$table1Id = $this->_client->createTableAsync($this->getTestBucketId(self::STAGE_IN), 'languages1', new CsvFile($filesBasePath . 'languages.csv'));

$runId = $this->_client->generateRunId();
$this->_client->setRunId($runId);

$exporter = new TableExporter($this->_client);
$file1 = __DIR__ . '/../../_tmp/languages1.csv';
$exports = [
[
'tableId' => $table1Id,
'destination' => $file1,
'exportOptions' => [
'limit' => 1,
'columns' => ['name', 'id'],
],
],
];
$exporter->exportTables($exports);

// compare data
$this->assertTrue(file_exists($file1));

$csvFile = new \Keboola\Csv\CsvFile($file1);
$this->assertEquals(['name', 'id'], $csvFile->getHeader());

$csvFile->next();
$this->assertEquals(['- unchecked -', '0'], $csvFile->current());

// check that columns has been set in export job params
$jobs = $this->_client->listJobs(['limit' => 1]);
$table1Job = reset($jobs);

$this->assertSame($runId, $table1Job['runId']);
$this->assertSame('tableExport', $table1Job['operationName']);
$this->assertSame($table1Id, $table1Job['tableId']);
$this->assertNotEmpty($table1Job['operationParams']['export']['columns']);
$this->assertSame(['name', 'id'], $table1Job['operationParams']['export']['columns']);
}

public function testExportTablesMissingTableId()
Expand Down Expand Up @@ -169,7 +261,7 @@ public function testExportTablesInvalidTable()
}
}

public function tableImportData()
public function tableExportData()
{
$filesBasePath = __DIR__ . '/../../_data/';
return array(
Expand Down

0 comments on commit 1eccd06

Please sign in to comment.