From dce4e73e5cd669bbe53960b814313f1d25375105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 16:21:35 +0100 Subject: [PATCH 1/8] install google sdk & update backup lib --- composer.json | 3 +- composer.lock | 370 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 358 insertions(+), 15 deletions(-) diff --git a/composer.json b/composer.json index 41119da..892adb9 100644 --- a/composer.json +++ b/composer.json @@ -4,7 +4,8 @@ "license": "MIT", "require": { "php": "^8.2", - "keboola/kbc-project-backup": "^1.13", + "google/apiclient": "^2.18", + "keboola/kbc-project-backup": "dev-PST-2374-ondra", "keboola/php-component": "^10.1", "keboola/php-file-storage-utils": "^0.2.6", "microsoft/azure-storage-blob": "^1.5" diff --git a/composer.lock b/composer.lock index e70a36e..76e864d 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "47d78d9aac0d0465e861af0b379c523d", + "content-hash": "c2bc9454cad7592a8009707cd6ebcaf8", "packages": [ { "name": "aws/aws-crt-php", @@ -281,6 +281,119 @@ }, "time": "2024-05-18T18:05:11+00:00" }, + { + "name": "google/apiclient", + "version": "v2.18.2", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-api-php-client.git", + "reference": "d8d201ba8a189a3cd7fb34e4da569f2ed440eee7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-api-php-client/zipball/d8d201ba8a189a3cd7fb34e4da569f2ed440eee7", + "reference": "d8d201ba8a189a3cd7fb34e4da569f2ed440eee7", + "shasum": "" + }, + "require": { + "firebase/php-jwt": "^6.0", + "google/apiclient-services": "~0.350", + "google/auth": "^1.37", + "guzzlehttp/guzzle": "^7.4.5", + "guzzlehttp/psr7": "^2.6", + "monolog/monolog": "^2.9||^3.0", + "php": "^8.0", + "phpseclib/phpseclib": "^3.0.36" + }, + "require-dev": { + "cache/filesystem-adapter": "^1.1", + "composer/composer": "^1.10.23", + "phpcompatibility/php-compatibility": "^9.2", + "phpspec/prophecy-phpunit": "^2.1", + "phpunit/phpunit": "^9.6", + "squizlabs/php_codesniffer": "^3.8", + "symfony/css-selector": "~2.1", + "symfony/dom-crawler": "~2.1" + }, + "suggest": { + "cache/filesystem-adapter": "For caching certs and tokens (using Google\\Client::setCache)" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-main": "2.x-dev" + } + }, + "autoload": { + "files": [ + "src/aliases.php" + ], + "psr-4": { + "Google\\": "src/" + }, + "classmap": [ + "src/aliases.php" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Client library for Google APIs", + "homepage": "http://developers.google.com/api-client-library/php", + "keywords": [ + "google" + ], + "support": { + "issues": "https://github.com/googleapis/google-api-php-client/issues", + "source": "https://github.com/googleapis/google-api-php-client/tree/v2.18.2" + }, + "time": "2024-12-16T22:52:40+00:00" + }, + { + "name": "google/apiclient-services", + "version": "v0.390.0", + "source": { + "type": "git", + "url": "https://github.com/googleapis/google-api-php-client-services.git", + "reference": "2c1ff37aea15dd9e7a434c4fcbec777d9421385c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/googleapis/google-api-php-client-services/zipball/2c1ff37aea15dd9e7a434c4fcbec777d9421385c", + "reference": "2c1ff37aea15dd9e7a434c4fcbec777d9421385c", + "shasum": "" + }, + "require": { + "php": "^8.0" + }, + "require-dev": { + "phpunit/phpunit": "^9.6" + }, + "type": "library", + "autoload": { + "files": [ + "autoload.php" + ], + "psr-4": { + "Google\\Service\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "description": "Client library for Google APIs", + "homepage": "http://developers.google.com/api-client-library/php", + "keywords": [ + "google" + ], + "support": { + "issues": "https://github.com/googleapis/google-api-php-client-services/issues", + "source": "https://github.com/googleapis/google-api-php-client-services/tree/v0.390.0" + }, + "time": "2025-01-12T01:00:53+00:00" + }, { "name": "google/auth", "version": "v1.42.0", @@ -1216,16 +1329,16 @@ }, { "name": "keboola/kbc-project-backup", - "version": "1.13.0", + "version": "dev-PST-2374-ondra", "source": { "type": "git", "url": "https://github.com/keboola/php-kbc-project-backup.git", - "reference": "38a5214a202e3c3a7214c198a64797326cb9a8ee" + "reference": "bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/keboola/php-kbc-project-backup/zipball/38a5214a202e3c3a7214c198a64797326cb9a8ee", - "reference": "38a5214a202e3c3a7214c198a64797326cb9a8ee", + "url": "https://api.github.com/repos/keboola/php-kbc-project-backup/zipball/bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9", + "reference": "bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9", "shasum": "" }, "require": { @@ -1271,9 +1384,9 @@ "description": "Backup KBC project", "support": { "issues": "https://github.com/keboola/php-kbc-project-backup/issues", - "source": "https://github.com/keboola/php-kbc-project-backup/tree/1.13.0" + "source": "https://github.com/keboola/php-kbc-project-backup/tree/PST-2374-ondra" }, - "time": "2025-01-11T21:27:54+00:00" + "time": "2025-01-13T22:13:52+00:00" }, { "name": "keboola/notification-api-php-client", @@ -1947,6 +2060,233 @@ ], "time": "2022-08-04T09:53:51+00:00" }, + { + "name": "paragonie/constant_time_encoding", + "version": "v3.0.0", + "source": { + "type": "git", + "url": "https://github.com/paragonie/constant_time_encoding.git", + "reference": "df1e7fde177501eee2037dd159cf04f5f301a512" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/constant_time_encoding/zipball/df1e7fde177501eee2037dd159cf04f5f301a512", + "reference": "df1e7fde177501eee2037dd159cf04f5f301a512", + "shasum": "" + }, + "require": { + "php": "^8" + }, + "require-dev": { + "phpunit/phpunit": "^9", + "vimeo/psalm": "^4|^5" + }, + "type": "library", + "autoload": { + "psr-4": { + "ParagonIE\\ConstantTime\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com", + "role": "Maintainer" + }, + { + "name": "Steve 'Sc00bz' Thomas", + "email": "steve@tobtu.com", + "homepage": "https://www.tobtu.com", + "role": "Original Developer" + } + ], + "description": "Constant-time Implementations of RFC 4648 Encoding (Base-64, Base-32, Base-16)", + "keywords": [ + "base16", + "base32", + "base32_decode", + "base32_encode", + "base64", + "base64_decode", + "base64_encode", + "bin2hex", + "encoding", + "hex", + "hex2bin", + "rfc4648" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/constant_time_encoding/issues", + "source": "https://github.com/paragonie/constant_time_encoding" + }, + "time": "2024-05-08T12:36:18+00:00" + }, + { + "name": "paragonie/random_compat", + "version": "v9.99.100", + "source": { + "type": "git", + "url": "https://github.com/paragonie/random_compat.git", + "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/paragonie/random_compat/zipball/996434e5492cb4c3edcb9168db6fbb1359ef965a", + "reference": "996434e5492cb4c3edcb9168db6fbb1359ef965a", + "shasum": "" + }, + "require": { + "php": ">= 7" + }, + "require-dev": { + "phpunit/phpunit": "4.*|5.*", + "vimeo/psalm": "^1" + }, + "suggest": { + "ext-libsodium": "Provides a modern crypto API that can be used to generate random bytes." + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Paragon Initiative Enterprises", + "email": "security@paragonie.com", + "homepage": "https://paragonie.com" + } + ], + "description": "PHP 5.x polyfill for random_bytes() and random_int() from PHP 7", + "keywords": [ + "csprng", + "polyfill", + "pseudorandom", + "random" + ], + "support": { + "email": "info@paragonie.com", + "issues": "https://github.com/paragonie/random_compat/issues", + "source": "https://github.com/paragonie/random_compat" + }, + "time": "2020-10-15T08:29:30+00:00" + }, + { + "name": "phpseclib/phpseclib", + "version": "3.0.43", + "source": { + "type": "git", + "url": "https://github.com/phpseclib/phpseclib.git", + "reference": "709ec107af3cb2f385b9617be72af8cf62441d02" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/phpseclib/phpseclib/zipball/709ec107af3cb2f385b9617be72af8cf62441d02", + "reference": "709ec107af3cb2f385b9617be72af8cf62441d02", + "shasum": "" + }, + "require": { + "paragonie/constant_time_encoding": "^1|^2|^3", + "paragonie/random_compat": "^1.4|^2.0|^9.99.99", + "php": ">=5.6.1" + }, + "require-dev": { + "phpunit/phpunit": "*" + }, + "suggest": { + "ext-dom": "Install the DOM extension to load XML formatted public keys.", + "ext-gmp": "Install the GMP (GNU Multiple Precision) extension in order to speed up arbitrary precision integer arithmetic operations.", + "ext-libsodium": "SSH2/SFTP can make use of some algorithms provided by the libsodium-php extension.", + "ext-mcrypt": "Install the Mcrypt extension in order to speed up a few other cryptographic operations.", + "ext-openssl": "Install the OpenSSL extension in order to speed up a wide variety of cryptographic operations." + }, + "type": "library", + "autoload": { + "files": [ + "phpseclib/bootstrap.php" + ], + "psr-4": { + "phpseclib3\\": "phpseclib/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Jim Wigginton", + "email": "terrafrost@php.net", + "role": "Lead Developer" + }, + { + "name": "Patrick Monnerat", + "email": "pm@datasphere.ch", + "role": "Developer" + }, + { + "name": "Andreas Fischer", + "email": "bantu@phpbb.com", + "role": "Developer" + }, + { + "name": "Hans-Jürgen Petrich", + "email": "petrich@tronic-media.com", + "role": "Developer" + }, + { + "name": "Graham Campbell", + "email": "graham@alt-three.com", + "role": "Developer" + } + ], + "description": "PHP Secure Communications Library - Pure-PHP implementations of RSA, AES, SSH2, SFTP, X.509 etc.", + "homepage": "http://phpseclib.sourceforge.net", + "keywords": [ + "BigInteger", + "aes", + "asn.1", + "asn1", + "blowfish", + "crypto", + "cryptography", + "encryption", + "rsa", + "security", + "sftp", + "signature", + "signing", + "ssh", + "twofish", + "x.509", + "x509" + ], + "support": { + "issues": "https://github.com/phpseclib/phpseclib/issues", + "source": "https://github.com/phpseclib/phpseclib/tree/3.0.43" + }, + "funding": [ + { + "url": "https://github.com/terrafrost", + "type": "github" + }, + { + "url": "https://www.patreon.com/phpseclib", + "type": "patreon" + }, + { + "url": "https://tidelift.com/funding/github/packagist/phpseclib/phpseclib", + "type": "tidelift" + } + ], + "time": "2024-12-14T21:12:59+00:00" + }, { "name": "psr/cache", "version": "3.0.0", @@ -3214,16 +3554,16 @@ }, { "name": "symfony/process", - "version": "v6.4.8", + "version": "v6.4.15", "source": { "type": "git", "url": "https://github.com/symfony/process.git", - "reference": "8d92dd79149f29e89ee0f480254db595f6a6a2c5" + "reference": "3cb242f059c14ae08591c5c4087d1fe443564392" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/process/zipball/8d92dd79149f29e89ee0f480254db595f6a6a2c5", - "reference": "8d92dd79149f29e89ee0f480254db595f6a6a2c5", + "url": "https://api.github.com/repos/symfony/process/zipball/3cb242f059c14ae08591c5c4087d1fe443564392", + "reference": "3cb242f059c14ae08591c5c4087d1fe443564392", "shasum": "" }, "require": { @@ -3255,7 +3595,7 @@ "description": "Executes commands in sub-processes", "homepage": "https://symfony.com", "support": { - "source": "https://github.com/symfony/process/tree/v6.4.8" + "source": "https://github.com/symfony/process/tree/v6.4.15" }, "funding": [ { @@ -3271,7 +3611,7 @@ "type": "tidelift" } ], - "time": "2024-05-31T14:49:08+00:00" + "time": "2024-11-06T14:19:14+00:00" }, { "name": "symfony/property-access", @@ -6086,7 +6426,9 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": [], + "stability-flags": { + "keboola/kbc-project-backup": 20 + }, "prefer-stable": false, "prefer-lowest": false, "platform": { From f0912b19919d94fee072006738301cbc85b2912e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 16:21:56 +0100 Subject: [PATCH 2/8] GCS Config --- src/Config/Config.php | 7 ++++++ src/Config/ConfigDefinition.php | 12 ++++++++++ src/Config/GcsConfig.php | 39 +++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+) create mode 100644 src/Config/GcsConfig.php diff --git a/src/Config/Config.php b/src/Config/Config.php index 48da2bd..4ea6b91 100644 --- a/src/Config/Config.php +++ b/src/Config/Config.php @@ -12,6 +12,8 @@ class Config extends BaseConfig public const STORAGE_BACKEND_ABS = 'abs'; + public const STORAGE_BACKEND_GCS = 'gcs'; + public function getBackupId(): string { return $this->getStringValue(['parameters', 'backupId'], ''); @@ -71,4 +73,9 @@ public function getAbsConfig(): AbsConfig { return new AbsConfig($this->getCredentialsParameters()); } + + public function getGcsConfig(): GcsConfig + { + return new GcsConfig($this->getCredentialsParameters(), $this->isUserDefinedCredentials()); + } } diff --git a/src/Config/ConfigDefinition.php b/src/Config/ConfigDefinition.php index 8a43e34..10618e8 100644 --- a/src/Config/ConfigDefinition.php +++ b/src/Config/ConfigDefinition.php @@ -43,6 +43,16 @@ protected function getParametersDefinition(): ArrayNodeDefinition ->validate()->always(function ($v) { if (!empty($v['storageBackendType'])) { switch ($v['storageBackendType']) { + case Config::STORAGE_BACKEND_GCS: + foreach (['#jsonKey', '#bucket', 'region'] as $item) { + if (empty($v[$item])) { + throw new InvalidConfigurationException(sprintf( + 'Missing required parameter "%s".', + $item, + )); + } + } + break; case Config::STORAGE_BACKEND_ABS: foreach (['backupPath', 'accountName', '#accountKey'] as $item) { if (empty($v[$item])) { @@ -94,6 +104,8 @@ protected function getParametersDefinition(): ArrayNodeDefinition ->scalarNode('access_key_id')->end() ->scalarNode('#secret_access_key')->end() ->scalarNode('#bucket')->end() + ->scalarNode('#jsonKey')->end() + ->scalarNode('region')->end() ->end() ; // @formatter:on diff --git a/src/Config/GcsConfig.php b/src/Config/GcsConfig.php new file mode 100644 index 0000000..5bac6af --- /dev/null +++ b/src/Config/GcsConfig.php @@ -0,0 +1,39 @@ +jsonKey = $params['#jsonKey']; + $this->bucket = $params['#bucket']; + $this->region = $params['region']; + } + + public function getJsonKey(): string + { + return $this->jsonKey; + } + + public function getBucket(): string + { + return $this->bucket; + } + + public function getRegion(): string + { + return $this->region; + } + + public function isUserDefinedCredentials(): bool + { + return $this->isUserDefinedCredentials; + } +} From 518f24868f6d4fff997b664b4c86c2a3583eb733 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 16:22:20 +0100 Subject: [PATCH 3/8] backup to google storage --- src/Application.php | 9 ++++++ src/Storages/GoogleCloudStorage.php | 45 +++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 src/Storages/GoogleCloudStorage.php diff --git a/src/Application.php b/src/Application.php index da7b076..573784e 100644 --- a/src/Application.php +++ b/src/Application.php @@ -8,8 +8,10 @@ use Keboola\App\ProjectBackup\Config\Config; use Keboola\App\ProjectBackup\Storages\AwsS3Storage; use Keboola\App\ProjectBackup\Storages\AzureBlobStorage; +use Keboola\App\ProjectBackup\Storages\GoogleCloudStorage; use Keboola\App\ProjectBackup\Storages\IStorage; use Keboola\Component\UserException; +use Keboola\ProjectBackup\GcsBackup; use Keboola\StorageApi\Client as StorageApi; use Psr\Log\LoggerInterface; @@ -37,6 +39,9 @@ public function __construct(Config $config, LoggerInterface $logger) case Config::STORAGE_BACKEND_ABS: $this->storageBackend = new AzureBlobStorage($config->getAbsConfig(), $logger); break; + case Config::STORAGE_BACKEND_GCS: + $this->storageBackend = new GoogleCloudStorage($config->getGcsConfig(), $logger); + break; default: throw new UserException(sprintf( 'Unknown storage backend type "%s".', @@ -73,6 +78,10 @@ public function run(): void $backup->backupTriggers(); $backup->backupNotifications(); $backup->backupPermanentFiles(); + + if ($backup instanceof GcsBackup && !$this->config->isUserDefinedCredentials()) { + $backup->backupSignedUrls(); + } } public function generateReadCredentials(): array diff --git a/src/Storages/GoogleCloudStorage.php b/src/Storages/GoogleCloudStorage.php new file mode 100644 index 0000000..44bb75b --- /dev/null +++ b/src/Storages/GoogleCloudStorage.php @@ -0,0 +1,45 @@ + json_decode($this->config->getJsonKey(), true), + ]); + + if (!str_ends_with($path, '/')) { + $path .= '/'; + } + + return new GcsBackup( + sapiClient: $sapi, + storageClient: $storageClient, + bucketName: $this->config->getBucket(), + path: $path, + generateSignedUrls: !$this->config->isUserDefinedCredentials(), + logger: $this->logger, + ); + } +} From 03d921089c2ad7b2ead1dd40c53f6eb9e43ed114 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 16:22:30 +0100 Subject: [PATCH 4/8] tests: backup to google storage --- tests/phpunit/FunctionalGCSTest.php | 262 ++++++++++++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 tests/phpunit/FunctionalGCSTest.php diff --git a/tests/phpunit/FunctionalGCSTest.php b/tests/phpunit/FunctionalGCSTest.php new file mode 100644 index 0000000..a594e03 --- /dev/null +++ b/tests/phpunit/FunctionalGCSTest.php @@ -0,0 +1,262 @@ +temp = new Temp('project-backup'); + + $this->sapiClient = new StorageApi([ + 'url' => getenv('TEST_GCP_STORAGE_API_URL'), + 'token' => getenv('TEST_GCP_STORAGE_API_TOKEN'), + ]); + + $this->cleanupKbcProject(); + + $this->storageClient = new StorageClient([ + 'keyFile' => json_decode((string) getenv('TEST_GCP_SERVICE_ACCOUNT'), true), + ]); + + $this->cleanupGCSBucket(); + + $component = new Components($this->sapiClient); + + $config = new Configuration(); + $config->setComponentId('keboola.snowflake-transformation'); + $config->setDescription('Test Configuration'); + $config->setConfigurationId('sapi-php-test'); + $config->setName('test-configuration'); + $component->addConfiguration($config); + + $this->testRunId = $this->sapiClient->generateRunId(); + } + + public function testSuccessfulRun(): void + { + // run backup + $fileSystem = new Filesystem(); + $fileSystem->dumpFile( + $this->temp->getTmpFolder() . '/config.json', + (string) json_encode([ + 'action' => 'run', + 'parameters' => [ + 'backupId' => $this->sapiClient->generateId(), + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => getenv('TEST_GCP_SERVICE_ACCOUNT'), + 'region' => getenv('TEST_GCP_REGION'), + '#bucket' => getenv('TEST_GCP_BUCKET'), + ], + ]), + ); + + $runProcess = $this->createTestProcess(); + $runProcess->mustRun(); + + $this->assertEmpty($runProcess->getErrorOutput()); + + $output = $runProcess->getOutput(); + $this->assertStringContainsString('Exporting buckets', $output); + $this->assertStringContainsString('Exporting tables', $output); + $this->assertStringContainsString('Exporting configurations', $output); + $this->assertStringContainsString('Exporting permanent files', $output); + + $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]); + self::assertGreaterThan(0, count($events)); + } + + public function testSuccessfulRunOnlyStructure(): void + { + $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]); + self::assertCount(0, $events); + + $tmp = new Temp(); + + $file = $tmp->createFile('testStructureOnly.csv'); + file_put_contents($file->getPathname(), 'a,b,c,d,e,f'); + + $csvFile = new CsvFile($file->getPathname()); + + $this->sapiClient->createBucket('test-bucket', 'out'); + $this->sapiClient->createTableAsync('out.c-test-bucket', 'test-table', $csvFile); + + // run backup + $fileSystem = new Filesystem(); + $fileSystem->dumpFile( + $this->temp->getTmpFolder() . '/config.json', + (string) json_encode([ + 'action' => 'run', + 'parameters' => [ + 'backupId' => $this->sapiClient->generateId(), + 'exportStructureOnly' => true, + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => getenv('TEST_GCP_SERVICE_ACCOUNT'), + 'region' => getenv('TEST_GCP_REGION'), + '#bucket' => getenv('TEST_GCP_BUCKET'), + ], + ]), + ); + + $runProcess = $this->createTestProcess(); + $runProcess->mustRun(); + + $this->assertEmpty($runProcess->getErrorOutput()); + + $output = $runProcess->getOutput(); + $this->assertStringContainsString('Exporting buckets', $output); + $this->assertStringContainsString('Exporting tables', $output); + $this->assertStringContainsString('Exporting configurations', $output); + $this->assertStringNotContainsString('Table ', $output); + + $events = $this->sapiClient->listEvents(['runId' => $this->testRunId]); + self::assertGreaterThan(0, count($events)); + } + + public function testCreateUnexistsBackupFolder(): void + { + $fileSystem = new Filesystem(); + $fileSystem->dumpFile( + $this->temp->getTmpFolder() . '/config.json', + (string) json_encode([ + 'action' => 'run', + 'parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => getenv('TEST_GCP_SERVICE_ACCOUNT'), + 'region' => getenv('TEST_GCP_REGION'), + '#bucket' => getenv('TEST_GCP_BUCKET'), + 'backupPath' => 'unexists/backup/folder', + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + 'access_key_id' => '', + '#secret_access_key' => '', + 'region' => '', + '#bucket' => '', + ], + ]), + ); + + $runProcess = $this->createTestProcess(); + $runProcess->run(); + + $this->assertEquals(0, $runProcess->getExitCode()); + + $objects = $this->storageClient->bucket((string) getenv('TEST_GCP_BUCKET'))->objects(); + + $files = []; + foreach ($objects as $object) { + $files[] = $object->name(); + } + + $this->assertNotEmpty($files); + } + + public function testRegionErrorRun(): void + { + $fileSystem = new Filesystem(); + $fileSystem->dumpFile( + $this->temp->getTmpFolder() . '/config.json', + (string) json_encode([ + 'action' => 'run', + 'parameters' => [ + 'backupId' => $this->sapiClient->generateId(), + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => getenv('TEST_GCP_SERVICE_ACCOUNT'), + 'region' => 'unknown-custom-region', + '#bucket' => getenv('TEST_GCP_BUCKET'), + ], + ]), + ); + + $runProcess = $this->createTestProcess(); + $runProcess->run(); + + $this->assertEquals(2, $runProcess->getExitCode()); + + $output = $runProcess->getOutput(); + $errorOutput = $runProcess->getErrorOutput(); + + $this->assertEmpty($output); + $this->assertStringContainsString('is not located in', $errorOutput); + $this->assertStringContainsString('unknown-custom-region', $errorOutput); + } + + private function cleanupKbcProject(): void + { + $components = new Components($this->sapiClient); + foreach ($components->listComponents() as $component) { + foreach ($component['configurations'] as $configuration) { + $components->deleteConfiguration($component['id'], $configuration['id']); + + // delete configuration from trash + $components->deleteConfiguration($component['id'], $configuration['id']); + } + } + + // drop linked buckets + foreach ($this->sapiClient->listBuckets() as $bucket) { + if (isset($bucket['sourceBucket'])) { + $this->sapiClient->dropBucket($bucket['id'], ['force' => true]); + } + } + + foreach ($this->sapiClient->listBuckets() as $bucket) { + $this->sapiClient->dropBucket($bucket['id'], ['force' => true]); + } + } + + private function createTestProcess(): Process + { + $runCommand = 'php /code/src/run.php'; + return Process::fromShellCommandline($runCommand, null, [ + 'KBC_DATADIR' => $this->temp->getTmpFolder(), + 'KBC_URL' => getenv('TEST_GCP_STORAGE_API_URL'), + 'KBC_TOKEN' => getenv('TEST_GCP_STORAGE_API_TOKEN'), + 'KBC_RUNID' => $this->testRunId, + ]); + } + + private function cleanupGCSBucket(): void + { + $objects = $this->storageClient->bucket((string) getenv('TEST_GCP_BUCKET'))->objects(); + + /** @var StorageObject $object */ + foreach ($objects as $object) { + $object->delete(); + } + } +} From 3aecad7e592a4f71afa80c4f7e669d6946e263d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 16:23:12 +0100 Subject: [PATCH 5/8] update GH workflow --- .github/workflows/push.yml | 16 ++++++++++++++++ .gitignore | 1 + docker-compose.yml | 5 +++++ 3 files changed, 22 insertions(+) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index e1ad74b..c4d03ec 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,18 +9,29 @@ env: KBC_DEVELOPERPORTAL_PASSWORD: ${{ secrets.KBC_DEVELOPERPORTAL_PASSWORD }} DOCKERHUB_USER: keboolabot DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + + # AWS Credentials TEST_STORAGE_API_URL: https://connection.eu-central-1.keboola.com/ TEST_STORAGE_API_TOKEN: ${{ secrets.TEST_STORAGE_API_TOKEN }} TEST_AWS_ACCESS_KEY_ID: AKIAQLZBTO5VMQJGVGNK TEST_AWS_SECRET_ACCESS_KEY: ${{ secrets.TEST_AWS_SECRET_ACCESS_KEY }} TEST_AWS_REGION: eu-central-1 TEST_AWS_S3_BUCKET: ci-app-project-backup-s3filesbucket-7qyiq6iqfrse + + # Azure Credentials TEST_AZURE_STORAGE_API_URL: https://connection.north-europe.azure.keboola.com/ TEST_AZURE_STORAGE_API_TOKEN: ${{ secrets.TEST_AZURE_STORAGE_API_TOKEN }} TEST_AZURE_ACCOUNT_NAME: ciappprojectbackup TEST_AZURE_ACCOUNT_KEY: ${{ secrets.TEST_AZURE_ACCOUNT_KEY }} TEST_AZURE_REGION: eu-west-1 + # GCP Credentials + TEST_GCP_STORAGE_API_URL: "https://connection.europe-west3.gcp.keboola.com/" + TEST_GCP_STORAGE_API_TOKEN: ${{ secrets.TEST_GCP_STORAGE_API_TOKEN }} + TEST_GCP_SERVICE_ACCOUNT: ${{ secrets.TEST_GCP_SERVICE_ACCOUNT }} + TEST_GCP_BUCKET: "ci-app-project-backup" + TEST_GCP_REGION: "europe-west3" + jobs: build: runs-on: ubuntu-latest @@ -92,6 +103,11 @@ jobs: -e TEST_AZURE_ACCOUNT_NAME -e TEST_AZURE_ACCOUNT_KEY -e TEST_AZURE_REGION + -e TEST_GCP_STORAGE_API_URL + -e TEST_GCP_STORAGE_API_TOKEN + -e TEST_GCP_BUCKET + -e TEST_GCP_REGION + -e TEST_GCP_SERVICE_ACCOUNT $APP_IMAGE composer ci deploy: diff --git a/.gitignore b/.gitignore index 2c39a5f..1ac07f1 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ /vendor /.idea /data +.phpunit.result.cache diff --git a/docker-compose.yml b/docker-compose.yml index 2b383ea..1e27bc9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,3 +20,8 @@ services: - TEST_AZURE_ACCOUNT_NAME - TEST_AZURE_ACCOUNT_KEY - TEST_AZURE_REGION + - TEST_GCP_STORAGE_API_URL + - TEST_GCP_STORAGE_API_TOKEN + - TEST_GCP_BUCKET + - TEST_GCP_REGION + - TEST_GCP_SERVICE_ACCOUNT From 0eb63084d434f6923ca1d9ddc97ef2a7c36d0dd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Tue, 21 Jan 2025 21:25:50 +0100 Subject: [PATCH 6/8] update keboola/kbc-project-backup --- composer.json | 2 +- composer.lock | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/composer.json b/composer.json index 892adb9..35c8257 100644 --- a/composer.json +++ b/composer.json @@ -5,7 +5,7 @@ "require": { "php": "^8.2", "google/apiclient": "^2.18", - "keboola/kbc-project-backup": "dev-PST-2374-ondra", + "keboola/kbc-project-backup": "^1.14", "keboola/php-component": "^10.1", "keboola/php-file-storage-utils": "^0.2.6", "microsoft/azure-storage-blob": "^1.5" diff --git a/composer.lock b/composer.lock index 76e864d..43814ea 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "c2bc9454cad7592a8009707cd6ebcaf8", + "content-hash": "ec6f4cacd8f36824022959a4171abcb4", "packages": [ { "name": "aws/aws-crt-php", @@ -1329,16 +1329,16 @@ }, { "name": "keboola/kbc-project-backup", - "version": "dev-PST-2374-ondra", + "version": "1.14.0", "source": { "type": "git", "url": "https://github.com/keboola/php-kbc-project-backup.git", - "reference": "bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9" + "reference": "e05038a058068698b5e04f1951da11ffef149798" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/keboola/php-kbc-project-backup/zipball/bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9", - "reference": "bbc5c24f84ef27bf9f95bddbe6dc882635a2d3d9", + "url": "https://api.github.com/repos/keboola/php-kbc-project-backup/zipball/e05038a058068698b5e04f1951da11ffef149798", + "reference": "e05038a058068698b5e04f1951da11ffef149798", "shasum": "" }, "require": { @@ -1384,9 +1384,9 @@ "description": "Backup KBC project", "support": { "issues": "https://github.com/keboola/php-kbc-project-backup/issues", - "source": "https://github.com/keboola/php-kbc-project-backup/tree/PST-2374-ondra" + "source": "https://github.com/keboola/php-kbc-project-backup/tree/1.14.0" }, - "time": "2025-01-13T22:13:52+00:00" + "time": "2025-01-21T20:08:54+00:00" }, { "name": "keboola/notification-api-php-client", @@ -6426,9 +6426,7 @@ ], "aliases": [], "minimum-stability": "stable", - "stability-flags": { - "keboola/kbc-project-backup": 20 - }, + "stability-flags": [], "prefer-stable": false, "prefer-lowest": false, "platform": { From 8158f37d1c6fbbb776fa2d752897ce6365c6a27d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Wed, 22 Jan 2025 10:05:21 +0100 Subject: [PATCH 7/8] update readme --- README.md | 53 ++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4e68130..fe71d5e 100644 --- a/README.md +++ b/README.md @@ -89,21 +89,31 @@ cd app-project-backup - `TEST_AZURE_STORAGE_API_URL` - KBC Storage with Blob Storage backend API endpoint - `TEST_AZURE_STORAGE_API_TOKEN` - KBC Storage with Blob Storage backend API token - `TEST_AZURE_ACCOUNT_` - Storage Account in your Azure Subscription + - `TEST_GCP_STORAGE_API_URL` - KBC Storage with Google backend API endpoint + - `TEST_GCP_STORAGE_API_TOKEN` - KBC Storage with Google backend API token + - `TEST_GCP_SERVICE_ACCOUNT` - Service account with permissions to write to the bucket + - `TEST_GCP_BUCKET` - Bucket in your Google Storage + - `TEST_GCP_REGION` - KBC Storage with Google backend region ``` +TEST_STORAGE_API_URL= +TEST_STORAGE_API_TOKEN= TEST_AWS_ACCESS_KEY_ID= TEST_AWS_SECRET_ACCESS_KEY= TEST_AWS_REGION= TEST_AWS_S3_BUCKET= -TEST_STORAGE_API_URL= -TEST_STORAGE_API_TOKEN= - -TEST_AZURE_REGION= TEST_AZURE_STORAGE_API_URL= TEST_AZURE_STORAGE_API_TOKEN= +TEST_AZURE_REGION= TEST_AZURE_ACCOUNT_NAME= TEST_AZURE_ACCOUNT_KEY= + +TEST_GCP_STORAGE_API_URL= +TEST_GCP_STORAGE_API_TOKEN= +TEST_GCP_SERVICE_ACCOUNT= +TEST_GCP_BUCKET= +TEST_GCP_REGION= ``` - Build Docker image @@ -156,6 +166,23 @@ docker-compose run --rm dev composer ci } ``` + +- GCP example (Generate Read Credentials action) +```json +{ + "action": "generate-read-credentials", + "parameters": { + "backupId": null + }, + "image_parameters": { + "storageBackendType": "gcs", + "#jsonKey": "", + "#bucket": "", + "region": "europe-west4" + } +} +``` + - AWS example (Backup action) ```json { @@ -189,7 +216,23 @@ docker-compose run --rm dev composer ci } } ``` - + + +- GCP example (Backup action) +```json +{ + "action": "run", + "parameters": { + "backupId": "backupId from `generate-read-credentials` action - if not set will be created" + }, + "image_parameters": { + "storageBackendType": "gcs", + "#jsonKey": "", + "#bucket": "", + "region": "europe-west4" + } +} +``` # Integration For information about deployment and integration with KBC, please refer to the [deployment section of developers documentation](https://developers.keboola.com/extend/component/deployment/) From 896748cfefdeec8552e0522e021eedbc5bd06368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ond=C5=99ej=20Jodas?= Date: Wed, 22 Jan 2025 10:19:24 +0100 Subject: [PATCH 8/8] add config tests --- tests/phpunit/ConfigTest.php | 63 ++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/tests/phpunit/ConfigTest.php b/tests/phpunit/ConfigTest.php index 982e8dc..3660552 100644 --- a/tests/phpunit/ConfigTest.php +++ b/tests/phpunit/ConfigTest.php @@ -356,5 +356,68 @@ public function invalidConfigDataProvider(): Generator ], 'The child node "backupId" at path "root.parameters" must be configured.', ]; + + yield 'gcs-missing-backupId' => [ + [ + 'action' => 'run', + 'parameters' => [ + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => 'testJsonKey', + '#bucket' => 'testBucket', + 'region' => 'testRegion', + ], + ], + 'The child node "backupId" at path "root.parameters" must be configured.', + ]; + + yield 'gcs-missing-jsonKey' => [ + [ + 'action' => 'run', + 'parameters' => [ + 'backupId' => 'testBackupId', + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#bucket' => 'testBucket', + 'region' => 'testRegion', + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + ], + ], + 'Missing required parameter "#jsonKey".', + ]; + + yield 'gcs-missing-bucket' => [ + [ + 'action' => 'run', + 'parameters' => [ + 'backupId' => 'testBackupId', + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => 'testJsonKey', + 'region' => 'testRegion', + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + ], + ], + 'Missing required parameter "#bucket".', + ]; + + yield 'gcs-missing-region' => [ + [ + 'action' => 'run', + 'parameters' => [ + 'backupId' => 'testBackupId', + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + '#jsonKey' => 'testJsonKey', + '#bucket' => 'testBucket', + ], + 'image_parameters' => [ + 'storageBackendType' => Config::STORAGE_BACKEND_GCS, + ], + ], + 'Missing required parameter "region".', + ]; } }