Refactor Cleanup class #47
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,6 +17,10 @@ | |
|
|
||
| class Cleanup | ||
| { | ||
| private const OWNERSHIP = 'OWNERSHIP'; | ||
| private const USER = 'USER'; | ||
| private const ROLE = 'ROLE'; | ||
|
|
||
| public function __construct( | ||
| readonly Config $config, | ||
| readonly Connection $sourceConnection, | ||
|
|
@@ -33,7 +37,14 @@ public function sourceAccount(): void | |
| $databaseRole = $this->sourceConnection->getOwnershipRoleOnDatabase($database); | ||
| $projectUser = $this->getProjectUser($databaseRole); | ||
|
|
||
| $grantedOnDatabaseRole = $this->sourceConnection->fetchAll(sprintf( | ||
| 'SHOW GRANTS ON ROLE %s', | ||
|
There was a problem hiding this comment. Myslel jsem, že jet o TO ROLE? ale funguje podle dokumentace oboje :D There was a problem hiding this comment. Tohle teda vezme všechny granty, která má project role There was a problem hiding this comment. SHOW GRANTS ON ROLE KEBOOLA_123 There was a problem hiding this comment. tohle souvisí i s tím druhým commentem SHOW GRANT ON ROLE KEBOOLA_123 - načte granty které vlastní tuto roli - takže KEBOOLA_STORAGE |
||
| Helper::quoteIdentifier($databaseRole) | ||
| )); | ||
| assert(count($grantedOnDatabaseRole) === 1); | ||
|
There was a problem hiding this comment. Nrozumím proč je jen jeden ten grant. Může jich být imho klidně víc - třeba na ws usery, schemata, ws role, etc., ne? There was a problem hiding this comment. Jako je super, že je tu assert, ale nerozumím proč to funguje There was a problem hiding this comment. funguje to protože - #47 (comment) |
||
| $mainRoleName = current($grantedOnDatabaseRole)['granted_by']; | ||
|
|
||
| $data = $this->getDataToRemove($this->sourceConnection, $databaseRole, $mainRoleName); | ||
|
|
||
| // drop roles | ||
| $sqls[] = sprintf('DROP ROLE %s;', Helper::quoteIdentifier($databaseRole)); | ||
|
|
@@ -87,13 +98,12 @@ public function sourceAccount(): void | |
|
|
||
| public function preMigration(string $mainRoleName): void | ||
| { | ||
| $this->logger->info(sprintf('Starting pre-migration cleanup with main role: %s', $mainRoleName)); | ||
| $sqls = []; | ||
| $currentRole = null; | ||
|
|
||
| // Check if main role is assigned to target user | ||
| $this->logger->info('Checking main role assignment and ownership'); | ||
| $grantsToTargetUser = $this->destinationConnection->fetchAll(sprintf( | ||
| 'SHOW GRANTS TO USER %s', | ||
| QueryBuilder::quoteIdentifier($this->config->getTargetSnowflakeUser()), | ||
|
|
@@ -104,6 +114,11 @@ public function preMigration(string $mainRoleName): void | |
| false, | ||
| ); | ||
|
|
||
| // Check if target role has ownership of main role | ||
| $mainRole = $this->destinationConnection->fetchAll(sprintf( | ||
| 'SHOW ROLES LIKE %s', | ||
| QueryBuilder::quote($mainRoleName), | ||
| )); | ||
| $hasMainRoleOwnership = array_reduce( | ||
| $mainRole, | ||
| fn ($found, $v) => $found || $v['owner'] === $this->config->getTargetSnowflakeRole(), | ||
|
|
@@ -118,123 +133,117 @@ public function preMigration(string $mainRoleName): void | |
| } | ||
| $this->destinationConnection->grantRoleToUser($this->config->getTargetSnowflakeUser(), $mainRoleName); | ||
| } | ||
|
|
||
| // Process each database | ||
| foreach ($this->config->getDatabases() as $database) { | ||
| $this->logger->info(sprintf('Processing database: %s', $database)); | ||
| $this->destinationConnection->useRole($this->config->getTargetSnowflakeRole()); | ||
|
|
||
| // Check if database exists | ||
| $dbExists = $this->destinationConnection->fetchAll(sprintf( | ||
| 'SHOW DATABASES LIKE %s;', | ||
| QueryBuilder::quote($database) | ||
| )); | ||
|
|
||
| if (!$dbExists) { | ||
| $this->logger->info(sprintf('Database %s does not exist, checking for role with same name', $database)); | ||
| // Check if role exists with exact or lowercase name | ||
| $roleName = null; | ||
| foreach ([$database, strtolower($database)] as $nameVariant) { | ||
| $roleExists = $this->destinationConnection->fetchAll(sprintf( | ||
| 'SHOW ROLES LIKE %s', | ||
| QueryBuilder::quote($nameVariant) | ||
| )); | ||
| if ($roleExists) { | ||
| $roleName = $nameVariant; | ||
| break; | ||
| } | ||
| } | ||
|
|
||
| if ($roleName === null) { | ||
| continue; | ||
| } | ||
|
|
||
| $dataToRemove = $this->getDataToRemove($this->destinationConnection, $roleName, $mainRoleName); | ||
|
There was a problem hiding this comment. A co projektový user KEBOOLA_123? There was a problem hiding this comment. jestli myslíš smazání ho tak se přidává v metodě "addProjectRoleAndUserToRemove" |
||
| } else { | ||
| $this->logger->info(sprintf('Database %s exists, getting ownership role', $database)); | ||
| $databaseRole = $this->destinationConnection->getOwnershipRoleOnDatabase($database); | ||
| $dataToRemove = $this->getDataToRemove($this->destinationConnection, $databaseRole, $mainRoleName); | ||
| } | ||
|
|
||
| // First revoke all future grants from roles | ||
| foreach ($dataToRemove['ROLE'] as $role) { | ||
| $this->destinationConnection->useRole($role['granted_by']); | ||
| $futureGrants = array_map( | ||
| fn(array $v) => FutureGrantToRole::fromArray($v), | ||
| $this->destinationConnection->fetchAll(sprintf( | ||
| 'SHOW FUTURE GRANTS TO ROLE %s', | ||
| Helper::quoteIdentifier($role['name']) | ||
| )) | ||
| ); | ||
|
|
||
| if ($futureGrants) { | ||
| [$sqls, $currentRole] = $this->switchRole($role['granted_by'], $mainRoleName, $sqls, $currentRole); | ||
| } | ||
| foreach ($futureGrants as $futureGrant) { | ||
| $sqls[] = sprintf( | ||
| 'REVOKE %s ON FUTURE TABLES IN SCHEMA %s FROM ROLE %s;', | ||
| $futureGrant->getPrivilege(), | ||
| $futureGrant->getName(), | ||
| Helper::quoteIdentifier($futureGrant->getGranteeName()) | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| // Drop users owned by roles | ||
| if (!empty($dataToRemove['USER'])) { | ||
ondrajodas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| $this->logger->info(sprintf('Dropping %d users', count($dataToRemove['USER']))); | ||
|
There was a problem hiding this comment. Tady bych zalogoval které přesně to dropuje, ne? There was a problem hiding this comment. to samé - #47 (comment) |
||
| } | ||
| foreach ($dataToRemove['USER'] as $user) { | ||
| [$sqls, $currentRole] = $this->switchRole($user['granted_by'], $mainRoleName, $sqls, $currentRole); | ||
| $sqls[] = sprintf( | ||
| 'DROP USER IF EXISTS %s;', | ||
| Helper::quoteIdentifier($user['name']) | ||
| ); | ||
| } | ||
|
|
||
| // Drop roles | ||
| if (!empty($dataToRemove['ROLE'])) { | ||
| $this->logger->info(sprintf('Dropping %d roles', count($dataToRemove['ROLE']))); | ||
|
There was a problem hiding this comment. Taky bych si myslel, že by tu mělo být které přesně There was a problem hiding this comment. no nevím jestli bych to tu měl vypsat 🤷♂️ seznam který chceme migrovat těch rolí a userů má několik stovek/tisíc |
||
| } | ||
| foreach ($dataToRemove['ROLE'] as $role) { | ||
| [$sqls, $currentRole] = $this->switchRole($role['granted_by'], $mainRoleName, $sqls, $currentRole); | ||
| $sqls[] = sprintf( | ||
| 'DROP ROLE IF EXISTS %s;', | ||
| Helper::quoteIdentifier($role['name']) | ||
| ); | ||
| } | ||
|
|
||
| // Drop database role and handle database if exists | ||
| [$sqls, $currentRole] = $this->switchRole($mainRoleName, $mainRoleName, $sqls, $currentRole); | ||
|
|
||
| if ($dbExists) { | ||
| $sqls[] = sprintf( | ||
| 'DROP DATABASE IF EXISTS %s;', | ||
| Helper::quoteIdentifier($database . '_OLD') | ||
| ); | ||
| $sqls[] = sprintf( | ||
| 'ALTER DATABASE IF EXISTS %s RENAME TO %s;', | ||
| Helper::quoteIdentifier($database), | ||
| Helper::quoteIdentifier($database . '_OLD') | ||
| ); | ||
| } | ||
| } | ||
|
|
||
| // Execute all SQL commands | ||
| foreach ($sqls as $sql) { | ||
| if ($this->config->getSynchronizeDryPremigrationCleanupRun()) { | ||
| $this->logger->info($sql); | ||
| } else { | ||
| $this->destinationConnection->query($sql); | ||
| } | ||
| } | ||
|
|
||
| if ($this->config->getSynchronizeDryPremigrationCleanupRun() && $sqls) { | ||
| throw new UserException('!!! PLEASE RUN SQLS ON TARGET SNOWFLAKE ACCOUNT !!!'); | ||
| } | ||
|
|
@@ -297,36 +306,100 @@ public function postMigration(): void | |
| } | ||
| } | ||
|
|
||
| private function getDataToRemove(Connection $connection, string $name, string $mainRoleName): array | ||
|
There was a problem hiding this comment. tyjo, ten naming... $roleName, ne? There was a problem hiding this comment. a getDataToRemoveForRole There was a problem hiding this comment. naming zajišťoval cursor, tak jsem se svezl... přejmenuju |
||
| { | ||
| $this->logger->debug(sprintf('Getting data to remove for: %s', $name)); | ||
ondrajodas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| $result = [ | ||
| self::ROLE => [], | ||
| self::USER => [], | ||
| ]; | ||
|
|
||
| $grants = $connection->fetchAll(sprintf( | ||
| 'SHOW GRANTS TO ROLE %s', | ||
| Helper::quoteIdentifier($name) | ||
| )); | ||
|
|
||
| $ownershipGrants = array_filter( | ||
| $grants, | ||
| fn($v) => $v['privilege'] === self::OWNERSHIP | ||
| ); | ||
|
|
||
| $result = $this->processOwnershipGrants($ownershipGrants, $result); | ||
| $result = $this->addUserIfExists($connection, $name, $mainRoleName, $result); | ||
| $result = $this->addRoleToRemove($name, $mainRoleName, $result); | ||
|
|
||
| return $result; | ||
| } | ||
|
|
||
| private function processOwnershipGrants(array $ownershipGrants, array $result): array | ||
| { | ||
| foreach ($ownershipGrants as $grant) { | ||
| if ($grant['granted_on'] === self::ROLE) { | ||
| $this->logger->debug(sprintf('Found owned role: %s', $grant['name'])); | ||
| $result[self::ROLE][] = $grant; | ||
| } elseif ($grant['granted_on'] === self::USER) { | ||
| $this->logger->debug(sprintf('Found directly owned user: %s', $grant['name'])); | ||
| $result[self::USER][] = $grant; | ||
| } | ||
| } | ||
| return $result; | ||
| } | ||
|
|
||
| private function addUserIfExists(Connection $connection, string $name, string $mainRoleName, array $result): array | ||
ondrajodas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| { | ||
| $userExists = $connection->fetchAll(sprintf( | ||
| 'SHOW USERS LIKE %s', | ||
| QueryBuilder::quote($name) | ||
| )); | ||
| if ($userExists) { | ||
| $this->logger->debug(sprintf('Found user with same name: %s', $name)); | ||
| $userExists[0]['granted_by'] = $mainRoleName; | ||
ondrajodas marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| $result[self::USER][] = $userExists[0]; | ||
| } | ||
| return $result; | ||
| } | ||
|
|
||
| private function addRoleToRemove(string $name, string $mainRoleName, array $result): array | ||
| { | ||
| $result[self::ROLE][] = [ | ||
| 'name' => $name, | ||
| 'granted_by' => $mainRoleName, | ||
| ]; | ||
| return $result; | ||
| } | ||
|
|
||
| private function switchRole(string $role, string $mainRoleName, array $sqls, ?string $currentRole): array | ||
| { | ||
| if ($currentRole === $role) { | ||
| return [$sqls, $currentRole]; | ||
| } | ||
|
|
||
| try { | ||
| $this->logger->debug(sprintf('Switching to role: %s', $role)); | ||
| $this->destinationConnection->useRole($role); | ||
| $sqls[] = sprintf('USE ROLE %s;', Helper::quoteIdentifier($role)); | ||
| $currentRole = $role; | ||
| } catch (RuntimeException $e) { | ||
| $this->logger->info(sprintf( | ||
| 'Cannot switch directly to role %s, trying through main role %s', | ||
| $role, | ||
| $mainRoleName | ||
| )); | ||
| $this->destinationConnection->useRole($mainRoleName); | ||
| if ($currentRole !== $mainRoleName) { | ||
| $sqls[] = sprintf('USE ROLE %s;', Helper::quoteIdentifier($mainRoleName)); | ||
| $currentRole = $mainRoleName; | ||
| } | ||
| $this->destinationConnection->grantRoleToUser( | ||
| $this->config->getTargetSnowflakeUser(), | ||
| $role | ||
| ); | ||
| $this->logger->debug(sprintf('Granted role %s to user, switching to it', $role)); | ||
| $this->destinationConnection->useRole($role); | ||
| $sqls[] = sprintf('USE ROLE %s;', Helper::quoteIdentifier($role)); | ||
| $currentRole = $role; | ||
| } | ||
| return [$sqls, $currentRole]; | ||
| } | ||
|
|
||
| private function getProjectUser(string $databaseRole): GrantToUser | ||
|
|
||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Owner DB je "project role".
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
KEBOOLA_123
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
jo je...jo je to role která vlastní databázi, ano projektová role no