From 37ade2c55680a8189dcf50bebe3e61a8da382ef9 Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 17:40:29 +0200 Subject: [PATCH 1/7] add verify token method --- kbcstorage/client.py | 2 + kbcstorage/tokens.py | 42 ++++++++++ tests/functional/test_tokens.py | 16 ++++ tests/mocks/test_tokens.py | 34 ++++++++ tests/mocks/token_responses.py | 141 ++++++++++++++++++++++++++++++++ 5 files changed, 235 insertions(+) create mode 100644 kbcstorage/tokens.py create mode 100644 tests/functional/test_tokens.py create mode 100644 tests/mocks/test_tokens.py create mode 100644 tests/mocks/token_responses.py diff --git a/kbcstorage/client.py b/kbcstorage/client.py index 00ac102..0e39c5d 100644 --- a/kbcstorage/client.py +++ b/kbcstorage/client.py @@ -5,6 +5,7 @@ from kbcstorage.buckets import Buckets from kbcstorage.components import Components from kbcstorage.configurations import Configurations +from kbcstorage.tokens import Tokens from kbcstorage.workspaces import Workspaces from kbcstorage.jobs import Jobs from kbcstorage.tables import Tables @@ -37,6 +38,7 @@ def __init__(self, api_domain, token, branch_id='default'): self.workspaces = Workspaces(self.root_url, self.token) self.components = Components(self.root_url, self.token, self.branch_id) self.configurations = Configurations(self.root_url, self.token, self.branch_id) + self.tokens = Tokens(self.root_url, self.token) @property def token(self): diff --git a/kbcstorage/tokens.py b/kbcstorage/tokens.py new file mode 100644 index 0000000..fa48d6c --- /dev/null +++ b/kbcstorage/tokens.py @@ -0,0 +1,42 @@ +""" +Manages calls to the Storage API relating to tokens + +Full documentation https://keboola.docs.apiary.io/#reference/tokens-and-permissions/. + +""" +import tempfile +import os +from kbcstorage.base import Endpoint +from kbcstorage.files import Files +from kbcstorage.jobs import Jobs + + +class Tokens(Endpoint): + """ + Tables Endpoint + """ + def __init__(self, root_url, token): + """ + Create a Tables endpoint. + + Args: + root_url (:obj:`str`): The base url for the API. + token (:obj:`str`): A storage API key. + """ + super().__init__(root_url, 'tokens', token) + + def verify(self): + """ + Verify token. + + Args: + include (list): Properties to list (configuration, rows, state) + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + url = '{}/verify'.format(self.base_url) + return self._get(url) + diff --git a/tests/functional/test_tokens.py b/tests/functional/test_tokens.py new file mode 100644 index 0000000..b32d931 --- /dev/null +++ b/tests/functional/test_tokens.py @@ -0,0 +1,16 @@ +import os +from kbcstorage.tokens import Tokens +from tests.base_test_case import BaseTestCase + + +class TestEndpoint(BaseTestCase): + def setUp(self): + self.tokens = Tokens(os.getenv('KBC_TEST_API_URL'), + os.getenv('KBC_TEST_TOKEN')) + + def test_verify(self): + token_info = self.tokens.verify() + self.assertTrue('id' in token_info) + self.assertTrue('description' in token_info) + self.assertTrue('canManageBuckets' in token_info) + self.assertTrue('owner' in token_info) diff --git a/tests/mocks/test_tokens.py b/tests/mocks/test_tokens.py new file mode 100644 index 0000000..d515a0c --- /dev/null +++ b/tests/mocks/test_tokens.py @@ -0,0 +1,34 @@ +""" +Test basic functionality of the Tokens endpoint +""" +import unittest + +import responses + +from kbcstorage.tokens import Tokens +from tests.mocks.token_responses import verify_token_response + + +class TestTablesEndpointWithMocks(unittest.TestCase): + """ + Test the methods of a Tables endpoint instance with mock HTTP responses + """ + def setUp(self): + token = 'dummy_token' + base_url = 'https://connection.keboola.com/' + self.tokens = Tokens(base_url, token) + + @responses.activate + def test_verify(self): + """ + Tables mocks list correctly + """ + responses.add( + responses.Response( + method='GET', + url='https://connection.keboola.com/v2/storage/tokens/verify', + json=verify_token_response + ) + ) + token_info = self.tokens.verify() + self.assertEqual(verify_token_response, token_info) diff --git a/tests/mocks/token_responses.py b/tests/mocks/token_responses.py new file mode 100644 index 0000000..153c0d9 --- /dev/null +++ b/tests/mocks/token_responses.py @@ -0,0 +1,141 @@ +verify_token_response = { + "id": "373115", + "created": "2021-06-23T09:33:36+0200", + "refreshed": "2021-06-23T09:33:36+0200", + "description": "devel@keboola.com", + "uri": "https://connection.keboola.com/v2/storage/tokens/373115", + "isMasterToken": True, + "canManageBuckets": True, + "canManageTokens": True, + "canReadAllFileUploads": True, + "canPurgeTrash": True, + "expires": None, + "isExpired": False, + "isDisabled": False, + "dailyCapacity": 0, + "token": "8625-373115-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", + "bucketPermissions": { + "out.c-test2": "manage", + "in.c-test3": "manage", + "out.c-test": "manage", + "out.c-some-other": "manage" + }, + "owner": { + "id": 8625, + "name": "Test Project", + "type": "production", + "region": "us-east-1", + "created": "2021-06-23T09:33:32+0200", + "expires": None, + "features": [ + "syrup-jobs-limit-10", + "oauth-v3", + "queuev2", + "storage-types", + "allow-ai", + "alternat", + ], + "dataSizeBytes": 31474176, + "rowsCount": 797, + "hasMysql": False, + "hasSynapse": False, + "hasRedshift": True, + "hasSnowflake": True, + "hasExasol": False, + "hasTeradata": False, + "hasBigquery": False, + "defaultBackend": "snowflake", + "hasTryModeOn": "0", + "limits": { + "components.jobsParallelism": { + "name": "components.jobsParallelism", + "value": 10 + }, + "goodData.dataSizeBytes": { + "name": "goodData.dataSizeBytes", + "value": 1000000000 + }, + "goodData.demoTokenEnabled": { + "name": "goodData.demoTokenEnabled", + "value": 1 + }, + "goodData.prodTokenEnabled": { + "name": "goodData.prodTokenEnabled", + "value": 0 + }, + "goodData.usersCount": { + "name": "goodData.usersCount", + "value": 30 + }, + "kbc.adminsCount": { + "name": "kbc.adminsCount", + "value": 10 + }, + "kbc.extractorsCount": { + "name": "kbc.extractorsCount", + "value": 0 + }, + "kbc.monthlyProjectPowerLimit": { + "name": "kbc.monthlyProjectPowerLimit", + "value": 50 + }, + "kbc.writersCount": { + "name": "kbc.writersCount", + "value": 0 + }, + "orchestrations.count": { + "name": "orchestrations.count", + "value": 10 + }, + "storage.dataSizeBytes": { + "name": "storage.dataSizeBytes", + "value": 50000000000 + }, + "storage.jobsParallelism": { + "name": "storage.jobsParallelism", + "value": 10 + } + }, + "metrics": { + "kbc.adminsCount": { + "name": "kbc.adminsCount", + "value": 1 + }, + "orchestrations.count": { + "name": "orchestrations.count", + "value": 0 + }, + "storage.dataSizeBytes": { + "name": "storage.dataSizeBytes", + "value": 31474176 + }, + "storage.rowsCount": { + "name": "storage.rowsCount", + "value": 797 + } + }, + "isDisabled": False, + "billedMonthlyPrice": None, + "dataRetentionTimeInDays": 7, + "fileStorageProvider": "aws", + "redshift": { + "connectionId": 365, + "databaseName": "sapi_8625" + } + }, + "organization": { + "id": "111111" + }, + "admin": { + "name": "Devel", + "id": 59, + "features": [ + "ui-devel-preview", + "manage-try-mode", + "validate-sql", + "early-adopter-preview" + ], + "isOrganizationMember": True, + "role": "admin" + } +} From b04b44cc9c664fa131941b22cf15c20361fbe986 Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 17:51:27 +0200 Subject: [PATCH 2/7] fix cs --- kbcstorage/tokens.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/kbcstorage/tokens.py b/kbcstorage/tokens.py index fa48d6c..0e07285 100644 --- a/kbcstorage/tokens.py +++ b/kbcstorage/tokens.py @@ -4,20 +4,16 @@ Full documentation https://keboola.docs.apiary.io/#reference/tokens-and-permissions/. """ -import tempfile -import os from kbcstorage.base import Endpoint -from kbcstorage.files import Files -from kbcstorage.jobs import Jobs class Tokens(Endpoint): """ - Tables Endpoint + Tokens Endpoint """ def __init__(self, root_url, token): """ - Create a Tables endpoint. + Create a Tokens endpoint. Args: root_url (:obj:`str`): The base url for the API. @@ -29,8 +25,6 @@ def verify(self): """ Verify token. - Args: - include (list): Properties to list (configuration, rows, state) Returns: response_body: The parsed json from the HTTP response. @@ -39,4 +33,3 @@ def verify(self): """ url = '{}/verify'.format(self.base_url) return self._get(url) - From a680fbe7a95856becf6986b60e058fe48c116741 Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 18:22:08 +0200 Subject: [PATCH 3/7] add branch metadata endpoint --- kbcstorage/base.py | 2 -- kbcstorage/branches.py | 43 +++++++++++++++++++++++++++ kbcstorage/client.py | 3 +- tests/functional/test_branches.py | 14 +++++++++ tests/mocks/branches_responses.py | 8 +++++ tests/mocks/test_branches.py | 49 +++++++++++++++++++++++++++++++ tests/mocks/test_tokens.py | 6 ++-- 7 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 kbcstorage/branches.py create mode 100644 tests/functional/test_branches.py create mode 100644 tests/mocks/branches_responses.py create mode 100644 tests/mocks/test_branches.py diff --git a/kbcstorage/base.py b/kbcstorage/base.py index 4f8afd8..ab36a00 100644 --- a/kbcstorage/base.py +++ b/kbcstorage/base.py @@ -35,8 +35,6 @@ def __init__(self, root_url, path_component, token): """ if not root_url: raise ValueError("Root URL is required.") - if not path_component: - raise ValueError("Path component is required.") if not token: raise ValueError("Token is required.") self.root_url = root_url diff --git a/kbcstorage/branches.py b/kbcstorage/branches.py new file mode 100644 index 0000000..96e482d --- /dev/null +++ b/kbcstorage/branches.py @@ -0,0 +1,43 @@ +""" +Manages calls to the Storage API relating to development branches + +Full documentation https://keboola.docs.apiary.io/#reference/development-branches + +""" +from kbcstorage.base import Endpoint + + +class Branches(Endpoint): + """ + Tokens Endpoint + """ + def __init__(self, root_url, token): + """ + Create a Tokens endpoint. + + Args: + root_url (:obj:`str`): The base url for the API. + token (:obj:`str`): A storage API key. + """ + # Branches have inconsistent endpoint naming - it's either dev-branches or branch, so it need to be resolved + # endpoint by endpoint. + super().__init__(root_url, '', token) + + def metadata(self, branch_id): + """ + Get branch metadata + + Args: + branch_id (str): The id of the branch or "default" to get metadata for the main branch (production). + + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(branch_id, str) or branch_id == "": + raise ValueError(f"Invalid branch_id '{branch_id}'") + + url = f"{self.base_url}branch/{branch_id}/metadata" + return self._get(url) diff --git a/kbcstorage/client.py b/kbcstorage/client.py index 0e39c5d..bedc0b3 100644 --- a/kbcstorage/client.py +++ b/kbcstorage/client.py @@ -1,7 +1,7 @@ """" Entry point for the Storage API client. """ - +from kbcstorage.branches import Branches from kbcstorage.buckets import Buckets from kbcstorage.components import Components from kbcstorage.configurations import Configurations @@ -39,6 +39,7 @@ def __init__(self, api_domain, token, branch_id='default'): self.components = Components(self.root_url, self.token, self.branch_id) self.configurations = Configurations(self.root_url, self.token, self.branch_id) self.tokens = Tokens(self.root_url, self.token) + self.branches = Branches(self.root_url, self.token) @property def token(self): diff --git a/tests/functional/test_branches.py b/tests/functional/test_branches.py new file mode 100644 index 0000000..4b6dc06 --- /dev/null +++ b/tests/functional/test_branches.py @@ -0,0 +1,14 @@ +import os + +from kbcstorage.branches import Branches +from tests.base_test_case import BaseTestCase + + +class TestEndpoint(BaseTestCase): + def setUp(self): + self.branches = Branches(os.getenv('KBC_TEST_API_URL'), + os.getenv('KBC_TEST_TOKEN')) + + def test_metadata(self): + metadata = self.branches.metadata('default') + self.assertTrue(isinstance(metadata, list)) diff --git a/tests/mocks/branches_responses.py b/tests/mocks/branches_responses.py new file mode 100644 index 0000000..5b763ec --- /dev/null +++ b/tests/mocks/branches_responses.py @@ -0,0 +1,8 @@ +branches_metadata_response = [ + { + "id": "93670", + "key": "KBC.projectDescription", + "value": "Testing project one", + "timestamp": "2023-05-31T17:52:18+0200" + } +] diff --git a/tests/mocks/test_branches.py b/tests/mocks/test_branches.py new file mode 100644 index 0000000..52cb6c0 --- /dev/null +++ b/tests/mocks/test_branches.py @@ -0,0 +1,49 @@ +""" +Test basic functionality of the Branches endpoint +""" +import unittest + +import responses + +from kbcstorage.branches import Branches +from tests.mocks.branches_responses import branches_metadata_response + + +class TestBranchesEndpointWithMocks(unittest.TestCase): + """ + Test the methods of a Branches endpoint instance with mock HTTP responses + """ + def setUp(self): + token = 'dummy_token' + base_url = 'https://connection.keboola.com/' + self.branches = Branches(base_url, token) + + @responses.activate + def test_metadata_no_branch(self): + """ + Branches lists metadata correctly + """ + responses.add( + responses.Response( + method='GET', + url='https://connection.keboola.com/v2/storage/branch/default/metadata', + json=branches_metadata_response + ) + ) + branch_metadata = self.branches.metadata('default') + self.assertEqual(branches_metadata_response, branch_metadata) + + @responses.activate + def test_metadata_some_branch(self): + """ + Branches lists metadata correctly + """ + responses.add( + responses.Response( + method='GET', + url='https://connection.keboola.com/v2/storage/branch/1234/metadata', + json=branches_metadata_response + ) + ) + branch_metadata = self.branches.metadata('1234') + self.assertEqual(branches_metadata_response, branch_metadata) diff --git a/tests/mocks/test_tokens.py b/tests/mocks/test_tokens.py index d515a0c..43ac2dc 100644 --- a/tests/mocks/test_tokens.py +++ b/tests/mocks/test_tokens.py @@ -9,9 +9,9 @@ from tests.mocks.token_responses import verify_token_response -class TestTablesEndpointWithMocks(unittest.TestCase): +class TestTokensEndpointWithMocks(unittest.TestCase): """ - Test the methods of a Tables endpoint instance with mock HTTP responses + Test the methods of a Tokens endpoint instance with mock HTTP responses """ def setUp(self): token = 'dummy_token' @@ -21,7 +21,7 @@ def setUp(self): @responses.activate def test_verify(self): """ - Tables mocks list correctly + Verify token returns correctly """ responses.add( responses.Response( From 59aa351a9f72889cb4e6352b6bb10ce0b73d06db Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 18:29:34 +0200 Subject: [PATCH 4/7] fix cs --- tests/functional/test_branches.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/functional/test_branches.py b/tests/functional/test_branches.py index 4b6dc06..a17dad0 100644 --- a/tests/functional/test_branches.py +++ b/tests/functional/test_branches.py @@ -6,8 +6,7 @@ class TestEndpoint(BaseTestCase): def setUp(self): - self.branches = Branches(os.getenv('KBC_TEST_API_URL'), - os.getenv('KBC_TEST_TOKEN')) + self.branches = Branches(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN')) def test_metadata(self): metadata = self.branches.metadata('default') From f26b87e2214a31c20af999199edc8760ca902af1 Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 22:04:22 +0200 Subject: [PATCH 5/7] not relevant any more --- tests/functional/test_base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/functional/test_base.py b/tests/functional/test_base.py index 3836795..d671352 100644 --- a/tests/functional/test_base.py +++ b/tests/functional/test_base.py @@ -97,11 +97,6 @@ def test_missing_url(self): with self.assertRaisesRegex(ValueError, "Root URL is required."): Endpoint(None, '', None) - def test_missing_part(self): - with self.assertRaisesRegex(ValueError, - "Path component is required."): - Endpoint('https://connection.keboola.com/', '', None) - def test_missing_token(self): with self.assertRaisesRegex(ValueError, "Token is required."): Endpoint('https://connection.keboola.com/', 'tables', None) From 31e780b02efbf981012ba60cf5be11439bcc5d82 Mon Sep 17 00:00:00 2001 From: odin Date: Wed, 31 May 2023 22:50:13 +0200 Subject: [PATCH 6/7] make default branch default --- kbcstorage/branches.py | 2 +- tests/mocks/test_branches.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kbcstorage/branches.py b/kbcstorage/branches.py index 96e482d..7ae27e9 100644 --- a/kbcstorage/branches.py +++ b/kbcstorage/branches.py @@ -23,7 +23,7 @@ def __init__(self, root_url, token): # endpoint by endpoint. super().__init__(root_url, '', token) - def metadata(self, branch_id): + def metadata(self, branch_id = 'default'): """ Get branch metadata diff --git a/tests/mocks/test_branches.py b/tests/mocks/test_branches.py index 52cb6c0..33f9426 100644 --- a/tests/mocks/test_branches.py +++ b/tests/mocks/test_branches.py @@ -30,7 +30,7 @@ def test_metadata_no_branch(self): json=branches_metadata_response ) ) - branch_metadata = self.branches.metadata('default') + branch_metadata = self.branches.metadata() self.assertEqual(branches_metadata_response, branch_metadata) @responses.activate From c46604892e30e7eeb19528c52a0198cff0c0e2d4 Mon Sep 17 00:00:00 2001 From: odin Date: Thu, 1 Jun 2023 07:44:01 +0200 Subject: [PATCH 7/7] fix cs --- kbcstorage/branches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kbcstorage/branches.py b/kbcstorage/branches.py index 7ae27e9..1e4ecd9 100644 --- a/kbcstorage/branches.py +++ b/kbcstorage/branches.py @@ -21,9 +21,9 @@ def __init__(self, root_url, token): """ # Branches have inconsistent endpoint naming - it's either dev-branches or branch, so it need to be resolved # endpoint by endpoint. - super().__init__(root_url, '', token) + super().__init__(root_url, "", token) - def metadata(self, branch_id = 'default'): + def metadata(self, branch_id="default"): """ Get branch metadata