From 1c2ba37a2c25bc98159f3d5b9271d9b7ce3cfde4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Fri, 19 Apr 2024 15:11:41 +0200 Subject: [PATCH 01/12] fix docker --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 95a8f7d..91d50c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,6 @@ services: <<: *ci tty: true stdin_open: true - command: bash + entrypoint: bash volumes: - .:/code From 1a1962f19494c5095d50e2e2a1ca2a194b23c86e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Mon, 22 Apr 2024 10:27:47 +0200 Subject: [PATCH 02/12] KAB-46 table metadata post --- kbcstorage/tables.py | 3 ++ kbcstorage/tables_metadata.py | 60 +++++++++++++++++++++++++++++++++ tests/functional/test_tables.py | 48 ++++++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 kbcstorage/tables_metadata.py diff --git a/kbcstorage/tables.py b/kbcstorage/tables.py index bf650e3..75918fe 100644 --- a/kbcstorage/tables.py +++ b/kbcstorage/tables.py @@ -11,6 +11,8 @@ from kbcstorage.base import Endpoint from kbcstorage.files import Files from kbcstorage.jobs import Jobs +from kbcstorage.tables_metadata import TablesMetadata + class Tables(Endpoint): @@ -26,6 +28,7 @@ def __init__(self, root_url, token): token (:obj:`str`): A storage API key. """ super().__init__(root_url, 'tables', token) + self.metadata = TablesMetadata(root_url, token) def list(self, include=None): """ diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py new file mode 100644 index 0000000..3492e2f --- /dev/null +++ b/kbcstorage/tables_metadata.py @@ -0,0 +1,60 @@ +""" +Manages calls to the Storage API relating to table metadatas + +Full documentation `here`. + +.. _here: + http://docs.keboola.apiary.io/#reference/metadata/table-metadata +""" +import json +import tempfile +import os +from kbcstorage.base import Endpoint +from kbcstorage.files import Files +from kbcstorage.jobs import Jobs + + +class TablesMetadata(Endpoint): + """ + Tables Metadata Endpoint + """ + def __init__(self, root_url, token): + """ + Create a Tables endpoint. + + Args: + root_url (:obj:`str`): The base url for the API. + token (:obj:`str`): A storage API key. + """ + super().__init__(root_url, 'tables', token) + + def create(self, table_id, provider, metadata, columns_metadata): + """ + Post metadata to a table. + + Args: + table_id (str): Table id + provider (str): Provider of the metadata + metadata (list): List of metadata dictionaries with 'key' and 'value' + columns_metadata (dict): Dictionary with column metadata + + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(table_id, str) or table_id == '': + raise ValueError("Invalid table_id '{}'.".format(table_id)) + + url = '{}/{}/metadata'.format(self.base_url, table_id) + headers = { + 'Content-Type': 'application/json', + 'X-StorageApi-Token': self.token + } + data = { + "provider": provider, + "metadata": metadata, + "columnsMetadata": columns_metadata + } + return self._post(url, data=json.dumps(data), headers=headers) diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 3720299..746c479 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -5,6 +5,7 @@ from requests import exceptions from kbcstorage.tables import Tables from kbcstorage.buckets import Buckets +from kbcstorage.tables_metadata import TablesMetadata from tests.base_test_case import BaseTestCase @@ -327,3 +328,50 @@ def test_table_columns(self): with open(local_path, mode='rt') as file: lines = file.readlines() self.assertEqual(['"col3","col2"\n', '"king","pong"\n'], sorted(lines)) + + def test_table_with_metadata(self): + file, path = tempfile.mkstemp(prefix='sapi-test') + with open(path, 'w') as csv_file: + writer = csv.DictWriter(csv_file, fieldnames=['col1', 'col2'], + lineterminator='\n', delimiter=',', + quotechar='"') + writer.writeheader() + writer.writerow({'col1': 'ping', 'col2': 'pong'}) + os.close(file) + table_id = self.tables.create(name='some-table', file_path=path, + bucket_id='in.c-py-test-tables') + + self.tables.metadata.create( + table_id=table_id, + provider='test', + metadata=[{ + 'key' : 'test_table_with_metadata', + 'value' : 'success' + }], + columns_metadata={ + 'col1' : [{ + 'key' : 'test_column_with_metadata', + 'value' : 'success' + }] + }) + + table_info = self.tables.detail(table_id) + with self.subTest("Test metadata key in response"): + self.assertIn('metadata', table_info) + with self.subTest("Test metadata structure"): + self.assertEqual(1, len(table_info['metadata'])) + self.assertIn('id', table_info['metadata'][0]) + self.assertEqual('test_table_with_metadata', table_info['metadata'][0]['key']) + self.assertEqual('test', table_info['metadata'][0]['provider']) + self.assertIn('timestamp', table_info['metadata'][0]) + self.assertEqual('success', table_info['metadata'][0]['value']) + with self.subTest('Test columns metadata key in response'): + self.assertIn('columnMetadata', table_info) + with self.subTest('Test columns metadata structure'): + self.assertIn('col1', table_info['columnMetadata']) + self.assertEqual(1, len(table_info['columnMetadata']['col1'])) + self.assertIn('id', table_info['columnMetadata']['col1'][0]) + self.assertEqual('test_column_with_metadata', table_info['columnMetadata']['col1'][0]['key']) + self.assertEqual('test', table_info['columnMetadata']['col1'][0]['provider']) + self.assertIn('timestamp', table_info['columnMetadata']['col1'][0]) + self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value']) \ No newline at end of file From 1335852ac1190a2e0ed9609e4eb7afed670aeb1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Mon, 22 Apr 2024 14:16:02 +0200 Subject: [PATCH 03/12] KAB-46 table metadata list --- kbcstorage/tables_metadata.py | 18 ++++++++++++++++++ tests/functional/test_tables.py | 10 +++++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 3492e2f..4b1e8b5 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -28,6 +28,24 @@ def __init__(self, root_url, token): """ super().__init__(root_url, 'tables', token) + def list(self, table_id): + """ + List all metadata for table + + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + ValueError: If the table_id is not a string or is empty. + """ + if not isinstance(table_id, str) or table_id == '': + raise ValueError("Invalid table_id '{}'.".format(table_id)) + + url = '{}/{}/metadata'.format(self.base_url, table_id) + + return self._get(url) + def create(self, table_id, provider, metadata, columns_metadata): """ Post metadata to a table. diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 746c479..17bf086 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -374,4 +374,12 @@ def test_table_with_metadata(self): self.assertEqual('test_column_with_metadata', table_info['columnMetadata']['col1'][0]['key']) self.assertEqual('test', table_info['columnMetadata']['col1'][0]['provider']) self.assertIn('timestamp', table_info['columnMetadata']['col1'][0]) - self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value']) \ No newline at end of file + self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value']) + + listedMetadata = self.tables.metadata.list(table_id=table_id); + + with self.subTest("Test metadata key in list response"): + self.assertEqual(1, len(listedMetadata)) + self.assertEqual('test_table_with_metadata', listedMetadata[0]['key']) + self.assertEqual('test', listedMetadata[0]['provider']) + self.assertEqual('success', listedMetadata[0]['value']) \ No newline at end of file From bc4edbe5d0bb5b3af36efd4d16ef295a215570d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Mon, 22 Apr 2024 14:26:51 +0200 Subject: [PATCH 04/12] KAB-46 table metadata delete --- kbcstorage/tables_metadata.py | 20 ++++++++++++++++++++ tests/functional/test_tables.py | 8 +++++++- 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 4b1e8b5..61445a5 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -46,6 +46,26 @@ def list(self, table_id): return self._get(url) + def delete(self, table_id, metadata_id): + """ + Delete a table referenced by ``table_id``. + + Args: + table_id (str): The id of the table to be deleted. + + Raises: + requests.HTTPError: If the API request fails. + ValueError: If the table_id is not a string or is empty. + """ + if not isinstance(table_id, str) or table_id == '': + raise ValueError("Invalid table_id '{}'.".format(table_id)) + if not isinstance(metadata_id, str) or metadata_id == '': + raise ValueError("Invalid metadata_id '{}'.".format(metadata_id)) + + url = '{}/{}/metadata/{}'.format(self.base_url, table_id, metadata_id) + + self._delete(url) + def create(self, table_id, provider, metadata, columns_metadata): """ Post metadata to a table. diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 17bf086..fde93fb 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -382,4 +382,10 @@ def test_table_with_metadata(self): self.assertEqual(1, len(listedMetadata)) self.assertEqual('test_table_with_metadata', listedMetadata[0]['key']) self.assertEqual('test', listedMetadata[0]['provider']) - self.assertEqual('success', listedMetadata[0]['value']) \ No newline at end of file + self.assertEqual('success', listedMetadata[0]['value']) + + self.tables.metadata.delete(table_id=table_id, metadata_id=listedMetadata[0]['id']) + + listedMetadata = self.tables.metadata.list(table_id=table_id); + with self.subTest('Test metadata can was deleted'): + self.assertEqual(0, len(listedMetadata)) \ No newline at end of file From 93dbfda113440d032d59540bcfd3e09826f86ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Mon, 22 Apr 2024 14:36:05 +0200 Subject: [PATCH 05/12] KAB-46 flake8 --- kbcstorage/tables.py | 1 - kbcstorage/tables_metadata.py | 62 +++++++++++++++------------------ tests/functional/test_tables.py | 17 +++++---- 3 files changed, 37 insertions(+), 43 deletions(-) diff --git a/kbcstorage/tables.py b/kbcstorage/tables.py index 75918fe..5285fb1 100644 --- a/kbcstorage/tables.py +++ b/kbcstorage/tables.py @@ -14,7 +14,6 @@ from kbcstorage.tables_metadata import TablesMetadata - class Tables(Endpoint): """ Tables Endpoint diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 61445a5..e1f5982 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -7,11 +7,7 @@ http://docs.keboola.apiary.io/#reference/metadata/table-metadata """ import json -import tempfile -import os from kbcstorage.base import Endpoint -from kbcstorage.files import Files -from kbcstorage.jobs import Jobs class TablesMetadata(Endpoint): @@ -67,32 +63,32 @@ def delete(self, table_id, metadata_id): self._delete(url) def create(self, table_id, provider, metadata, columns_metadata): - """ - Post metadata to a table. - - Args: - table_id (str): Table id - provider (str): Provider of the metadata - metadata (list): List of metadata dictionaries with 'key' and 'value' - columns_metadata (dict): Dictionary with column metadata - - Returns: - response_body: The parsed json from the HTTP response. - - Raises: - requests.HTTPError: If the API request fails. - """ - if not isinstance(table_id, str) or table_id == '': - raise ValueError("Invalid table_id '{}'.".format(table_id)) - - url = '{}/{}/metadata'.format(self.base_url, table_id) - headers = { - 'Content-Type': 'application/json', - 'X-StorageApi-Token': self.token - } - data = { - "provider": provider, - "metadata": metadata, - "columnsMetadata": columns_metadata - } - return self._post(url, data=json.dumps(data), headers=headers) + """ + Post metadata to a table. + + Args: + table_id (str): Table id + provider (str): Provider of the metadata + metadata (list): List of metadata dictionaries with 'key' and 'value' + columns_metadata (dict): Dictionary with column metadata + + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(table_id, str) or table_id == '': + raise ValueError("Invalid table_id '{}'.".format(table_id)) + + url = '{}/{}/metadata'.format(self.base_url, table_id) + headers = { + 'Content-Type': 'application/json', + 'X-StorageApi-Token': self.token + } + data = { + "provider": provider, + "metadata": metadata, + "columnsMetadata": columns_metadata + } + return self._post(url, data=json.dumps(data), headers=headers) diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index fde93fb..6d34fc7 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -5,7 +5,6 @@ from requests import exceptions from kbcstorage.tables import Tables from kbcstorage.buckets import Buckets -from kbcstorage.tables_metadata import TablesMetadata from tests.base_test_case import BaseTestCase @@ -345,13 +344,13 @@ def test_table_with_metadata(self): table_id=table_id, provider='test', metadata=[{ - 'key' : 'test_table_with_metadata', - 'value' : 'success' + 'key': 'test_table_with_metadata', + 'value': 'success' }], columns_metadata={ - 'col1' : [{ - 'key' : 'test_column_with_metadata', - 'value' : 'success' + 'col1': [{ + 'key': 'test_column_with_metadata', + 'value': 'success' }] }) @@ -376,7 +375,7 @@ def test_table_with_metadata(self): self.assertIn('timestamp', table_info['columnMetadata']['col1'][0]) self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value']) - listedMetadata = self.tables.metadata.list(table_id=table_id); + listedMetadata = self.tables.metadata.list(table_id=table_id) with self.subTest("Test metadata key in list response"): self.assertEqual(1, len(listedMetadata)) @@ -386,6 +385,6 @@ def test_table_with_metadata(self): self.tables.metadata.delete(table_id=table_id, metadata_id=listedMetadata[0]['id']) - listedMetadata = self.tables.metadata.list(table_id=table_id); + listedMetadata = self.tables.metadata.list(table_id=table_id) with self.subTest('Test metadata can was deleted'): - self.assertEqual(0, len(listedMetadata)) \ No newline at end of file + self.assertEqual(0, len(listedMetadata)) From 50f429f41af9d07d2a0fd5474b21efb864c8d30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 14:32:35 +0200 Subject: [PATCH 06/12] KAB-46 fix configuration creation (throws 501) --- kbcstorage/configurations.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kbcstorage/configurations.py b/kbcstorage/configurations.py index e25f096..26671d2 100644 --- a/kbcstorage/configurations.py +++ b/kbcstorage/configurations.py @@ -3,6 +3,7 @@ Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations """ +import json from kbcstorage.base import Endpoint @@ -111,6 +112,6 @@ def create(self, component_id, name, description='', configuration=None, state=N 'isDisabled': is_disabled } if configuration_id: - body['id'] = configuration_id + body['configurationId'] = configuration_id url = '{}/{}/configs'.format(self.base_url, component_id) - return self._post(url, data=body) + return self._post(url, data=json.dumps(body), headers={'Content-Type': 'application/json'}) From a7bbdf9501b89363d2dece6972cefd51abfd1536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 14:33:12 +0200 Subject: [PATCH 07/12] KAB-46 add configuration metadata create, list and delete --- kbcstorage/configurations.py | 2 + kbcstorage/configurations_metadata.py | 115 ++++++++++++++++++++++++ tests/functional/test_configurations.py | 53 +++++++++++ 3 files changed, 170 insertions(+) create mode 100644 kbcstorage/configurations_metadata.py diff --git a/kbcstorage/configurations.py b/kbcstorage/configurations.py index 26671d2..0e2e8da 100644 --- a/kbcstorage/configurations.py +++ b/kbcstorage/configurations.py @@ -5,6 +5,7 @@ """ import json from kbcstorage.base import Endpoint +from kbcstorage.configurations_metadata import ConfigurationsMetadata class Configurations(Endpoint): @@ -22,6 +23,7 @@ def __init__(self, root_url, token, branch_id): branch_id (str): The ID of branch to use, use 'default' to work without branch (in main). """ super().__init__(root_url, f"branch/{branch_id}/components", token) + self.metadata = ConfigurationsMetadata(root_url, token, branch_id) def detail(self, component_id, configuration_id): """ diff --git a/kbcstorage/configurations_metadata.py b/kbcstorage/configurations_metadata.py new file mode 100644 index 0000000..3e87b85 --- /dev/null +++ b/kbcstorage/configurations_metadata.py @@ -0,0 +1,115 @@ +""" +Manages calls to the Storage API relating to configurations + +Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations +""" +import json +from kbcstorage.base import Endpoint + + +class ConfigurationsMetadata(Endpoint): + """ + Configurations Endpoint + """ + + def __init__(self, root_url, token, branch_id): + """ + Create a Component endpoint. + + Args: + root_url (:obj:`str`): The base url for the API. + token (:obj:`str`): A storage API key. + branch_id (str): The ID of branch to use, use 'default' to work without branch (in main). + """ + super().__init__(root_url, f"branch/{branch_id}/components", token) + + def detail(self, component_id, configuration_id): + """ + Retrieves information about a given configuration. + + Args: + component_id (str): The id of the component. + configuration_id (str): The id of the configuration. + + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(component_id, str) or component_id == '': + raise ValueError("Invalid component_id '{}'.".format(component_id)) + if not isinstance(configuration_id, str) or configuration_id == '': + raise ValueError("Invalid component_id '{}'.".format(configuration_id)) + url = '{}/{}/configs/{}'.format(self.base_url, component_id, configuration_id) + return self._get(url) + + def delete(self, component_id, configuration_id, metadata_id): + """ + Deletes the configuration. + + Args: + component_id (str): The id of the component. + configuration_id (str): The id of the configuration. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(component_id, str) or component_id == '': + raise ValueError("Invalid component_id '{}'.".format(component_id)) + if not isinstance(configuration_id, str) or configuration_id == '': + raise ValueError("Invalid component_id '{}'.".format(configuration_id)) + url = '{}/{}/configs/{}/metadata/{}'.format(self.base_url, component_id, configuration_id, metadata_id) + self._delete(url) + + def list(self, component_id, configuration_id): + """ + Lists configurations of the given component. + + Args: + component_id (str): The id of the component. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(component_id, str) or component_id == '': + raise ValueError("Invalid component_id '{}'.".format(component_id)) + url = '{}/{}/configs/{}/metadata'.format(self.base_url, component_id, configuration_id) + return self._get(url) + + def create(self, component_id, configuration_id, provider, metadata): + """ + Create a new configuration. + + Args: + component_id (str): The id of the component. + configuration (str): The id of the configuration. + provider (str): The provider of the configuration (currently ignored and "user" is sent). + key (str): The key of the configuration. + value (str): The value of the configuration. + Returns: + response_body: The parsed json from the HTTP response. + + Raises: + requests.HTTPError: If the API request fails. + """ + if not isinstance(component_id, str) or component_id == '': + raise ValueError("Invalid component_id '{}'.".format(component_id)) + if not isinstance(configuration_id, str) or configuration_id == '': + raise ValueError("Invalid component_id '{}'.".format(configuration_id)) + url = '{}/{}/configs/{}/metadata'.format(self.base_url, component_id, configuration_id) + if not isinstance(metadata, list): + raise ValueError("Metadata must be a list '{}'.".format(metadata)) + for metadataItem in metadata: + if not isinstance(metadataItem, dict): + raise ValueError("Metadata item must be a dictionary '{}'.".format(metadataItem)) + + headers = { + 'Content-Type': 'application/json', + 'X-StorageApi-Token': self.token + } + data = { + # 'provider': provider, # not yet implemented + 'metadata': metadata + } + return self._post(url, data=json.dumps(data), headers=headers) diff --git a/tests/functional/test_configurations.py b/tests/functional/test_configurations.py index 161710b..714bc16 100644 --- a/tests/functional/test_configurations.py +++ b/tests/functional/test_configurations.py @@ -50,3 +50,56 @@ def testListConfigurations(self): with self.subTest(): with self.assertRaises(exceptions.HTTPError): configurations = self.configurations.list('non-existent-component') + + def testConfigurationMetadata(self): + self.configurations.create( + component_id=self.TEST_COMPONENT_NAME, + configuration_id='test_configuration_metadata', + name='test_configuration_metadata', + ) + metadataPayload = [ + { + 'key': 'testConfigurationMetadata', + 'value': 'success', + } + ] + metadataList = self.configurations.metadata.create( + component_id=self.TEST_COMPONENT_NAME, + configuration_id='test_configuration_metadata', + provider='test', + metadata=metadataPayload, + ) + + with (self.subTest('assert metadata create response')): + self.assertEqual(1, len(metadataList)) + metadataItem = metadataList[0] + self.assertTrue('id' in metadataItem) + # self.assertTrue('provider' in metadata) not yet + self.assertTrue('key' in metadataItem) + self.assertTrue('value' in metadataItem) + + metadataList = self.configurations.metadata.list( + component_id=self.TEST_COMPONENT_NAME, + configuration_id='test_configuration_metadata' + ) + + with (self.subTest('assert metadata list response')): + self.assertTrue(len(metadataList) > 0) + for metadataList in metadataList: + self.assertTrue('id' in metadataList) + # self.assertTrue('provider' in metadata) not yet + self.assertTrue('key' in metadataList) + self.assertTrue('value' in metadataList) + + self.configurations.metadata.delete( + component_id=self.TEST_COMPONENT_NAME, + configuration_id='test_configuration_metadata', + metadata_id=metadataList['id'] + ) + metadataList = self.configurations.metadata.list( + component_id=self.TEST_COMPONENT_NAME, + configuration_id='test_configuration_metadata' + ) + + with (self.subTest('assert metadata delete means metadata no longer in list')): + self.assertTrue(len(metadataList) == 0) From efb1fd1fcb91b2aad3c7ed8bdff123120d643213 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 15:06:59 +0200 Subject: [PATCH 08/12] KAB-46 remove token as it's added automatically --- kbcstorage/configurations_metadata.py | 1 - kbcstorage/tables_metadata.py | 1 - 2 files changed, 2 deletions(-) diff --git a/kbcstorage/configurations_metadata.py b/kbcstorage/configurations_metadata.py index 3e87b85..163d8c7 100644 --- a/kbcstorage/configurations_metadata.py +++ b/kbcstorage/configurations_metadata.py @@ -106,7 +106,6 @@ def create(self, component_id, configuration_id, provider, metadata): headers = { 'Content-Type': 'application/json', - 'X-StorageApi-Token': self.token } data = { # 'provider': provider, # not yet implemented diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index e1f5982..54e0f68 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -84,7 +84,6 @@ def create(self, table_id, provider, metadata, columns_metadata): url = '{}/{}/metadata'.format(self.base_url, table_id) headers = { 'Content-Type': 'application/json', - 'X-StorageApi-Token': self.token } data = { "provider": provider, From 9cdaf7d48946bcecab4911e2f7aed6c963d89059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 15:14:05 +0200 Subject: [PATCH 09/12] KAB-46 add missing validation --- kbcstorage/configurations_metadata.py | 4 +++- kbcstorage/tables_metadata.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/kbcstorage/configurations_metadata.py b/kbcstorage/configurations_metadata.py index 163d8c7..75c5b24 100644 --- a/kbcstorage/configurations_metadata.py +++ b/kbcstorage/configurations_metadata.py @@ -58,7 +58,9 @@ def delete(self, component_id, configuration_id, metadata_id): if not isinstance(component_id, str) or component_id == '': raise ValueError("Invalid component_id '{}'.".format(component_id)) if not isinstance(configuration_id, str) or configuration_id == '': - raise ValueError("Invalid component_id '{}'.".format(configuration_id)) + raise ValueError("Invalid configuration_id '{}'.".format(configuration_id)) + if not isinstance(metadata_id, str) or metadata_id == '': + raise ValueError("Invalid metadata_id '{}'.".format(metadata_id)) url = '{}/{}/configs/{}/metadata/{}'.format(self.base_url, component_id, configuration_id, metadata_id) self._delete(url) diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 54e0f68..298a7dd 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -80,6 +80,12 @@ def create(self, table_id, provider, metadata, columns_metadata): """ if not isinstance(table_id, str) or table_id == '': raise ValueError("Invalid table_id '{}'.".format(table_id)) + if not isinstance(provider, str) or provider == '': + raise ValueError("Invalid provider '{}'.".format(provider)) + if not isinstance(metadata, list): + raise ValueError("Invalid metadata '{}'.".format(metadata)) + if not isinstance(columns_metadata, dict): + raise ValueError("Invalid columns_metadata '{}'.".format(columns_metadata)) url = '{}/{}/metadata'.format(self.base_url, table_id) headers = { From 7c31af9ccfb7b3949fdccefa0a59bb63befab947 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 20:01:06 +0200 Subject: [PATCH 10/12] KAB-46 fix docblocks --- kbcstorage/configurations_metadata.py | 48 ++++++++++----------------- kbcstorage/tables_metadata.py | 18 +++++++--- 2 files changed, 31 insertions(+), 35 deletions(-) diff --git a/kbcstorage/configurations_metadata.py b/kbcstorage/configurations_metadata.py index 75c5b24..d7ce0bd 100644 --- a/kbcstorage/configurations_metadata.py +++ b/kbcstorage/configurations_metadata.py @@ -1,7 +1,7 @@ """ -Manages calls to the Storage API relating to configurations +Manages calls to the Storage API relating to configurations metadata -Full documentation https://keboola.docs.apiary.io/#reference/components-and-configurations +Full documentation https://keboola.docs.apiary.io/#reference/metadata/components-configurations-metadata/ """ import json from kbcstorage.base import Endpoint @@ -9,12 +9,12 @@ class ConfigurationsMetadata(Endpoint): """ - Configurations Endpoint + Configurations metadata Endpoint """ def __init__(self, root_url, token, branch_id): """ - Create a Component endpoint. + Create a Component metadata endpoint. Args: root_url (:obj:`str`): The base url for the API. @@ -23,37 +23,18 @@ def __init__(self, root_url, token, branch_id): """ super().__init__(root_url, f"branch/{branch_id}/components", token) - def detail(self, component_id, configuration_id): - """ - Retrieves information about a given configuration. - - Args: - component_id (str): The id of the component. - configuration_id (str): The id of the configuration. - - Returns: - response_body: The parsed json from the HTTP response. - - Raises: - requests.HTTPError: If the API request fails. - """ - if not isinstance(component_id, str) or component_id == '': - raise ValueError("Invalid component_id '{}'.".format(component_id)) - if not isinstance(configuration_id, str) or configuration_id == '': - raise ValueError("Invalid component_id '{}'.".format(configuration_id)) - url = '{}/{}/configs/{}'.format(self.base_url, component_id, configuration_id) - return self._get(url) - def delete(self, component_id, configuration_id, metadata_id): """ - Deletes the configuration. + Deletes the configuration metadata identified by ``metadata_id``. Args: component_id (str): The id of the component. configuration_id (str): The id of the configuration. + metadata_id (str): The id of the metadata (not key!). Raises: requests.HTTPError: If the API request fails. + ValueError: If the component_id/configuration_id/metadata_id is not a string or is empty. """ if not isinstance(component_id, str) or component_id == '': raise ValueError("Invalid component_id '{}'.".format(component_id)) @@ -66,34 +47,41 @@ def delete(self, component_id, configuration_id, metadata_id): def list(self, component_id, configuration_id): """ - Lists configurations of the given component. + Lists metadata for a given component configuration. Args: component_id (str): The id of the component. + configuration_id (str): The id of the configuration. Raises: requests.HTTPError: If the API request fails. + ValueError: If the component_id/configuration_id is not a string or is empty. """ if not isinstance(component_id, str) or component_id == '': raise ValueError("Invalid component_id '{}'.".format(component_id)) + if not isinstance(configuration_id, str) or configuration_id == '': + raise ValueError("Invalid configuration_id '{}'.".format(configuration_id)) url = '{}/{}/configs/{}/metadata'.format(self.base_url, component_id, configuration_id) return self._get(url) def create(self, component_id, configuration_id, provider, metadata): """ - Create a new configuration. + Writes metadata for a given component configuration. Args: component_id (str): The id of the component. configuration (str): The id of the configuration. provider (str): The provider of the configuration (currently ignored and "user" is sent). - key (str): The key of the configuration. - value (str): The value of the configuration. + metadata (list): A list of metadata items. Item is a dictionary with 'key' and 'value' keys. + Returns: response_body: The parsed json from the HTTP response. Raises: requests.HTTPError: If the API request fails. + ValueError: If the component_id/configuration_id is not a string or is empty. + ValueError: If the metadata is not a list. + ValueError: If the metadata item is not a dictionary. """ if not isinstance(component_id, str) or component_id == '': raise ValueError("Invalid component_id '{}'.".format(component_id)) diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 298a7dd..83337bd 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -16,7 +16,7 @@ class TablesMetadata(Endpoint): """ def __init__(self, root_url, token): """ - Create a Tables endpoint. + Create a Tables metadata endpoint. Args: root_url (:obj:`str`): The base url for the API. @@ -28,6 +28,9 @@ def list(self, table_id): """ List all metadata for table + Args: + table_id (str): Table id + Returns: response_body: The parsed json from the HTTP response. @@ -44,14 +47,15 @@ def list(self, table_id): def delete(self, table_id, metadata_id): """ - Delete a table referenced by ``table_id``. + Delete a table metadata referenced by ``metadata_id``. Args: - table_id (str): The id of the table to be deleted. + table_id (str): The id of the table. + metadata_id (str): The id of the table metdata entry to be deleted. Raises: requests.HTTPError: If the API request fails. - ValueError: If the table_id is not a string or is empty. + ValueError: If the table_id/metadata_id is not a string or is empty. """ if not isinstance(table_id, str) or table_id == '': raise ValueError("Invalid table_id '{}'.".format(table_id)) @@ -70,13 +74,17 @@ def create(self, table_id, provider, metadata, columns_metadata): table_id (str): Table id provider (str): Provider of the metadata metadata (list): List of metadata dictionaries with 'key' and 'value' - columns_metadata (dict): Dictionary with column metadata + columns_metadata (dict): Dictionary with lists of metadata dictionaries with 'key', 'value', 'columnName'. Returns: response_body: The parsed json from the HTTP response. Raises: requests.HTTPError: If the API request fails. + ValueError: If the table_id is not a string or is empty. + ValueError: If the provider is not a string or is empty. + ValueError: If the metadata is not a list. + ValueError: If the columns_metadata is not a list """ if not isinstance(table_id, str) or table_id == '': raise ValueError("Invalid table_id '{}'.".format(table_id)) From 2a4e1dcac146d01936b34d456ddeba6be63bd0e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Tue, 23 Apr 2024 20:01:32 +0200 Subject: [PATCH 11/12] KAB-46 passing column as key is deprecated --- kbcstorage/tables_metadata.py | 2 +- tests/functional/test_tables.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/kbcstorage/tables_metadata.py b/kbcstorage/tables_metadata.py index 83337bd..295e6d3 100644 --- a/kbcstorage/tables_metadata.py +++ b/kbcstorage/tables_metadata.py @@ -92,7 +92,7 @@ def create(self, table_id, provider, metadata, columns_metadata): raise ValueError("Invalid provider '{}'.".format(provider)) if not isinstance(metadata, list): raise ValueError("Invalid metadata '{}'.".format(metadata)) - if not isinstance(columns_metadata, dict): + if not isinstance(columns_metadata, list): raise ValueError("Invalid columns_metadata '{}'.".format(columns_metadata)) url = '{}/{}/metadata'.format(self.base_url, table_id) diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 6d34fc7..15f1c1f 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -348,9 +348,10 @@ def test_table_with_metadata(self): 'value': 'success' }], columns_metadata={ - 'col1': [{ + [{ 'key': 'test_column_with_metadata', - 'value': 'success' + 'value': 'success', + 'columnName': 'col1' }] }) From 859aeb5d7e214c0b52b48c22616d9ffdb2b62257 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Fejfar?= Date: Wed, 24 Apr 2024 10:18:06 +0200 Subject: [PATCH 12/12] KAB-46 fix dict --- tests/functional/test_tables.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 15f1c1f..bc3ff43 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -347,13 +347,16 @@ def test_table_with_metadata(self): 'key': 'test_table_with_metadata', 'value': 'success' }], - columns_metadata={ - [{ - 'key': 'test_column_with_metadata', - 'value': 'success', - 'columnName': 'col1' - }] - }) + columns_metadata=[ + [ + { + 'key': 'test_column_with_metadata', + 'value': 'success', + 'columnName': 'col1' + } + ] + ] + ) table_info = self.tables.detail(table_id) with self.subTest("Test metadata key in response"):