KAB-46 prepare structure for metastore metadata to be stored with storage objects

docker-compose.yml

@@ -12,6 +12,6 @@ services:
     <<: *ci
     tty: true
     stdin_open: true
-    command: bash
+    entrypoint: bash
     volumes:
       - .:/code

kbcstorage/tables.py

@@ -11,6 +11,7 @@
 from kbcstorage.base import Endpoint
 from kbcstorage.files import Files
 from kbcstorage.jobs import Jobs
+from kbcstorage.tables_metadata import TablesMetadata
 
 
 class Tables(Endpoint):
@@ -26,6 +27,7 @@ def __init__(self, root_url, token):
             token (:obj:`str`): A storage API key.
         """
         super().__init__(root_url, 'tables', token)
+        self.metadata = TablesMetadata(root_url, token)
 
     def list(self, include=None):
         """

kbcstorage/tables_metadata.py

@@ -0,0 +1,94 @@
+"""
+Manages calls to the Storage API relating to table metadatas
+
+Full documentation `here`.
+
+.. _here:
+    http://docs.keboola.apiary.io/#reference/metadata/table-metadata
+"""
+import json
+from kbcstorage.base import Endpoint
+
+
+class TablesMetadata(Endpoint):
+    """
+    Tables Metadata Endpoint
+    """
+    def __init__(self, root_url, token):
+        """
+        Create a Tables endpoint.
+
+        Args:
+            root_url (:obj:`str`): The base url for the API.
+            token (:obj:`str`): A storage API key.
+        """
+        super().__init__(root_url, 'tables', token)
+
+    def list(self, table_id):
+        """
+        List all metadata for table
+
+        Returns:
+            response_body: The parsed json from the HTTP response.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+            ValueError: If the table_id is not a string or is empty.
+        """
+        if not isinstance(table_id, str) or table_id == '':
+            raise ValueError("Invalid table_id '{}'.".format(table_id))
+
+        url = '{}/{}/metadata'.format(self.base_url, table_id)
+
+        return self._get(url)
+
+    def delete(self, table_id, metadata_id):
+        """
+        Delete a table referenced by ``table_id``.
+
+        Args:
+            table_id (str): The id of the table to be deleted.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+            ValueError: If the table_id is not a string or is empty.
+        """
+        if not isinstance(table_id, str) or table_id == '':
+            raise ValueError("Invalid table_id '{}'.".format(table_id))
+        if not isinstance(metadata_id, str) or metadata_id == '':
+            raise ValueError("Invalid metadata_id '{}'.".format(metadata_id))
+
+        url = '{}/{}/metadata/{}'.format(self.base_url, table_id, metadata_id)
+
+        self._delete(url)
+
+    def create(self, table_id, provider, metadata, columns_metadata):
+        """
+        Post metadata to a table.
+
+        Args:
+            table_id (str): Table id
+            provider (str): Provider of the metadata
+            metadata (list): List of metadata dictionaries with 'key' and 'value'
+            columns_metadata (dict): Dictionary with column metadata
+
+        Returns:
+            response_body: The parsed json from the HTTP response.
+
+        Raises:
+            requests.HTTPError: If the API request fails.
+        """
+        if not isinstance(table_id, str) or table_id == '':
+            raise ValueError("Invalid table_id '{}'.".format(table_id))
+
+        url = '{}/{}/metadata'.format(self.base_url, table_id)
+        headers = {
+            'Content-Type': 'application/json',
+            'X-StorageApi-Token': self.token
+        }
+        data = {
+            "provider": provider,
+            "metadata": metadata,
+            "columnsMetadata": columns_metadata
+        }
+        return self._post(url, data=json.dumps(data), headers=headers)

tests/functional/test_tables.py

@@ -327,3 +327,64 @@ def test_table_columns(self):
         with open(local_path, mode='rt') as file:
             lines = file.readlines()
         self.assertEqual(['"col3","col2"\n', '"king","pong"\n'], sorted(lines))
+
+    def test_table_with_metadata(self):
+        file, path = tempfile.mkstemp(prefix='sapi-test')
+        with open(path, 'w') as csv_file:
+            writer = csv.DictWriter(csv_file, fieldnames=['col1', 'col2'],
+                                    lineterminator='\n', delimiter=',',
+                                    quotechar='"')
+            writer.writeheader()
+            writer.writerow({'col1': 'ping', 'col2': 'pong'})
+        os.close(file)
+        table_id = self.tables.create(name='some-table', file_path=path,
+                                      bucket_id='in.c-py-test-tables')
+
+        self.tables.metadata.create(
+            table_id=table_id,
+            provider='test',
+            metadata=[{
+                'key': 'test_table_with_metadata',
+                'value': 'success'
+                }],
+            columns_metadata={
+                'col1': [{
+                    'key': 'test_column_with_metadata',
+                    'value': 'success'
+                    }]
+                })
+
+        table_info = self.tables.detail(table_id)
+        with self.subTest("Test metadata key in response"):
+            self.assertIn('metadata', table_info)
+        with self.subTest("Test metadata structure"):
+            self.assertEqual(1, len(table_info['metadata']))
+            self.assertIn('id', table_info['metadata'][0])
+            self.assertEqual('test_table_with_metadata', table_info['metadata'][0]['key'])
+            self.assertEqual('test', table_info['metadata'][0]['provider'])
+            self.assertIn('timestamp', table_info['metadata'][0])
+            self.assertEqual('success', table_info['metadata'][0]['value'])
+        with self.subTest('Test columns metadata key in response'):
+            self.assertIn('columnMetadata', table_info)
+        with self.subTest('Test columns metadata structure'):
+            self.assertIn('col1', table_info['columnMetadata'])
+            self.assertEqual(1, len(table_info['columnMetadata']['col1']))
+            self.assertIn('id', table_info['columnMetadata']['col1'][0])
+            self.assertEqual('test_column_with_metadata', table_info['columnMetadata']['col1'][0]['key'])
+            self.assertEqual('test', table_info['columnMetadata']['col1'][0]['provider'])
+            self.assertIn('timestamp', table_info['columnMetadata']['col1'][0])
+            self.assertEqual('success', table_info['columnMetadata']['col1'][0]['value'])
+
+        listedMetadata = self.tables.metadata.list(table_id=table_id)
+
+        with self.subTest("Test metadata key in list response"):
+            self.assertEqual(1, len(listedMetadata))
+            self.assertEqual('test_table_with_metadata', listedMetadata[0]['key'])
+            self.assertEqual('test', listedMetadata[0]['provider'])
+            self.assertEqual('success', listedMetadata[0]['value'])
+
+        self.tables.metadata.delete(table_id=table_id, metadata_id=listedMetadata[0]['id'])
+
+        listedMetadata = self.tables.metadata.list(table_id=table_id)
+        with self.subTest('Test metadata can was deleted'):
+            self.assertEqual(0, len(listedMetadata))