Merge pull request #63 from keboola/PS-2218-and-tags

PS-2218 Support AND tags for import files to workspace
keboola · May 21, 2021 · 5a93926 · 5a93926
2 parents b4e8255 + 2726d11
commit 5a93926
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 2 deletions.
diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py
@@ -155,7 +155,10 @@ def load_files(self, workspace_id, file_mapping):
         Args:
             workspace_id (int or str): The id of the workspace to which to load
                 the tables.
-            file_mapping (:obj:`dict`): contains tags: [], destination: string path without trailing /
+            file_mapping (:obj:`dict`):
+                tags: [],
+                operator: enum('or', 'and') default or,
+                destination: string path without trailing /
 
         Raises:
             requests.HTTPError: If the API request fails.
@@ -164,7 +167,12 @@ def load_files(self, workspace_id, file_mapping):
         if (workspace['type'] != 'file' and workspace['connection']['backend'] != 'abs'):
             raise Exception('Loading files to workspace is only available for ABS workspaces')
         files = Files(self.root_url, self.token)
-        file_list = files.list(tags=file_mapping['tags'])
+        if ('operator' in file_mapping and file_mapping['operator'] == 'and'):
+            query = ' AND '.join(map(lambda tag: 'tags:"' + tag + '"', file_mapping['tags']))
+            file_list = files.list(q=query)
+        else:
+            file_list = files.list(tags=file_mapping['tags'])
+
         jobs = Jobs(self.root_url, self.token)
         jobs_list = []
         for file in file_list:

diff --git a/tests/functional/test_workspaces.py b/tests/functional/test_workspaces.py
@@ -5,6 +5,7 @@
 import warnings
 
 from azure.storage.blob import BlobServiceClient
+from azure.core.exceptions import ResourceNotFoundError
 from requests import exceptions
 from kbcstorage.buckets import Buckets
 from kbcstorage.jobs import Jobs
@@ -134,6 +135,32 @@ def test_load_files_to_workspace(self):
         )
         self.assertEqual('fooBar', blob_client_2.download_blob().readall().decode('utf-8'))
 
+        # now let's test that we can use the 'and' operator.  in this case file2 should not get loaded
+        self.workspaces.load_files(
+            workspace['id'],
+            {
+                'tags': ['sapi-client-python-tests', 'file1'],
+                'operator': 'and',
+                'destination': 'data/in/and_files'
+            }
+        )
+        # file 1 should be there
+        blob_client_1 = blob_service_client.get_blob_client(
+            container=workspace['connection']['container'],
+            blob='data/in/and_files/%s/%s' % (file1['name'], str(file1['id']))
+        )
+        self.assertEqual('fooBar', blob_client_1.download_blob().readall().decode('utf-8'))
+
+        # file 2 should not
+        blob_client_2 = blob_service_client.get_blob_client(
+            container=workspace['connection']['container'],
+            blob='data/in/and_files/%s/%s' % (file2['name'], str(file2['id']))
+        )
+        with self.assertRaises(ResourceNotFoundError) as context:
+            blob_client_2.download_blob().readall().decode('utf-8')
+
+        self.assertTrue('The specified blob does not exist' in str(context.exception))
+
     def __create_table(self, bucket_id, table_name, row):
         file, path = tempfile.mkstemp(prefix='sapi-test')
         with open(path, 'w') as csv_file: