diff --git a/kbcstorage/workspaces.py b/kbcstorage/workspaces.py index 73ebdfb..f6f3135 100644 --- a/kbcstorage/workspaces.py +++ b/kbcstorage/workspaces.py @@ -155,7 +155,10 @@ def load_files(self, workspace_id, file_mapping): Args: workspace_id (int or str): The id of the workspace to which to load the tables. - file_mapping (:obj:`dict`): contains tags: [], destination: string path without trailing / + file_mapping (:obj:`dict`): + tags: [], + operator: enum('or', 'and') default or, + destination: string path without trailing / Raises: requests.HTTPError: If the API request fails. @@ -164,7 +167,12 @@ def load_files(self, workspace_id, file_mapping): if (workspace['type'] != 'file' and workspace['connection']['backend'] != 'abs'): raise Exception('Loading files to workspace is only available for ABS workspaces') files = Files(self.root_url, self.token) - file_list = files.list(tags=file_mapping['tags']) + if ('operator' in file_mapping and file_mapping['operator'] == 'and'): + query = ' AND '.join(map(lambda tag: 'tags:"' + tag + '"', file_mapping['tags'])) + file_list = files.list(q=query) + else: + file_list = files.list(tags=file_mapping['tags']) + jobs = Jobs(self.root_url, self.token) jobs_list = [] for file in file_list: diff --git a/tests/functional/test_workspaces.py b/tests/functional/test_workspaces.py index 4dbb210..3182773 100644 --- a/tests/functional/test_workspaces.py +++ b/tests/functional/test_workspaces.py @@ -5,6 +5,7 @@ import warnings from azure.storage.blob import BlobServiceClient +from azure.core.exceptions import ResourceNotFoundError from requests import exceptions from kbcstorage.buckets import Buckets from kbcstorage.jobs import Jobs @@ -134,6 +135,32 @@ def test_load_files_to_workspace(self): ) self.assertEqual('fooBar', blob_client_2.download_blob().readall().decode('utf-8')) + # now let's test that we can use the 'and' operator. in this case file2 should not get loaded + self.workspaces.load_files( + workspace['id'], + { + 'tags': ['sapi-client-python-tests', 'file1'], + 'operator': 'and', + 'destination': 'data/in/and_files' + } + ) + # file 1 should be there + blob_client_1 = blob_service_client.get_blob_client( + container=workspace['connection']['container'], + blob='data/in/and_files/%s/%s' % (file1['name'], str(file1['id'])) + ) + self.assertEqual('fooBar', blob_client_1.download_blob().readall().decode('utf-8')) + + # file 2 should not + blob_client_2 = blob_service_client.get_blob_client( + container=workspace['connection']['container'], + blob='data/in/and_files/%s/%s' % (file2['name'], str(file2['id'])) + ) + with self.assertRaises(ResourceNotFoundError) as context: + blob_client_2.download_blob().readall().decode('utf-8') + + self.assertTrue('The specified blob does not exist' in str(context.exception)) + def __create_table(self, bucket_id, table_name, row): file, path = tempfile.mkstemp(prefix='sapi-test') with open(path, 'w') as csv_file: