From 58b9897d51495479731ed4329e276e07f9d37801 Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 26 Jan 2023 09:25:28 +0100 Subject: [PATCH 01/18] needed for package registration --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 67593d9..7956738 100644 --- a/setup.py +++ b/setup.py @@ -18,5 +18,6 @@ test_suite='tests', tests_require=['responses'], long_description=readme, + long_description_content_type='text/markdown', license="MIT" ) From 4366f5d19bde6526beeab5be1b77e112f78e9f6b Mon Sep 17 00:00:00 2001 From: Marc Date: Thu, 26 Jan 2023 09:59:42 +0100 Subject: [PATCH 02/18] push to pypi in gh actions --- .github/workflows/python-publish.yml | 39 ++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/python-publish.yml diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml new file mode 100644 index 0000000..bdaab28 --- /dev/null +++ b/.github/workflows/python-publish.yml @@ -0,0 +1,39 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + deploy: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} From d07754cbdd8c0267f8298ac186542308cd67771f Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 1 Feb 2023 14:42:02 +0100 Subject: [PATCH 03/18] remove travis, use actions --- .github/workflows/push.yml | 68 ++++++++++++++++++++++++++++ .github/workflows/python-publish.yml | 39 ---------------- .travis.yml | 23 ---------- 3 files changed, 68 insertions(+), 62 deletions(-) create mode 100644 .github/workflows/push.yml delete mode 100644 .github/workflows/python-publish.yml delete mode 100644 .travis.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100644 index 0000000..0b0e853 --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,68 @@ +# This workflow will upload a Python Package using Twine when a release is created +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Upload Python Package + +on: + release: + types: [published] + +permissions: + contents: read + +jobs: + + build_and_test: + runs-on: ubuntu-latest + outputs: + is_semantic_tag: ${{steps.tag.outputs.is_semantic_tag}} + uses: actions/checkout@v3 + steps: + - name: Print Docker version + run: | + docker -v + - name: Docker login + if: env.DOCKERHUB_TOKEN + run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN" + - name: build + run: | + docker-compose build sapi-python-client + - name: run tests + run: | + docker-compose run --rm --entrypoint=flake8 sapi-python-client + docker-compose run --rm -e KBC_TEST_TOKEN -e KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover + docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_AZ_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_AZ_TEST_API_URL sapi-python-client -m unittest discover + - name: Check if tag is semantic + id: tag + run: | + TAG="${GITHUB_REF##*/}" + IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) + echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" + echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG" + deploy_to_pypi: + needs: + - build_and_test + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/') && needs.build_and_test.outputs.is_semantic_tag == 'true' + uses: actions/checkout@v3 + steps: + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: '3.x' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install build + - name: Build package + run: python -m build + - name: Publish package + uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 + with: + user: __token__ + password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml deleted file mode 100644 index bdaab28..0000000 --- a/.github/workflows/python-publish.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries - -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package - -on: - release: - types: [published] - -permissions: - contents: read - -jobs: - deploy: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: '3.x' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install build - - name: Build package - run: python -m build - - name: Publish package - uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29 - with: - user: __token__ - password: ${{ secrets.PYPI_API_TOKEN }} diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index b53122e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,23 +0,0 @@ -sudo: required -language: bash -services: -- docker -before_script: -- docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN" -- docker-compose build sapi-python-client -script: -- docker-compose run --rm --entrypoint=flake8 sapi-python-client -- docker-compose run --rm -e KBC_TEST_TOKEN -e KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover -- docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_AZ_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_AZ_TEST_API_URL sapi-python-client -m unittest discover -after_success: -- docker images -deploy: - provider: script - skip_cleanup: true - script: "./deploy.sh" - on: - tags: true -notifications: - slack: - rooms: - secure: IHfa2pHnjC/lY3H4062fTTz7y1CmHw0FbVtyxG8mMD9iuAbEIvgOs8+BRLFzqx6Baej1TnELFek7QCFdMV5SVDda7XMCOY5WLMoG1YoQ4UtnYE2mhvEsoKEu5C73rZ/OIm1jIihLFGpznwhq+ZSbPT2wiKfJglic87NW5XPsYE52XvZRQ3RY0dZpTxpXIdKOKUvcMK0FZfzDbkGHOB8GTVRNyaa1r4uI4XTyNW3c3l8sTOQo3l4rCtQXyMVqWIKXywxSfo7r+h1XPqrxpUtm+2d/B05bYRlD39OGR3O5LwK1YDFl7F3Dc9u503IhMs1p5gc3jRb4L1QCJXy0VcXgHPyaKMSnE6Ambja6S8oXXao167iR7qJ3sHgd1PAs48pSZPoqSvVB8K26X59l9jKf0xAqFm2tZVEOuYD2yGKoWpB4f201yTfVQaQ69P/lLFhao+kI2XuupCgyVTeuOFyc4fYS6k1ooJvFY8NPlhnCVW6xYJCfRcy0gYFXCAVgqpl6Icm4nMVQ66AU6tQckfuZK/7vfpw1SsfifeHLFLQp0AiXydS4ozhFYzejDxVnKH8a1S4ZGfvgPEVfc2xp5C8nJR8YJn9KA3V0JJ01foBofymWnB2QPLUQr3iaLbPsep3ixtjxTSTTjFWCv5kd6Sh193KcK/0RkHWs+1Tz0Lb+FZM= From fdb71ee05b7106b5142c2022f1c25196831e63ff Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 1 Feb 2023 14:52:51 +0100 Subject: [PATCH 04/18] fix workflow config --- .github/workflows/push.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 0b0e853..9ea6357 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -21,8 +21,9 @@ jobs: runs-on: ubuntu-latest outputs: is_semantic_tag: ${{steps.tag.outputs.is_semantic_tag}} - uses: actions/checkout@v3 steps: + - name: Checkout the repo + uses: actions/checkout@v3 - name: Print Docker version run: | docker -v @@ -49,8 +50,9 @@ jobs: - build_and_test runs-on: ubuntu-latest if: startsWith(github.ref, 'refs/tags/') && needs.build_and_test.outputs.is_semantic_tag == 'true' - uses: actions/checkout@v3 steps: + - name: Checkout the repo + uses: actions/checkout@v3 - name: Set up Python uses: actions/setup-python@v3 with: From 3d056294e339c1ed7308d247a3b5997978ed1b36 Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 1 Feb 2023 15:09:50 +0100 Subject: [PATCH 05/18] allow on dispatch --- .github/workflows/push.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 9ea6357..338c3c8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,14 +1,8 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries -# This workflow uses actions that are not certified by GitHub. -# They are provided by a third-party and are governed by -# separate terms of service, privacy policy, and support -# documentation. - -name: Upload Python Package +name: Build test and release on: + workflow_dispatch: release: types: [published] @@ -16,7 +10,6 @@ permissions: contents: read jobs: - build_and_test: runs-on: ubuntu-latest outputs: From 70bf0823892695e7063ebed1b642db0d4b37047d Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 1 Feb 2023 15:31:04 +0100 Subject: [PATCH 06/18] on push --- .github/workflows/push.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 338c3c8..bdabe10 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -3,8 +3,7 @@ name: Build test and release on: workflow_dispatch: - release: - types: [published] + push: permissions: contents: read From c95ce03534e542253cd91d38a43a02b113f5ba52 Mon Sep 17 00:00:00 2001 From: marc Date: Wed, 1 Feb 2023 15:38:53 +0100 Subject: [PATCH 07/18] fix env --- .github/workflows/push.yml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index bdabe10..5870137 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,9 +5,6 @@ on: workflow_dispatch: push: -permissions: - contents: read - jobs: build_and_test: runs-on: ubuntu-latest @@ -25,11 +22,24 @@ jobs: - name: build run: | docker-compose build sapi-python-client + - name: debug + env: + KBC_TEST_TOKEN: ${{ secrets.KBC_TEST_TOKEN }} + KBC_TEST_API_URL: https://connection.keboola.com + KBC_AZ_TEST_TOKEN: ${{ secrets.KBC_AZ_TEST_TOKEN }} + KBC_AZ_TEST_API_URL: https://connection.north-europe.azure.keboola.com + run: echo "my kbc $KBC_TEST_API_URL test token is $KBC_TEST_TOKEN" - name: run tests + env: + KBC_TEST_TOKEN: ${{ secrets.KBC_TEST_TOKEN }} + KBC_TEST_API_URL: https://connection.keboola.com + KBC_AZ_TEST_TOKEN: ${{ secrets.KBC_AZ_TEST_TOKEN }} + KBC_AZ_TEST_API_URL: https://connection.north-europe.azure.keboola.com run: | docker-compose run --rm --entrypoint=flake8 sapi-python-client - docker-compose run --rm -e KBC_TEST_TOKEN -e KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover + docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_AZ_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_AZ_TEST_API_URL sapi-python-client -m unittest discover + - name: Check if tag is semantic id: tag run: | From b402f2af0307695878462b1d3550cf39079d16e6 Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 2 Feb 2023 17:04:09 +0100 Subject: [PATCH 08/18] separate tests --- .github/workflows/push.yml | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 5870137..1629f46 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -5,6 +5,12 @@ on: workflow_dispatch: push: +env: + KBC_TEST_TOKEN: ${{ secrets.KBC_TEST_TOKEN }} + KBC_TEST_API_URL: https://connection.keboola.com + KBC_AZ_TEST_TOKEN: ${{ secrets.KBC_AZ_TEST_TOKEN }} + KBC_AZ_TEST_API_URL: https://connection.north-europe.azure.keboola.com + jobs: build_and_test: runs-on: ubuntu-latest @@ -22,23 +28,14 @@ jobs: - name: build run: | docker-compose build sapi-python-client - - name: debug - env: - KBC_TEST_TOKEN: ${{ secrets.KBC_TEST_TOKEN }} - KBC_TEST_API_URL: https://connection.keboola.com - KBC_AZ_TEST_TOKEN: ${{ secrets.KBC_AZ_TEST_TOKEN }} - KBC_AZ_TEST_API_URL: https://connection.north-europe.azure.keboola.com - run: echo "my kbc $KBC_TEST_API_URL test token is $KBC_TEST_TOKEN" - - name: run tests - env: - KBC_TEST_TOKEN: ${{ secrets.KBC_TEST_TOKEN }} - KBC_TEST_API_URL: https://connection.keboola.com - KBC_AZ_TEST_TOKEN: ${{ secrets.KBC_AZ_TEST_TOKEN }} - KBC_AZ_TEST_API_URL: https://connection.north-europe.azure.keboola.com - run: | - docker-compose run --rm --entrypoint=flake8 sapi-python-client - docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover - docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_AZ_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_AZ_TEST_API_URL sapi-python-client -m unittest discover + - name: show images + run: docker images + - name: run tests flake8 + run: docker-compose run --rm --entrypoint=flake8 sapi-python-client + - name: run tests aws + run: docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_TEST_API_URL -e SKIP_ABS_TEST=1 sapi-python-client -m unittest discover + - name: run tests azure + run: docker-compose run --rm -e KBC_TEST_TOKEN=$KBC_AZ_TEST_TOKEN -e KBC_TEST_API_URL=$KBC_AZ_TEST_API_URL sapi-python-client -m unittest discover - name: Check if tag is semantic id: tag From 6bfd1304522e6bf6894d2582bf6d5193068cb538 Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 9 Feb 2023 09:57:16 +0100 Subject: [PATCH 09/18] add assync bucket delete" --- .github/workflows/push.yml | 3 +-- Dockerfile | 2 +- kbcstorage/buckets.py | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 1629f46..d0b3a4f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -26,8 +26,7 @@ jobs: if: env.DOCKERHUB_TOKEN run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN" - name: build - run: | - docker-compose build sapi-python-client + run: docker-compose build sapi-python-client - name: show images run: docker images - name: run tests flake8 diff --git a/Dockerfile b/Dockerfile index 961886a..86a6b31 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,6 @@ FROM python:3.8 WORKDIR /code COPY . /code/ -RUN pip3 install --no-cache-dir flake8 responses +RUN pip3 install --no-cache-dir flake8 responses typing_extensions RUN python setup.py install ENTRYPOINT ["python"] diff --git a/kbcstorage/buckets.py b/kbcstorage/buckets.py index 5203df1..f16498b 100644 --- a/kbcstorage/buckets.py +++ b/kbcstorage/buckets.py @@ -100,7 +100,7 @@ def create(self, name, stage='in', description='', backend=None): return self._post(self.base_url, data=body) - def delete(self, bucket_id, force=False): + def delete(self, bucket_id, force=False, asynchronous=True): """ Delete a bucket referenced by ``bucket_id``. @@ -116,7 +116,7 @@ def delete(self, bucket_id, force=False): # How does the API handle it when force == False and the bucket is non- # empty? url = '{}/{}'.format(self.base_url, bucket_id) - params = {'force': force} + params = {'force': force, 'async': asynchronous} self._delete(url, params=params) def link(self, *args, **kwargs): From 8fda8095e5013bdf148784f0fe038de8789016c8 Mon Sep 17 00:00:00 2001 From: marc Date: Thu, 9 Feb 2023 15:33:05 +0100 Subject: [PATCH 10/18] wait for delete --- tests/functional/test_buckets.py | 4 ++++ tests/functional/test_tables.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/tests/functional/test_buckets.py b/tests/functional/test_buckets.py index 0384313..7d88092 100644 --- a/tests/functional/test_buckets.py +++ b/tests/functional/test_buckets.py @@ -1,6 +1,7 @@ import csv import os import tempfile +import time import unittest import warnings @@ -18,6 +19,9 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise + # wait for asynchronous call completion + time.sleep(2) + # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index cefa1b3..9a60614 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -2,6 +2,7 @@ import unittest import tempfile import csv +import time import warnings from requests import exceptions from kbcstorage.tables import Tables @@ -19,6 +20,8 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise + # the delete is asyncronous so let's wait a bit until it finishes + time.sleep(3) self.buckets.create(name='py-test-tables', stage='in') # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) From 22c70e8e66459f1d339994b7f4f7b34576f031b1 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 10 Feb 2023 14:11:49 +0100 Subject: [PATCH 11/18] sleep more --- tests/functional/test_client.py | 3 +++ tests/functional/test_files.py | 2 ++ tests/functional/test_workspaces.py | 4 ++++ 3 files changed, 9 insertions(+) diff --git a/tests/functional/test_client.py b/tests/functional/test_client.py index 208cab8..f697b8e 100644 --- a/tests/functional/test_client.py +++ b/tests/functional/test_client.py @@ -1,6 +1,7 @@ import csv import os import tempfile +import time import unittest import warnings @@ -17,6 +18,8 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise + # give the async call time to complete + time.sleep(2) # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/functional/test_files.py b/tests/functional/test_files.py index 4ef8b37..c363fcc 100644 --- a/tests/functional/test_files.py +++ b/tests/functional/test_files.py @@ -145,6 +145,8 @@ def test_download_file_sliced(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise + # wait for async job to complete + time.sleep(2) buckets.create(name='py-test-files', stage='in') tables = Tables(os.getenv('KBC_TEST_API_URL'), diff --git a/tests/functional/test_workspaces.py b/tests/functional/test_workspaces.py index 3182773..eb78dee 100644 --- a/tests/functional/test_workspaces.py +++ b/tests/functional/test_workspaces.py @@ -1,6 +1,7 @@ import csv import os import tempfile +import time import unittest import warnings @@ -33,6 +34,9 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise + # give the async call time to complete + time.sleep(2) + # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) From bc2c31f49815f7b034cc6b5802f384c7e6f0df59 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 09:56:58 +0100 Subject: [PATCH 12/18] use async deletes --- kbcstorage/base.py | 6 ++++-- kbcstorage/buckets.py | 13 ++++++++++++- tests/mocks/test_buckets.py | 6 ++++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/kbcstorage/base.py b/kbcstorage/base.py index 17e82dc..715e537 100644 --- a/kbcstorage/base.py +++ b/kbcstorage/base.py @@ -10,7 +10,7 @@ http://docs.keboola.apiary.io/ """ import requests - +import json.decoder class Endpoint: """ @@ -145,4 +145,6 @@ def _delete(self, *args, **kwargs): except requests.HTTPError: # Handle different error codes raise - # Should delete return something on success? + + if 'application/json' in r.headers.get('Content-Type', ''): + return r.json() diff --git a/kbcstorage/buckets.py b/kbcstorage/buckets.py index f16498b..1455bdd 100644 --- a/kbcstorage/buckets.py +++ b/kbcstorage/buckets.py @@ -7,6 +7,7 @@ http://docs.keboola.apiary.io/#reference/buckets/ """ from kbcstorage.base import Endpoint +from kbcstorage.jobs import Jobs class Buckets(Endpoint): @@ -112,12 +113,22 @@ def delete(self, bucket_id, force=False, asynchronous=True): bucket_id (str): The id of the bucket to be deleted. force (bool): If ``True``, deletes the bucket even if it is not empty. Default ``False``. + asynchronous (bool): if asynchronous then the """ # How does the API handle it when force == False and the bucket is non- # empty? url = '{}/{}'.format(self.base_url, bucket_id) params = {'force': force, 'async': asynchronous} - self._delete(url, params=params) + if (asynchronous): + job = self._delete(url, params=params) + jobs = Jobs(self.root_url, self.token) + job = jobs.block_until_completed(job['id']) + if job['status'] == 'error': + raise RuntimeError(job['error']['message']) + else: + self._delete(url, params=params) + + def link(self, *args, **kwargs): """ diff --git a/tests/mocks/test_buckets.py b/tests/mocks/test_buckets.py index 6af60a1..4a3ca8e 100644 --- a/tests/mocks/test_buckets.py +++ b/tests/mocks/test_buckets.py @@ -51,12 +51,14 @@ def test_delete(self): responses.Response( method='DELETE', url='https://connection.keboola.com/v2/storage/buckets/1', - json={} + json={ + 'id': '12345' + } ) ) bucket_id = '1' deleted_detail = self.buckets.delete(bucket_id) - assert deleted_detail is None + assert deleted_detail.id is '12345' @responses.activate def test_create(self): From 46c4d66935e69ba9afb18dfe8e492477b1d8ead9 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 11:43:08 +0100 Subject: [PATCH 13/18] cs fixes --- kbcstorage/base.py | 2 +- kbcstorage/buckets.py | 2 -- tests/mocks/test_buckets.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/kbcstorage/base.py b/kbcstorage/base.py index 715e537..4f8afd8 100644 --- a/kbcstorage/base.py +++ b/kbcstorage/base.py @@ -10,7 +10,7 @@ http://docs.keboola.apiary.io/ """ import requests -import json.decoder + class Endpoint: """ diff --git a/kbcstorage/buckets.py b/kbcstorage/buckets.py index 1455bdd..6146229 100644 --- a/kbcstorage/buckets.py +++ b/kbcstorage/buckets.py @@ -128,8 +128,6 @@ def delete(self, bucket_id, force=False, asynchronous=True): else: self._delete(url, params=params) - - def link(self, *args, **kwargs): """ **Not implemented** diff --git a/tests/mocks/test_buckets.py b/tests/mocks/test_buckets.py index 4a3ca8e..816956a 100644 --- a/tests/mocks/test_buckets.py +++ b/tests/mocks/test_buckets.py @@ -58,7 +58,7 @@ def test_delete(self): ) bucket_id = '1' deleted_detail = self.buckets.delete(bucket_id) - assert deleted_detail.id is '12345' + assert deleted_detail.id == '12345' @responses.activate def test_create(self): From d08b5188c2278ca48a54cfd9dda0214a7313042f Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 13:10:00 +0100 Subject: [PATCH 14/18] fix delete mock test --- tests/mocks/test_buckets.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/mocks/test_buckets.py b/tests/mocks/test_buckets.py index 816956a..8650fe4 100644 --- a/tests/mocks/test_buckets.py +++ b/tests/mocks/test_buckets.py @@ -51,14 +51,11 @@ def test_delete(self): responses.Response( method='DELETE', url='https://connection.keboola.com/v2/storage/buckets/1', - json={ - 'id': '12345' - } + json={} ) ) - bucket_id = '1' - deleted_detail = self.buckets.delete(bucket_id) - assert deleted_detail.id == '12345' + deleted_detail = self.buckets.delete(bucket_id, asynchronous=False) + assert deleted_detail is None @responses.activate def test_create(self): From 063f089e051730599be4405c7236ca9c32f5b2e3 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 13:37:56 +0100 Subject: [PATCH 15/18] remove unnecessary sleeps --- tests/functional/test_buckets.py | 2 -- tests/functional/test_client.py | 2 -- tests/functional/test_files.py | 7 ------- tests/functional/test_tables.py | 2 -- tests/functional/test_workspaces.py | 2 -- tests/mocks/test_buckets.py | 3 ++- 6 files changed, 2 insertions(+), 16 deletions(-) diff --git a/tests/functional/test_buckets.py b/tests/functional/test_buckets.py index 7d88092..f2b1dfa 100644 --- a/tests/functional/test_buckets.py +++ b/tests/functional/test_buckets.py @@ -19,8 +19,6 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise - # wait for asynchronous call completion - time.sleep(2) # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/functional/test_client.py b/tests/functional/test_client.py index f697b8e..18be4d8 100644 --- a/tests/functional/test_client.py +++ b/tests/functional/test_client.py @@ -18,8 +18,6 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise - # give the async call time to complete - time.sleep(2) # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/functional/test_files.py b/tests/functional/test_files.py index c363fcc..8f2d5dd 100644 --- a/tests/functional/test_files.py +++ b/tests/functional/test_files.py @@ -13,9 +13,6 @@ class TestFiles(unittest.TestCase): def setUp(self): - - # timeout for files from previous tests to appear - time.sleep(1) self.files = Files(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN')) files = self.files.list(tags=['py-test']) @@ -25,8 +22,6 @@ def setUp(self): warnings.simplefilter("ignore", ResourceWarning) def tearDown(self): - # timeout for files from previous tests to appear - time.sleep(1) files = self.files.list(tags=['py-test']) for file in files: self.files.delete(file['id']) @@ -145,8 +140,6 @@ def test_download_file_sliced(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise - # wait for async job to complete - time.sleep(2) buckets.create(name='py-test-files', stage='in') tables = Tables(os.getenv('KBC_TEST_API_URL'), diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 9a60614..1fe2e08 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -20,8 +20,6 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise - # the delete is asyncronous so let's wait a bit until it finishes - time.sleep(3) self.buckets.create(name='py-test-tables', stage='in') # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/functional/test_workspaces.py b/tests/functional/test_workspaces.py index eb78dee..e648446 100644 --- a/tests/functional/test_workspaces.py +++ b/tests/functional/test_workspaces.py @@ -34,8 +34,6 @@ def setUp(self): except exceptions.HTTPError as e: if e.response.status_code != 404: raise - # give the async call time to complete - time.sleep(2) # https://github.com/boto/boto3/issues/454 warnings.simplefilter("ignore", ResourceWarning) diff --git a/tests/mocks/test_buckets.py b/tests/mocks/test_buckets.py index 8650fe4..0b20138 100644 --- a/tests/mocks/test_buckets.py +++ b/tests/mocks/test_buckets.py @@ -50,10 +50,11 @@ def test_delete(self): responses.add( responses.Response( method='DELETE', - url='https://connection.keboola.com/v2/storage/buckets/1', + url='https://connection.keboola.com/v2/storage/buckets/1?async=False', json={} ) ) + bucket_id = '1' deleted_detail = self.buckets.delete(bucket_id, asynchronous=False) assert deleted_detail is None From e4fbe4f551d89fc69bbb70dcad9f59bc4e46cee5 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 13:45:08 +0100 Subject: [PATCH 16/18] remove unused imports --- tests/functional/test_buckets.py | 1 - tests/functional/test_client.py | 1 - tests/functional/test_tables.py | 1 - tests/functional/test_workspaces.py | 1 - 4 files changed, 4 deletions(-) diff --git a/tests/functional/test_buckets.py b/tests/functional/test_buckets.py index f2b1dfa..9899e75 100644 --- a/tests/functional/test_buckets.py +++ b/tests/functional/test_buckets.py @@ -1,7 +1,6 @@ import csv import os import tempfile -import time import unittest import warnings diff --git a/tests/functional/test_client.py b/tests/functional/test_client.py index 18be4d8..208cab8 100644 --- a/tests/functional/test_client.py +++ b/tests/functional/test_client.py @@ -1,7 +1,6 @@ import csv import os import tempfile -import time import unittest import warnings diff --git a/tests/functional/test_tables.py b/tests/functional/test_tables.py index 1fe2e08..cefa1b3 100644 --- a/tests/functional/test_tables.py +++ b/tests/functional/test_tables.py @@ -2,7 +2,6 @@ import unittest import tempfile import csv -import time import warnings from requests import exceptions from kbcstorage.tables import Tables diff --git a/tests/functional/test_workspaces.py b/tests/functional/test_workspaces.py index e648446..be2fbf8 100644 --- a/tests/functional/test_workspaces.py +++ b/tests/functional/test_workspaces.py @@ -1,7 +1,6 @@ import csv import os import tempfile -import time import unittest import warnings From 9aefe4b92a76baad2b651e8e157e27c0569f94c6 Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 14:05:43 +0100 Subject: [PATCH 17/18] put back sleep for files operations --- tests/functional/test_files.py | 4 ++++ tests/mocks/test_buckets.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/test_files.py b/tests/functional/test_files.py index 8f2d5dd..cabdb2b 100644 --- a/tests/functional/test_files.py +++ b/tests/functional/test_files.py @@ -13,6 +13,8 @@ class TestFiles(unittest.TestCase): def setUp(self): + # timeout for files from previous tests to appear + time.sleep(1) self.files = Files(os.getenv('KBC_TEST_API_URL'), os.getenv('KBC_TEST_TOKEN')) files = self.files.list(tags=['py-test']) @@ -22,6 +24,8 @@ def setUp(self): warnings.simplefilter("ignore", ResourceWarning) def tearDown(self): + # timeout for files from previous tests to appear + time.sleep(1) files = self.files.list(tags=['py-test']) for file in files: self.files.delete(file['id']) diff --git a/tests/mocks/test_buckets.py b/tests/mocks/test_buckets.py index 0b20138..fba81ed 100644 --- a/tests/mocks/test_buckets.py +++ b/tests/mocks/test_buckets.py @@ -50,7 +50,7 @@ def test_delete(self): responses.add( responses.Response( method='DELETE', - url='https://connection.keboola.com/v2/storage/buckets/1?async=False', + url='https://connection.keboola.com/v2/storage/buckets/1?force=False&async=False', json={} ) ) From dd0ea443c0f144ebab1ca7867d6a08ae37b0c8ff Mon Sep 17 00:00:00 2001 From: marc Date: Fri, 24 Feb 2023 16:06:11 +0100 Subject: [PATCH 18/18] update for deprecation --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d0b3a4f..7def27b 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -42,7 +42,7 @@ jobs: TAG="${GITHUB_REF##*/}" IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false) echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'" - echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG" + echo "is_semantic_tag=$IS_SEMANTIC_TAG" >> $GITHUB_OUTPUT deploy_to_pypi: needs: - build_and_test