default params, common pagination

keboola · May 28, 2024 · 4fe76f5 · 4fe76f5
1 parent 0ad34c1
commit 4fe76f5
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 17 deletions.
diff --git a/python-sync-actions/requirements.txt b/python-sync-actions/requirements.txt
@@ -7,4 +7,3 @@ mock
 freezegun
 nested-lookup
 python-dateutil
-dlt==0.4.12
diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py
@@ -19,7 +19,6 @@
 from http_generic.auth import AuthMethodBuilder, AuthBuilderError
 from http_generic.client import GenericHttpClient, HttpClientError
 from placeholders_utils import PlaceholdersUtils
-from dlt.sources.helpers.rest_client import paginators
 
 MAX_CHILD_CALLS = 20
 
@@ -264,20 +263,24 @@ def _get_paginator(self, job):
         paginator = {}
 
         if job.request_parameters.scroller:
-            paginator_params = job.api.pagination.get("scrollers").get(job.request_parameters.scroller)
-
-            if paginator_params.get("method") == "offset":
-                # paginator = paginators.OffsetPaginator(limit=paginator_params.get("limit"),
-                #                                        offset=paginator_params.get("offset"),
-                #                                        offset_param=paginator_params.get("offsetParam"),
-                #                                        limit_param=paginator_params.get("limitParam")
-                #                                        )
-                paginator[paginator_params.get("offsetParam")] = paginator_params.get("offset", 0)
-                paginator[paginator_params.get("limitParam")] = paginator_params.get("limit")
-
-            elif paginator_params.get("method") == "pagenum":
-                if paginator_params.get("firstPageParams"):
-                    paginator[paginator_params.get("pageParam")] = paginator_params.get("page", 1)
+            paginator_params = job.api.pagination.get(job.request_parameters.scroller)
+            if not paginator_params:
+                raise UserException(f"Paginator '{job.request_parameters.scroller}' not found in the API configuration.")
+
+        else:
+            paginator_params = job.api.pagination.get("common")
+
+        if paginator_params.get("method") == "offset":
+            if paginator_params.get("firstPageParams", True):
+                paginator[paginator_params.get("offsetParam", "offset")] = paginator_params.get("offset", 0)
+                paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+
+        elif paginator_params.get("method") == "pagenum":
+            if paginator_params.get("firstPageParams"):
+                paginator[paginator_params.get("pageParam", "page")] = paginator_params.get("firstPage", 1)
+
+                if paginator_params.get("limit"):
+                    paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
 
         return paginator
 

diff --git a/python-sync-actions/src/configuration.py b/python-sync-actions/src/configuration.py
@@ -185,7 +185,11 @@ def convert_to_v2(configuration: dict) -> list[Configuration]:
     default_headers = _remove_auth_from_dict(default_headers_org, _return_ui_params(configuration))
     default_query_parameters = _remove_auth_from_dict(default_query_parameters_org, _return_ui_params(configuration))
 
-    pagination = api_json.get('pagination', {})
+    pagination = {}
+    if api_json.get('pagination', {}).get('scrollers'):
+        pagination = api_json.get('pagination', {}).get('scrollers')
+    else:
+        pagination['common'] = api_json.get('pagination', {})
 
     api_config = ApiConfig(base_url=base_url, default_headers=default_headers,
                            default_query_parameters=default_query_parameters, pagination=pagination)