From 4fe76f556138d8436ab4e74af27cd23bd65cd537 Mon Sep 17 00:00:00 2001
From: Josef Kudera <josef.kudera@keboola.com>
Date: Tue, 28 May 2024 12:50:17 +0200
Subject: [PATCH] default params, common pagination

---
 python-sync-actions/requirements.txt     |  1 -
 python-sync-actions/src/component.py     | 33 +++++++++++++-----------
 python-sync-actions/src/configuration.py |  6 ++++-
 3 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/python-sync-actions/requirements.txt b/python-sync-actions/requirements.txt
index 78ccd7b..ca2628b 100644
--- a/python-sync-actions/requirements.txt
+++ b/python-sync-actions/requirements.txt
@@ -7,4 +7,3 @@ mock
 freezegun
 nested-lookup
 python-dateutil
-dlt==0.4.12
\ No newline at end of file
diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py
index a93f719..282036e 100644
--- a/python-sync-actions/src/component.py
+++ b/python-sync-actions/src/component.py
@@ -19,7 +19,6 @@
 from http_generic.auth import AuthMethodBuilder, AuthBuilderError
 from http_generic.client import GenericHttpClient, HttpClientError
 from placeholders_utils import PlaceholdersUtils
-from dlt.sources.helpers.rest_client import paginators
 
 MAX_CHILD_CALLS = 20
 
@@ -264,20 +263,24 @@ def _get_paginator(self, job):
         paginator = {}
 
         if job.request_parameters.scroller:
-            paginator_params = job.api.pagination.get("scrollers").get(job.request_parameters.scroller)
-
-            if paginator_params.get("method") == "offset":
-                # paginator = paginators.OffsetPaginator(limit=paginator_params.get("limit"),
-                #                                        offset=paginator_params.get("offset"),
-                #                                        offset_param=paginator_params.get("offsetParam"),
-                #                                        limit_param=paginator_params.get("limitParam")
-                #                                        )
-                paginator[paginator_params.get("offsetParam")] = paginator_params.get("offset", 0)
-                paginator[paginator_params.get("limitParam")] = paginator_params.get("limit")
-
-            elif paginator_params.get("method") == "pagenum":
-                if paginator_params.get("firstPageParams"):
-                    paginator[paginator_params.get("pageParam")] = paginator_params.get("page", 1)
+            paginator_params = job.api.pagination.get(job.request_parameters.scroller)
+            if not paginator_params:
+                raise UserException(f"Paginator '{job.request_parameters.scroller}' not found in the API configuration.")
+
+        else:
+            paginator_params = job.api.pagination.get("common")
+
+        if paginator_params.get("method") == "offset":
+            if paginator_params.get("firstPageParams", True):
+                paginator[paginator_params.get("offsetParam", "offset")] = paginator_params.get("offset", 0)
+                paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+
+        elif paginator_params.get("method") == "pagenum":
+            if paginator_params.get("firstPageParams"):
+                paginator[paginator_params.get("pageParam", "page")] = paginator_params.get("firstPage", 1)
+
+                if paginator_params.get("limit"):
+                    paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
 
         return paginator
 
diff --git a/python-sync-actions/src/configuration.py b/python-sync-actions/src/configuration.py
index fb0c76f..50474a8 100644
--- a/python-sync-actions/src/configuration.py
+++ b/python-sync-actions/src/configuration.py
@@ -185,7 +185,11 @@ def convert_to_v2(configuration: dict) -> list[Configuration]:
     default_headers = _remove_auth_from_dict(default_headers_org, _return_ui_params(configuration))
     default_query_parameters = _remove_auth_from_dict(default_query_parameters_org, _return_ui_params(configuration))
 
-    pagination = api_json.get('pagination', {})
+    pagination = {}
+    if api_json.get('pagination', {}).get('scrollers'):
+        pagination = api_json.get('pagination', {}).get('scrollers')
+    else:
+        pagination['common'] = api_json.get('pagination', {})
 
     api_config = ApiConfig(base_url=base_url, default_headers=default_headers,
                            default_query_parameters=default_query_parameters, pagination=pagination)