Feature/python sync actions pagination

python-sync-actions/requirements.txt

@@ -6,4 +6,4 @@ keboola.json-to-csv
 mock
 freezegun
 nested-lookup
-python-dateutil
+python-dateutil

python-sync-actions/src/component.py

@@ -239,16 +239,51 @@ def _parse_data(self, data, path) -> list:
                     value = value[key]
                 result = value
             except KeyError:
-                raise UserException(f"Path {path.path} not found in the response data")
+                return [f"Path {path.path} not found in the response data"]
 
         # TODO: check if the result is list
-        # if not isinstance(result, list):
-        #     element_name = 'root' if path.path == '.' else path.path
-        #     raise UserException(f"The {element_name} element of the response is not list, "
-        #                         "please change your Record Selector path to list")
+        if not isinstance(result, list):
+            element_name = 'root' if path.path == '.' else path.path
+            return [f"The {element_name} element of the response is not a list, "
+                    f"please change your Record Selector path to list"]
 
         return result
 
+    def _get_paginator(self, job):
+        """
+        Get paginator object for the given job.
+
+        Args:
+            job: The job for which to get the paginator.
+
+        Returns:
+            The paginator object for the job.
+        """
+
+        paginator = {}
+
+        if job.request_parameters.scroller:
+            paginator_params = job.api.pagination.get(job.request_parameters.scroller)
+            if not paginator_params:
+                raise UserException(f"Paginator '{job.request_parameters.scroller}' not found in the configuration.")
+
+        else:
+            paginator_params = job.api.pagination.get("common")
+
+        if paginator_params.get("method") == "offset":
+            if paginator_params.get("firstPageParams", True):
+                paginator[paginator_params.get("offsetParam", "offset")] = paginator_params.get("offset", 0)
+                paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+
+        elif paginator_params.get("method") == "pagenum":
+            if paginator_params.get("firstPageParams"):
+                paginator[paginator_params.get("pageParam", "page")] = paginator_params.get("firstPage", 1)
+
+                if paginator_params.get("limit"):
+                    paginator[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+
+        return paginator
+
     def make_call(self) -> tuple[list, any, str, str]:
         """
         Make call to the API
@@ -294,6 +329,10 @@ def recursive_call(parent_result, config_index=0):
             ssl_verify = api_cfg.ssl_verification
             timeout = api_cfg.timeout
             # additional_params = self._build_request_parameters(additional_params_cfg)
+
+            paginator = self._get_paginator(job)
+            query_parameters.update(paginator)
+
             request_parameters = {'params': query_parameters,
                                   'headers': new_headers,
                                   'verify': ssl_verify,

python-sync-actions/src/configuration.py

@@ -68,6 +68,7 @@ class ApiConfig(ConfigurationBase):
     base_url: str
     default_query_parameters: dict = field(default_factory=dict)
     default_headers: dict = field(default_factory=dict)
+    pagination: dict = field(default_factory=dict)
     authentication: Authentication = None
     retry_config: RetryConfig = field(default_factory=RetryConfig)
     ssl_verification: bool = True
@@ -83,6 +84,7 @@ class ApiRequest(ConfigurationBase):
     query_parameters: dict = field(default_factory=dict)
     continue_on_failure: bool = False
     nested_job: dict = field(default_factory=dict)
+    scroller: str = None
 
 
 @dataclass
@@ -183,8 +185,14 @@ def convert_to_v2(configuration: dict) -> list[Configuration]:
     default_headers = _remove_auth_from_dict(default_headers_org, _return_ui_params(configuration))
     default_query_parameters = _remove_auth_from_dict(default_query_parameters_org, _return_ui_params(configuration))
 
+    pagination = {}
+    if api_json.get('pagination', {}).get('scrollers'):
+        pagination = api_json.get('pagination', {}).get('scrollers')
+    else:
+        pagination['common'] = api_json.get('pagination', {})
+
     api_config = ApiConfig(base_url=base_url, default_headers=default_headers,
-                           default_query_parameters=default_query_parameters)
+                           default_query_parameters=default_query_parameters, pagination=pagination)
 
     api_config.retry_config = build_retry_config(configuration)
     api_config.authentication = AuthMethodConverter.convert(configuration)
@@ -285,6 +293,8 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
 
         placeholders = endpoint_config.get('placeholders', {})
 
+        scroller = endpoint_config.get('scroller')
+
         if isinstance(data_field, dict):
             path = data_field.get('path')
             delimiter = data_field.get("delimiter", ".")
@@ -297,7 +307,8 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
                         endpoint_path=endpoint_path,
                         placeholders=placeholders,
                         headers=endpoint_config.get('headers', {}),
-                        query_parameters=endpoint_config.get('params', {}), ),
+                        query_parameters=endpoint_config.get('params', {}),
+                        scroller=scroller),
              request_content,
              DataPath(path=path, delimiter=delimiter)))
 

python-sync-actions/tests/data_tests/test_003_oauth_cc/config.json

@@ -81,8 +81,7 @@
           "dataField": {
             "path": ".",
             "delimiter": "."
-          },
-          "scroller": "events"
+          }
         }
       ],
       "includes": {

python-sync-actions/tests/data_tests/test_004_oauth_cc_post/config.json

@@ -64,8 +64,7 @@
           "dataField": {
             "path": ".",
             "delimiter": "."
-          },
-          "scroller": "events"
+          }
         }
       ],
       "includes": {