Merge pull request #186 from keboola/feature/python-sync-actions-pagi…

…nation-review Feature/python sync actions pagination review
keboola · Jul 18, 2024 · 62b78ba · 62b78ba
2 parents 5b1350a + c71e47a
commit 62b78ba
Show file tree

Hide file tree

Showing 25 changed files with 615 additions and 62 deletions.
diff --git a/python-sync-actions/requirements.txt b/python-sync-actions/requirements.txt
@@ -6,4 +6,4 @@ keboola.json-to-csv
 mock
 freezegun
 nested-lookup
-python-dateutil
+python-dateutil
diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py
@@ -18,6 +18,7 @@
 from configuration import Configuration, ConfigHelpers
 from http_generic.auth import AuthMethodBuilder, AuthBuilderError
 from http_generic.client import GenericHttpClient, HttpClientError
+from http_generic.pagination import PaginationBuilder
 from placeholders_utils import PlaceholdersUtils
 
 MAX_CHILD_CALLS = 20
@@ -288,6 +289,36 @@ def find_array_property_path(response_data: dict, result_arrays: list = None) ->
 
         return result
 
+    def _add_page_params(self, job: Configuration, query_parameters: dict) -> dict:
+        """
+        Add page parameters to the query parameters
+        Args:
+            job: job configuration
+            query_parameters: query parameters
+
+        Returns:
+            query_parameters: updated query parameters
+        """
+
+        if not job.api.pagination:
+            return query_parameters
+
+        paginator_config = job.api.pagination.get(job.request_parameters.scroller)
+        if not paginator_config:
+            raise UserException(f"Paginator '{job.request_parameters.scroller}' not found in the configuration.")
+
+        paginaton_method = PaginationBuilder.get_paginator(paginator_config.get("method"))
+        paginator_params = paginaton_method.get_page_params(paginator_config)
+
+        if paginator_config.get("offsetFromJob"):
+            for key, value in paginator_params.items():
+                if key not in query_parameters:
+                    query_parameters[key] = value
+        else:
+            query_parameters.update(paginator_params)
+
+        return query_parameters
+
     def make_call(self) -> tuple[list, any, str, str]:
         """
         Make call to the API
@@ -332,6 +363,9 @@ def recursive_call(parent_result, config_index=0):
             ssl_verify = api_cfg.ssl_verification
             timeout = api_cfg.timeout
             # additional_params = self._build_request_parameters(additional_params_cfg)
+
+            query_parameters = self._add_page_params(job, query_parameters)
+
             request_parameters = {'params': query_parameters,
                                   'headers': new_headers,
                                   'verify': ssl_verify,

diff --git a/python-sync-actions/src/configuration.py b/python-sync-actions/src/configuration.py
@@ -63,11 +63,18 @@ class Authentication(ConfigurationBase):
     parameters: dict = field(default_factory=dict)
 
 
+@dataclass
+class Pagination(ConfigurationBase):
+    type: str
+    parameters: dict = field(default_factory=dict)
+
+
 @dataclass
 class ApiConfig(ConfigurationBase):
     base_url: str
     default_query_parameters: dict = field(default_factory=dict)
     default_headers: dict = field(default_factory=dict)
+    pagination: dict = field(default_factory=dict)
     authentication: Authentication = None
     retry_config: RetryConfig = field(default_factory=RetryConfig)
     ssl_verification: bool = True
@@ -83,6 +90,8 @@ class ApiRequest(ConfigurationBase):
     query_parameters: dict = field(default_factory=dict)
     continue_on_failure: bool = False
     nested_job: dict = field(default_factory=dict)
+    scroller: str = None
+    pagination: Pagination = None
 
 
 @dataclass
@@ -183,8 +192,14 @@ def convert_to_v2(configuration: dict) -> list[Configuration]:
     default_headers = _remove_auth_from_dict(default_headers_org, _return_ui_params(configuration))
     default_query_parameters = _remove_auth_from_dict(default_query_parameters_org, _return_ui_params(configuration))
 
+    pagination = {}
+    if api_json.get('pagination', {}).get('scrollers'):
+        pagination = api_json.get('pagination', {}).get('scrollers')
+    elif api_json.get('pagination'):
+        pagination['common'] = api_json.get('pagination')
+
     api_config = ApiConfig(base_url=base_url, default_headers=default_headers,
-                           default_query_parameters=default_query_parameters)
+                           default_query_parameters=default_query_parameters, pagination=pagination)
 
     api_config.retry_config = build_retry_config(configuration)
     api_config.authentication = AuthMethodConverter.convert(configuration)
@@ -285,6 +300,8 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
 
         placeholders = endpoint_config.get('placeholders', {})
 
+        scroller = endpoint_config.get('scroller', 'common')
+
         if isinstance(data_field, dict):
             path = data_field.get('path')
             delimiter = data_field.get("delimiter", ".")
@@ -301,7 +318,8 @@ def build_api_request(configuration: dict) -> List[Tuple[ApiRequest, RequestCont
                         endpoint_path=endpoint_path,
                         placeholders=placeholders,
                         headers=endpoint_config.get('headers', {}),
-                        query_parameters=endpoint_config.get('params', {}), ),
+                        query_parameters=endpoint_config.get('params', {}),
+                        scroller=scroller),
              request_content,
              data_path))
 

diff --git a/python-sync-actions/src/http_generic/pagination.py b/python-sync-actions/src/http_generic/pagination.py
@@ -0,0 +1,40 @@
+class BasePagination:
+    def get_page_params(self, paginator_params):
+        raise NotImplementedError("Subclasses should implement this method")
+
+
+class DummyPagination(BasePagination):
+    def get_page_params(self, paginator_params):
+        return {}
+
+
+class OffsetPagination(BasePagination):
+    def get_page_params(self, paginator_params):
+        page_params = {}
+        if paginator_params.get("firstPageParams", True):
+            page_params[paginator_params.get("offsetParam", "offset")] = paginator_params.get("offset", 0)
+            page_params[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+        return page_params
+
+
+class PageNumPagination(BasePagination):
+    def get_page_params(self, paginator_params):
+        page_params = {}
+        if paginator_params.get("firstPageParams"):
+            page_params[paginator_params.get("pageParam", "page")] = paginator_params.get("firstPage", 1)
+            if paginator_params.get("limit"):
+                page_params[paginator_params.get("limitParam", "limit")] = paginator_params.get("limit")
+        return page_params
+
+
+class PaginationBuilder:
+
+    @classmethod
+    def get_paginator(cls, pagination):
+        """Factory function to create the appropriate paginator configuration."""
+        if pagination == 'offset':
+            return OffsetPagination()
+        elif pagination == 'pagenum':
+            return PageNumPagination()
+        else:
+            return DummyPagination()
diff --git a/python-sync-actions/tests/calls/002-token-body/orders.requestHeaders b/python-sync-actions/tests/calls/002-token-body/orders.requestHeaders
@@ -1 +1 @@
-accept: */*
+accept: */*
diff --git a/python-sync-actions/tests/calls/005-query/orders.requestHeaders b/python-sync-actions/tests/calls/005-query/orders.requestHeaders
@@ -1 +1 @@
-accept: */*
+accept: */*
diff --git a/python-sync-actions/tests/calls/006-login-auth-headers/config.json b/python-sync-actions/tests/calls/006-login-auth-headers/config.json
@@ -1,36 +1,36 @@
-{
-  "parameters": {
-    "__SELECTED_JOB": "0",
-    "api": {
-      "baseUrl": "http://mock-server:80/006-login-auth-headers/",
-      "authentication": {
-        "type": "login",
-        "loginRequest": {
-          "endpoint": "login",
-          "method": "GET",
-          "headers": {
-            "X-Login": "JohnDoe",
-            "X-Password": "TopSecret"
-          }
-        },
-        "apiRequest": {
-          "headers": {
-            "X-ApiToken": {
-              "response": "authorization.token"
-            }
-          }
-        }
-      }
-    },
-    "config": {
-      "debug": true,
-      "outputBucket": "mock-server",
-      "jobs": [
-        {
-          "endpoint": "users"
-        }
-      ]
-    }
-  },
-  "action": "test_request"
-}
+{
+  "parameters": {
+    "__SELECTED_JOB": "0",
+    "api": {
+      "baseUrl": "http://mock-server:80/006-login-auth-headers/",
+      "authentication": {
+        "type": "login",
+        "loginRequest": {
+          "endpoint": "login",
+          "method": "GET",
+          "headers": {
+            "X-Login": "JohnDoe",
+            "X-Password": "TopSecret"
+          }
+        },
+        "apiRequest": {
+          "headers": {
+            "X-ApiToken": {
+              "response": "authorization.token"
+            }
+          }
+        }
+      }
+    },
+    "config": {
+      "debug": true,
+      "outputBucket": "mock-server",
+      "jobs": [
+        {
+          "endpoint": "users"
+        }
+      ]
+    }
+  },
+  "action": "test_request"
+}
diff --git a/python-sync-actions/tests/calls/006-login-auth-headers/login.requestHeaders b/python-sync-actions/tests/calls/006-login-auth-headers/login.requestHeaders
@@ -1,2 +1,2 @@
-X-Login: JohnDoe
+X-Login: JohnDoe
 X-Password: TopSecret
diff --git a/python-sync-actions/tests/calls/006-login-auth-headers/login.response b/python-sync-actions/tests/calls/006-login-auth-headers/login.response
@@ -1,5 +1,5 @@
-{
-	"authorization": {
-		"token": "a1b2c3d435f6"
-	}
+{
+	"authorization": {
+		"token": "a1b2c3d435f6"
+	}
 }
diff --git a/python-sync-actions/tests/calls/006-login-auth-headers/users.requestHeaders b/python-sync-actions/tests/calls/006-login-auth-headers/users.requestHeaders
@@ -1,2 +1,2 @@
-accept: */*
-x-apitoken: a1b2c3d435f6
+accept: */*
+x-apitoken: a1b2c3d435f6
diff --git a/python-sync-actions/tests/calls/006-login-auth-headers/users.response b/python-sync-actions/tests/calls/006-login-auth-headers/users.response
@@ -1,10 +1,10 @@
-[
-    {
-        "id": 123,
-        "name": "John Doe"
-    },
-    {
-        "id": 234,
-        "name": "Jane Doe"
-    }
-]
+[
+    {
+        "id": 123,
+        "name": "John Doe"
+    },
+    {
+        "id": 234,
+        "name": "Jane Doe"
+    }
+]
diff --git a/python-sync-actions/tests/calls/007-page-params-pagenum/config.json b/python-sync-actions/tests/calls/007-page-params-pagenum/config.json
@@ -0,0 +1,70 @@
+{
+  "storage": {},
+  "parameters": {
+        "__SELECTED_JOB": "0",
+
+  "config": {
+    "jobs": [
+      {
+        "__NAME": "orders",
+        "endpoint": "007-page-params-pagenum/orders",
+        "method": "GET",
+        "dataType": "",
+        "dataField": "nested.orders",
+        "scroller": "pagenumber"
+      }
+    ],
+    "test": "test-value",
+    "concat": {
+      "function": "concat",
+      "args": [
+        "hello",
+        "-world"
+      ]
+    },
+    "debug": false,
+    "outputBucket": "in.c-",
+    "incrementalOutput": false,
+    "key": "asdfadsfadsf",
+    "__AUTH_METHOD": "basic",
+    "username": "name",
+    "#password": "pass"
+  },
+  "api": {
+    "baseUrl": "http://mock-server:80/",
+    "authentication": {
+      "type": "basic"
+    },
+    "pagination": {
+      "method": "multiple",
+      "scrollers": {
+        "pagenumber": {
+          "method": "pagenum",
+          "limit": 100,
+          "limitParam": "limit",
+          "pageParam": "page",
+          "firstPageParams": true,
+          "firstPage": 1
+        }
+      }
+    }
+  },
+  "http": {
+    "maxRetries": 10,
+    "codes": [
+      500,
+      502,
+      503,
+      504,
+      408,
+      420,
+      429
+    ]
+  }
+},
+  "action": "test_request",
+  "image_parameters": {
+  },
+  "authorization": {
+  }
+}
diff --git a/python-sync-actions/tests/calls/007-page-params-pagenum/orders.request b/python-sync-actions/tests/calls/007-page-params-pagenum/orders.request
@@ -0,0 +1 @@
+GET /007-page-params-pagenum/orders?page=1&limit=100
diff --git a/python-sync-actions/tests/calls/007-page-params-pagenum/orders.requestHeaders b/python-sync-actions/tests/calls/007-page-params-pagenum/orders.requestHeaders
@@ -0,0 +1,2 @@
+accept: */*
+authorization: Basic bmFtZTpwYXNz
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		GET /007-page-params-pagenum/orders?page=1&limit=100