diff --git a/python-sync-actions/requirements.txt b/python-sync-actions/requirements.txt index 779a926..78ccd7b 100644 --- a/python-sync-actions/requirements.txt +++ b/python-sync-actions/requirements.txt @@ -6,4 +6,5 @@ keboola.json-to-csv mock freezegun nested-lookup -python-dateutil \ No newline at end of file +python-dateutil +dlt==0.4.12 \ No newline at end of file diff --git a/python-sync-actions/src/component.py b/python-sync-actions/src/component.py index 6b17164..a93f719 100644 --- a/python-sync-actions/src/component.py +++ b/python-sync-actions/src/component.py @@ -19,6 +19,7 @@ from http_generic.auth import AuthMethodBuilder, AuthBuilderError from http_generic.client import GenericHttpClient, HttpClientError from placeholders_utils import PlaceholdersUtils +from dlt.sources.helpers.rest_client import paginators MAX_CHILD_CALLS = 20 @@ -249,6 +250,37 @@ def _parse_data(self, data, path) -> list: return result + def _get_paginator(self, job): + """ + Get paginator object for the given job. + + Args: + job: The job for which to get the paginator. + + Returns: + The paginator object for the job. + """ + + paginator = {} + + if job.request_parameters.scroller: + paginator_params = job.api.pagination.get("scrollers").get(job.request_parameters.scroller) + + if paginator_params.get("method") == "offset": + # paginator = paginators.OffsetPaginator(limit=paginator_params.get("limit"), + # offset=paginator_params.get("offset"), + # offset_param=paginator_params.get("offsetParam"), + # limit_param=paginator_params.get("limitParam") + # ) + paginator[paginator_params.get("offsetParam")] = paginator_params.get("offset", 0) + paginator[paginator_params.get("limitParam")] = paginator_params.get("limit") + + elif paginator_params.get("method") == "pagenum": + if paginator_params.get("firstPageParams"): + paginator[paginator_params.get("pageParam")] = paginator_params.get("page", 1) + + return paginator + def make_call(self) -> tuple[list, any, str, str]: """ Make call to the API @@ -294,6 +326,10 @@ def recursive_call(parent_result, config_index=0): ssl_verify = api_cfg.ssl_verification timeout = api_cfg.timeout # additional_params = self._build_request_parameters(additional_params_cfg) + + paginator = self._get_paginator(job) + query_parameters.update(paginator) + request_parameters = {'params': query_parameters, 'headers': new_headers, 'verify': ssl_verify,