mTLS support for Python sync actions (CFT-3328)

.gitignore

@@ -16,3 +16,4 @@ vendor/
 __pycache__/
 docker/keys/*
 !docker/keys/genkeys.sh
+!docker/keys/keys-to-config-json.py

.python-version

@@ -0,0 +1 @@
+3.12

README.md

@@ -29,7 +29,7 @@ Example:
 
 ```
 https://yourDomain.zendesk.com/api/v2/
-```   
+```
 
 -- OR --
 
@@ -148,7 +148,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
 - **authentication.type**: `query`
 - **authentication.query.apiKey**: `{"attr": "apiKey"}`
     - this will look for the *apiKey* query parameter value in the config attribute named *apiKey*
-- **authentication.query.sig**: 
+- **authentication.query.sig**:
     ```
     {
         "function": "md5",
@@ -169,7 +169,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
             }
         ]
     }
-    ```   
+    ```
     - this will generate a *sig* parameter value from MD5 of merged configuration table attributes *apiKey* and *secret*, followed by current *time()* at the time of the request (time() being the PHP function)
     - Allowed functions are listed below in the *User functions* section
     - If you're using any config parameter by using `"attr": "parameterName"`, it has to be identical string to the one in the actual config, including eventual `#` if KBC Docker's encryption is used.
@@ -468,7 +468,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
     - sets which query parameter should contain the limit value (default to `limit`)
 - **pagination.offsetParam**(optional)
     - sets which query parameter should contain the offset value (default to `offset`)
-        
+
         ```
             "api": {
                 "pagination": {
@@ -478,7 +478,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                     "offsetParam": "offset"
                 }
             }
-        ``` 
+        ```
 
 - **pagination.firstPageParams**(optional)
     - Whether or not include limit and offset params in the first request (default to `true`)
@@ -769,7 +769,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                 }
             ]
         }
-        ```    
+        ```
     - **dataType**: Type of data returned by the endpoint. It also describes a table name, where the results will be stored
     - **dataField**: Allows to override which field of the response will be exported.
         - If there's multiple arrays in the response "root" the extractor may not know which array to export and fail
@@ -842,7 +842,7 @@ Moved to [new docs](https://developers.keboola.com/extend/generic-extractor/conf
                 "data": {"object": "can't really parse this!"}
             }
         ]
-        ```  
+        ```
 
         - To be able to work with such response, set `"responseFilter": "data"` - it should be a path within each object of the response array, **not** including the key of the response array
         - To filter values within nested arrays, use `"responseFilter": "data.array[].key"`
@@ -1218,25 +1218,25 @@ Best way to create and test new configurations is run extractor in docker contai
 # Running tests:
 ```
 docker compose run --rm tests
-``` 
+```
 
 or (with local source code and vendor copy)
 
 ```
 docker compose run --rm tests-local
-``` 
+```
 
 # mTLS
-1. `cd docker/keys` and then run `./genkeys.sh`
-2. ```
+1. Run `cd docker/keys` and then `./genkeys.sh`. The script generates CA, server and client certificates
+    and also a config.json file with the following structure to be pasted into your own config.json:
+    ```
     "api": {
         "baseUrl": "https://server.local/",
-        "caCertificate": "conent of file rootCA.crt with \n at the end",
-        "#clientCertificate": "conent of file client.crt with \n at the end",
-        "#clientKey": "conent of file client.key with \n at the end"
+        "caCertificate": "-- rootCA.crt --",
+        "#clientCertificate": "-- client.crt bundled with client.key --",
     }
-   ```
-3. restart nginx
+    ```
+1. Restart nginx
 
 ## License
 

docker-compose.yml

@@ -1,4 +1,3 @@
-version: '3'
 services:
   app: &app
     build: .
@@ -40,7 +39,7 @@ services:
     ports:
       - "443:443"
     volumes:
-      - "./docker/nginx/default.conf:/etc/nginx/conf.d/default.conf"
-      - "./docker/keys/server.crt:/etc/nginx/server.crt"
-      - "./docker/keys/server.key:/etc/nginx/server.key"
-      - "./docker/keys/rootCA.crt:/etc/nginx/ca.crt"
+      - ./docker/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ./docker/keys/server.crt:/etc/nginx/server.crt
+      - ./docker/keys/server.key:/etc/nginx/server.key
+      - ./docker/keys/rootCA.crt:/etc/nginx/ca.crt

docker/keys/genkeys.sh

@@ -1,12 +1,17 @@
-cd keys
 echo "creating rootCA"
 openssl genrsa -out rootCA.key 4096
 openssl req -x509 -new -nodes -key rootCA.key -subj "/C=CZ/ST=CZ/O=authority" -days 1024 -out rootCA.crt
+
 echo "creating server keys"
 openssl genrsa -out server.key 2048
-openssl req -new  -key server.key -subj "/C=CZ/ST=CZ/O=mytest/CN=server.local" -out server.csr # CN = server.local name of service
-openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500
+# SAN is required as it is the main place where modern clients check host name (in fact CN can be ignored -- and is by e.g. chrome or requests)
+openssl req -new  -key server.key -subj "/C=CZ/ST=CZ/O=mytest/CN=server.local" -addext "subjectAltName=DNS:server.local" -out server.csr
+# Extensions such as SAN are not coppied by default from CSR when creating the certificate, -copy_extensions is required (semi-recent addition to OpenSSL)
+openssl x509 -req -in server.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out server.crt -days 500 -copy_extensions copy
+
 echo "creating client keys"
 openssl genrsa -out client.key 2048
-openssl req -new -key client.key -subj "/C=CZ/ST=CZ/O=mytest/CN=dev" -out client.csr # CN = dev name of service
-openssl x509 -req -in client.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out client.crt -days 500
+openssl req -new -key client.key -subj "/C=CZ/ST=CZ/O=mytest/CN=client.local" -addext "subjectAltName=DNS:client.local" -out client.csr
+openssl x509 -req -in client.csr -CA rootCA.crt -CAkey rootCA.key -CAcreateserial -out client.crt -days 500 -copy_extensions copy
+
+python3 keys-to-config-json.py

docker/keys/keys-to-config-json.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+import json
+
+
+with open("rootCA.crt") as f:
+    ca_cert = f.read()
+
+with open("client.crt") as f:
+    client_cert = f.read()
+
+with open("client.key") as f:
+    client_key = f.read()
+
+with open("config.json", "w") as f:
+    json.dump(
+        {
+            "api": {
+                "baseUrl": "https://server.local/",
+                "caCertificate": ca_cert,
+                "#clientCertificate": client_cert + client_key,
+            }
+        },
+        f,
+        indent=4,
+    )

python-sync-actions/Dockerfile

@@ -1,25 +1,19 @@
 FROM nikolaik/python-nodejs:python3.12-nodejs18
-ENV PYTHONIOENCODING utf-8
 
-COPY /src /code/src/
-COPY /tests /code/tests/
-COPY requirements.txt /code/requirements.txt
-COPY flake8.cfg /code/flake8.cfg
-
-# install gcc to be able to build packages - e.g. required by regex, dateparser, also required for pandas
-RUN apt-get update && apt-get install -y build-essential curl
+RUN apt-get update && apt-get install -y curl
 
-# Install curlconverter using npm
 RUN npm install --global curlconverter
 
-
 RUN pip install flake8
 
+COPY requirements.txt /code/requirements.txt
 RUN pip install -r /code/requirements.txt
 
+COPY flake8.cfg /code/flake8.cfg
 
+COPY /src /code/src/
+COPY /tests /code/tests/
 
 WORKDIR /code/
 
-
 CMD ["python", "-u", "/code/src/component.py"]

python-sync-actions/docker-compose.yml

@@ -1,11 +1,11 @@
-version: "2"
 services:
   # for development purposes
   dev:
     build: .
     volumes:
         - ./:/code
         - ./data:/data
+        - ../docker/keys:/code/keys
     environment:
       - KBC_DATADIR=./data
   test:
@@ -35,8 +35,20 @@ services:
     tty: true
     stdin_open: true
     ports:
-      - "8888:80"
+      - 8888:80
     volumes:
       - ./tests/calls:/examples/
     environment:
-      - KBC_EXAMPLES_DIR=/examples/
+      - KBC_EXAMPLES_DIR=/examples/
+
+  # i was about to create a common network with the generic-extractor compose file
+  # to avoid duplication, but that would create an unnecessary dependency
+  server.local:
+    image: nginx:alpine
+    ports:
+      - "443:443"
+    volumes:
+      - ../docker/nginx/default.conf:/etc/nginx/conf.d/default.conf
+      - ../docker/keys/server.crt:/etc/nginx/server.crt
+      - ../docker/keys/server.key:/etc/nginx/server.key
+      - ../docker/keys/rootCA.crt:/etc/nginx/ca.crt

python-sync-actions/requirements.in

@@ -0,0 +1,9 @@
+keboola.component
+dataconf
+keboola.http-client
+keboola.utils
+keboola.json-to-csv==0.0.12
+mock==5.1.0
+freezegun==1.5.1
+nested-lookup==0.2.25
+python-dateutil==2.9.0.post0

python-sync-actions/requirements.txt

@@ -1,9 +1,68 @@
-keboola.component==1.6.8
+# This file was autogenerated by uv via the following command:
+#    uv pip compile requirements.txt --universal --output-file requirements.all
+certifi==2025.1.31
+    # via requests
+charset-normalizer==3.4.1
+    # via requests
 dataconf==3.3.0
-keboola.http-client==1.0.1
-keboola.utils==1.1.0
-keboola.json-to-csv==0.0.12
-mock==5.1.0
+    # via -r requirements.txt
+dateparser==1.2.0
+    # via keboola-utils
+deprecated==1.2.18
+    # via keboola-component
 freezegun==1.5.1
+    # via -r requirements.txt
+idna==3.10
+    # via requests
+isodate==0.6.1
+    # via dataconf
+keboola-component==1.6.8
+    # via -r requirements.txt
+keboola-http-client==1.0.1
+    # via -r requirements.txt
+keboola-json-to-csv==0.0.12
+    # via -r requirements.txt
+keboola-utils==1.1.0
+    # via -r requirements.txt
+mock==5.1.0
+    # via -r requirements.txt
 nested-lookup==0.2.25
+    # via -r requirements.txt
+pygelf==0.4.2
+    # via keboola-component
+pyhocon==0.3.61
+    # via dataconf
+pyparsing==3.2.1
+    # via
+    #   dataconf
+    #   pyhocon
 python-dateutil==2.9.0.post0
+    # via
+    #   -r requirements.txt
+    #   dataconf
+    #   dateparser
+    #   freezegun
+pytz==2025.1
+    # via
+    #   dateparser
+    #   keboola-component
+    #   keboola-utils
+pyyaml==6.0.2
+    # via dataconf
+regex==2024.11.6
+    # via dateparser
+requests==2.32.3
+    # via keboola-http-client
+six==1.17.0
+    # via
+    #   isodate
+    #   nested-lookup
+    #   python-dateutil
+tzdata==2025.1 ; sys_platform == 'win32'
+    # via tzlocal
+tzlocal==5.2
+    # via dateparser
+urllib3==2.3.0
+    # via requests
+wrapt==1.17.2
+    # via deprecated