Merge pull request #181 from keboola/feature/python-sync-actions

Python Sync actions to support UI

.github/workflows/push.yml

@@ -26,30 +26,42 @@ jobs:
       app_image_tag: ${{ steps.tag.outputs.app_image_tag }}
       is_semantic_tag: ${{ steps.tag.outputs.is_semantic_tag }}
     steps:
-      -
-        name: Check out the repo
-        uses: actions/checkout@v2
-      -
-        name: Print Docker version
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Print Docker version
         run: docker -v
-      -
-        name: Docker login
+      - name: Docker login
         if: env.DOCKERHUB_TOKEN
         run: docker login --username "$DOCKERHUB_USER" --password "$DOCKERHUB_TOKEN"
-      -
-        name: Build image
-        run: docker build -t $APP_IMAGE .
-      -
-        name: Set image tag
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - name: Cache Docker layers
+        uses: actions/cache@v2
+        with:
+          path: /tmp/.buildx-cache
+          key: docker-${{ hashFiles('Dockerfile') }}-${{ github.sha }}
+          restore-keys: |
+            docker-${{ hashFiles('Dockerfile') }}
+      - name: Build and push Docker image with cache
+        run: |
+          docker buildx build \
+            --cache-from=type=local,src=/tmp/.buildx-cache \
+            --cache-to=type=local,dest=/tmp/.buildx-cache \
+            --output=type=docker \
+            --tag $APP_IMAGE .
+      - name: Set image tag
         id: tag
         run: |
           TAG="${GITHUB_REF##*/}"
           IS_SEMANTIC_TAG=$(echo "$TAG" | grep -q '^v\?[0-9]\+\.[0-9]\+\.[0-9]\+$' && echo true || echo false)
+          if [ "$IS_SEMANTIC_TAG" = "false" ]; then
+              TAG="${TAG}-${GITHUB_RUN_NUMBER}"
+          fi
           echo "Tag = '$TAG', is semantic tag = '$IS_SEMANTIC_TAG'"
           echo "::set-output name=app_image_tag::$TAG"
           echo "::set-output name=is_semantic_tag::$IS_SEMANTIC_TAG"
-      -
-        name: Push image to ECR
+      - name: Push image to ECR
         uses: keboola/action-push-to-ecr@master
         with:
           vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
@@ -58,17 +70,32 @@ jobs:
           password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
           tag: ${{ steps.tag.outputs.app_image_tag }}
           push_latest: ${{ steps.tag.outputs.is_semantic_tag }}
-          source_image: ${{ env.APP_IMAGE}}
+          source_image: ${{ env.APP_IMAGE }}
+
+  python-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: Build image
+        working-directory: ./python-sync-actions
+        run: docker compose build
+
+      - name: Run python lint
+        working-directory: ./python-sync-actions
+        run: docker compose run dev flake8 . --config=flake8.cfg
+
 
   tests:
-    needs: build
+    needs:
+      - build
+      - python-lint
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Check out the repo
-        uses: actions/checkout@v2
-      -
-        name: Pull image from ECR
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Pull image from ECR
         uses: keboola/action-pull-from-ecr@master
         with:
           vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
@@ -78,19 +105,23 @@ jobs:
           tag: ${{ needs.build.outputs.app_image_tag }}
           target_image: ${{ env.APP_IMAGE}}
           tag_as_latest: true
-      -
-        name: Tests
+      - name: Python Tests
+        run: docker-compose run --rm ci python -m unittest discover /code/python-sync-actions
+      - name: Run Python Functional Tests
+        working-directory: ./python-sync-actions
+        run: docker compose run test-calls
+      - name: PHP Tests
         run: docker-compose run --rm -v $(pwd)/build:/code/build ci
 
   tests-examples:
-    needs: build
+    needs:
+      - build
+      - python-lint
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Check out the repo
-        uses: actions/checkout@v2
-      -
-        name: Pull image from ECR
+      - name: Check out the repo
+        uses: actions/checkout@v4
+      - name: Pull image from ECR
         uses: keboola/action-pull-from-ecr@master
         with:
           vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
@@ -100,19 +131,20 @@ jobs:
           tag: ${{ needs.build.outputs.app_image_tag }}
           target_image: ${{ env.APP_IMAGE}}
           tag_as_latest: true
-      -
-        name: Test examples
+      - name: Test examples
         run: |
           cd doc
           ./run-samples.sh
+          
 
 
   tests-in-kbc:
-    needs: build
+    needs:
+      - build
+      - python-lint
     runs-on: ubuntu-latest
     steps:
-      -
-        name: Run KBC test jobs
+      - name: Run KBC test jobs
         if: env.KBC_STORAGE_TOKEN && env.KBC_TEST_PROJECT_CONFIGS
         uses: keboola/action-run-configs-parallel@master
         with:
@@ -130,13 +162,11 @@ jobs:
     runs-on: ubuntu-latest
     if: startsWith(github.ref, 'refs/tags/') && needs.build.outputs.is_semantic_tag == 'true'
     steps:
-      -
-        name: Set tag in the Deloper Portal
+      - name: Set tag in the Developer Portal
         uses: keboola/action-set-tag-developer-portal@master
         with:
           vendor: ${{ env.KBC_DEVELOPERPORTAL_VENDOR }}
           app_id: ${{ env.KBC_DEVELOPERPORTAL_APP }}
           username: ${{ env.KBC_DEVELOPERPORTAL_USERNAME }}
           password: ${{ env.KBC_DEVELOPERPORTAL_PASSWORD }}
           tag: ${{ needs.build.outputs.app_image_tag }}
-

.gitignore

@@ -1,5 +1,5 @@
 vendor/
-/data/
+/python-sync-actions/data/
 .idea/
 .tmp
 .phpunit.result.cache
@@ -8,3 +8,9 @@ vendor/
 /doc/examples/*/out/tables/*
 /build/logs
 /tests/phpunit/data/*/out
+/data/
+# Python specific
+/venv
+/python-sync-actions/src/test.py
+/python-sync-actions/data
+__pycache__

Dockerfile

@@ -9,19 +9,114 @@ WORKDIR /code/
 
 COPY docker/php.ini /usr/local/etc/php/php.ini
 COPY docker/composer-install.sh /tmp/composer-install.sh
+COPY python-sync-actions/requirements.txt /tmp/requirements.txt
+
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
+        curl \
         git \
         locales \
         unzip \
         ssh \
         netcat \
+        wget \
+        build-essential \
+        libbluetooth-dev \
+        libssl-dev \
+        zlib1g-dev \
+        libncurses5-dev \
+        libncursesw5-dev \
+        libreadline-dev \
+        libsqlite3-dev \
+        libffi-dev \
+        uuid-dev \
+        tk-dev \
+        liblzma-dev \
+        gnupg \
 	&& rm -r /var/lib/apt/lists/* \
 	&& sed -i 's/^# *\(en_US.UTF-8\)/\1/' /etc/locale.gen \
 	&& locale-gen \
 	&& chmod +x /tmp/composer-install.sh \
 	&& /tmp/composer-install.sh
 
+# Set environment variables for Python installation
+ENV GPG_KEY 7169605F62C751356D054A26A821E680E5FA6305
+ENV PYTHON_VERSION 3.12.3
+
+# Download, verify, and install Python from source
+RUN set -eux; \
+    wget -O python.tar.xz "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz"; \
+    wget -O python.tar.xz.asc "https://www.python.org/ftp/python/${PYTHON_VERSION%%[a-z]*}/Python-$PYTHON_VERSION.tar.xz.asc"; \
+    GNUPGHOME="$(mktemp -d)"; export GNUPGHOME; \
+    gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$GPG_KEY"; \
+    gpg --batch --verify python.tar.xz.asc python.tar.xz; \
+    gpgconf --kill all; \
+    rm -rf "$GNUPGHOME" python.tar.xz.asc; \
+    mkdir -p /usr/src/python; \
+    tar -xvf python.tar.xz --strip-components=1 -C /usr/src/python; \
+    rm python.tar.xz; \
+    \
+    cd /usr/src/python; \
+    ./configure \
+        --enable-optimizations \
+        --enable-option-checking=fatal \
+        --with-system-expat \
+        --with-lto \
+        --enable-loadable-sqlite-extensions; \
+    make -j "$(nproc)"; \
+    make install; \
+    \
+    cd /; \
+    rm -rf /usr/src/python; \
+    ldconfig
+
+# Create useful symlinks for Python tools
+RUN set -eux; \
+    for src in idle3 pydoc3 python3 python3-config; do \
+        dst="$(echo "$src" | tr -d 3)"; \
+        [ -s "/usr/local/bin/$src" ]; \
+        [ ! -e "/usr/local/bin/$dst" ]; \
+        ln -svT "$src" "/usr/local/bin/$dst"; \
+    done
+
+# Install pip and configure environment variables
+ENV PYTHON_PIP_VERSION 24.0
+ENV PYTHON_GET_PIP_URL https://github.com/pypa/get-pip/raw/dbf0c85f76fb6e1ab42aa672ffca6f0a675d9ee4/public/get-pip.py
+ENV PYTHON_GET_PIP_SHA256 dfe9fd5c28dc98b5ac17979a953ea550cec37ae1b47a5116007395bfacff2ab9
+
+RUN set -eux; \
+    wget -O get-pip.py "$PYTHON_GET_PIP_URL"; \
+    echo "$PYTHON_GET_PIP_SHA256 *get-pip.py" | sha256sum -c -; \
+    python3 get-pip.py --disable-pip-version-check --no-cache-dir --no-compile "pip==$PYTHON_PIP_VERSION"; \
+    rm -f get-pip.py; \
+    pip --version
+
+RUN pip install -r /tmp/requirements.txt
+
+# Install Node.js and set up symlinks
+RUN set -eux; \
+    NODE_VERSION="$(curl -fsSL https://nodejs.org/dist/latest/SHASUMS256.txt | head -n1 | awk '{ print $2 }' | awk -F - '{ print $2 }')" \
+    ARCH= && dpkgArch="$(dpkg --print-architecture)"; \
+    case "${dpkgArch##*-}" in \
+        amd64) ARCH='x64';; \
+        arm64) ARCH='arm64';; \
+        *) echo "unsupported architecture"; exit 1 ;; \
+    esac; \
+    for key in $(curl -sL https://raw.githubusercontent.com/nodejs/docker-node/HEAD/keys/node.keys); do \
+        gpg --batch --keyserver hkps://keys.openpgp.org --recv-keys "$key" || \
+        gpg --batch --keyserver keyserver.ubuntu.com --recv-keys "$key"; \
+    done; \
+    curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/node-$NODE_VERSION-linux-$ARCH.tar.xz"; \
+    curl -fsSLO --compressed "https://nodejs.org/dist/$NODE_VERSION/SHASUMS256.txt.asc"; \
+    gpg --batch --decrypt --output SHASUMS256.txt SHASUMS256.txt.asc; \
+    grep " node-$NODE_VERSION-linux-$ARCH.tar.xz\$" SHASUMS256.txt | sha256sum -c -; \
+    tar -xJf "node-$NODE_VERSION-linux-$ARCH.tar.xz" -C /usr/local --strip-components=1 --no-same-owner; \
+    rm "node-$NODE_VERSION-linux-$ARCH.tar.xz" SHASUMS256.txt.asc SHASUMS256.txt; \
+    ln -s /usr/local/bin/node /usr/local/bin/nodejs
+
+# Install curlconverter using npm
+RUN npm install --global curlconverter
+
 ENV LANGUAGE=en_US.UTF-8
 ENV LANG=en_US.UTF-8
 ENV LC_ALL=en_US.UTF-8

composer.json

@@ -20,7 +20,7 @@
         "symfony/config": "^5.2",
         "symfony/console": "^5.2",
         "symfony/finder": "^5.2",
-        "symfony/process": "^5.2",
+        "symfony/process": "^5.4",
         "symfony/validator": "^5.2"
     },
     "require-dev": {

python-sync-actions/Dockerfile

@@ -0,0 +1,25 @@
+FROM nikolaik/python-nodejs:python3.12-nodejs18
+ENV PYTHONIOENCODING utf-8
+
+COPY /src /code/src/
+COPY /tests /code/tests/
+COPY requirements.txt /code/requirements.txt
+COPY flake8.cfg /code/flake8.cfg
+
+# install gcc to be able to build packages - e.g. required by regex, dateparser, also required for pandas
+RUN apt-get update && apt-get install -y build-essential curl
+
+# Install curlconverter using npm
+RUN npm install --global curlconverter
+
+
+RUN pip install flake8
+
+RUN pip install -r /code/requirements.txt
+
+
+
+WORKDIR /code/
+
+
+CMD ["python", "-u", "/code/src/component.py"]