Merge pull request #268 from sighupio/feat/product540/improve-support…

…-for-arm64 improve support fo sync linux/arm64 images
sighupio · Nov 15, 2024 · bbfbb74 · bbfbb74
2 parents 82facd7 + 0bfce7d
commit bbfbb74
Show file tree

Hide file tree

Showing 33 changed files with 642 additions and 947 deletions.
diff --git a/.github/workflows/cve-scan-and-patching.yml b/.github/workflows/cve-scan-and-patching.yml
@@ -11,6 +11,15 @@ on:
       - '!CVEs/MAINTENANCE.md'
 
 jobs:
+  set_docker_config_env_var:
+    runs-on: ubuntu-latest
+    steps:
+      - name: set DOCKER_CONFIG
+        id: set_docker_config_env
+        run: |
+          echo "DOCKER_CONFIG=/tmp/${GITHUB_RUN_ID}/.docker" >> $GITHUB_OUTPUT
+    outputs:
+      docker_config: ${{ steps.set_docker_config_env.outputs.DOCKER_CONFIG }}
   fetch_kfd_versions:
     runs-on: ubuntu-latest
     steps:
@@ -26,9 +35,56 @@ jobs:
       kfd_versions_json: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }}
       kfd_versions_list: ${{ steps.set_output.outputs.KFD_VERSIONS_JSON }}
       today_date: ${{ steps.set_output.outputs.TODAY_DATE }}
+  install_tools:
+    runs-on: ubuntu-latest
+    needs:
+      - fetch_kfd_versions
+    steps:
+      - uses: actions/checkout@master
+      - name: cache tool used by jobs
+        id: cache_tools
+        uses: actions/cache@v4
+        with:
+          path: |
+            /tmp/.cache/trivy
+            /usr/local/bin/trivy
+            /usr/local/bin/furyctl
+            /usr/local/bin/buildctl
+            /usr/local/bin/copa
+          key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }}
+      - name: install tools required by jobs
+        if: steps.cache_tools.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get install wget apt-transport-https gnupg
+          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
+          echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
+          sudo apt-get update
+          sudo apt-get install trivy
+          trivy --version
+          cp /usr/bin/trivy /usr/local/bin/trivy
+          
+          wget https://github.com/sighupio/furyctl/releases/latest/download/furyctl-linux-amd64.tar.gz
+          tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/
+          furyctl version
+          
+          wget https://github.com/moby/buildkit/releases/download/v0.16.0/buildkit-v0.16.0.linux-amd64.tar.gz
+          tar -xzvf buildkit-v0.16.0.linux-amd64.tar.gz -C /usr/local/bin/  --strip-components=1
+          buildctl --version
+
+          wget https://github.com/project-copacetic/copacetic/releases/download/v0.9.0/copa_0.9.0_linux_amd64.tar.gz
+          tar -xzvf copa_0.9.0_linux_amd64.tar.gz
+          chmod +x copa
+          sudo mv copa /usr/local/bin/
+          copa --version
+          
+          cd CVEs
+          make trivy-download-db
   scan_pre_patch:
     runs-on: ubuntu-latest
-    needs: fetch_kfd_versions
+    needs:
+      - set_docker_config_env_var
+      - fetch_kfd_versions
+      - install_tools
     continue-on-error: true
     strategy:
 #      max-parallel: 3
@@ -43,24 +99,33 @@ jobs:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install furyctl, trivy
-        run: |
-          sudo apt-get install wget apt-transport-https gnupg
-          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
-          echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
-          sudo apt-get update
-          sudo apt-get install trivy
-          trivy --version
-
-          wget https://github.com/sighupio/furyctl/releases/latest/download/furyctl-linux-amd64.tar.gz
-          tar -xzvf furyctl-linux-amd64.tar.gz -C /usr/local/bin/
-          furyctl version
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
+      - name: cache tools
+        uses: actions/cache@v4
+        with:
+          fail-on-cache-miss: true
+          path: |
+            /tmp/.cache/trivy
+            /usr/local/bin/trivy
+            /usr/local/bin/furyctl
+            /usr/local/bin/buildctl
+            /usr/local/bin/copa
+          key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }}
+      - name: cache KFD vendors built manifest for KFD ${{ matrix.kfd_version }}
+        uses: actions/cache@v4
+        with:
+         path: |
+          CVEs/${{ matrix.kfd_version }}/built.yaml
+          CVEs/${{ matrix.kfd_version }}/images.txt
+         key: kfd-${{ matrix.kfd_version }}-vendors-built-manifests
       - name: Execute CVEs scan for KFD ${{ matrix.kfd_version }}
         id: scan_pre_patch
         run: |
           cd CVEs
-          make trivy-download-db
           make scan-pre-patch KFD_VERSIONS="${{ matrix.kfd_version }}"
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
       - name: publish CVE scan pre patch output files for KFD ${{ matrix.kfd_version }}
         uses: actions/upload-artifact@v4
         with:
@@ -89,6 +154,9 @@ jobs:
   patch:
     runs-on: ubuntu-latest
     needs:
+      - install_tools
+      - fetch_kfd_versions
+      - set_docker_config_env_var
       - fetch_kfd_images_to_patch
     continue-on-error: true
     strategy:
@@ -103,51 +171,58 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
       - name: Login to SIGHUP new Registry
         uses: docker/login-action@v3
         with:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install buildkit, trivy and copa
-        run: |
-          sudo apt-get install wget apt-transport-https gnupg
-          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
-          echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
-          sudo apt-get update
-          sudo apt-get install trivy
-          trivy --version
-
-          wget https://github.com/moby/buildkit/releases/download/v0.16.0/buildkit-v0.16.0.linux-amd64.tar.gz
-          tar -xzvf buildkit-v0.16.0.linux-amd64.tar.gz -C /usr/local/bin/  --strip-components=1
-          buildctl --version
-
-          wget https://github.com/project-copacetic/copacetic/releases/download/v0.8.0/copa_0.8.0_linux_amd64.tar.gz
-          tar -xzvf copa_0.8.0_linux_amd64.tar.gz
-          chmod +x copa
-          sudo mv copa /usr/local/bin/
-          copa --version
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
+      - # Add support for more platforms with QEMU (optional)
+        # https://github.com/docker/setup-qemu-action
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: amd64,arm64
+      - name: cache tools
+        uses: actions/cache@v4
+        with:
+          fail-on-cache-miss: true
+          path: |
+            /tmp/.cache/trivy
+            /usr/local/bin/trivy
+            /usr/local/bin/furyctl
+            /usr/local/bin/buildctl
+            /usr/local/bin/copa
+          key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }}
       - name: Execute CVEs patching
         id: patching
         run: |
           IMAGE_TO_PATCH=${{ matrix.image_to_patch }}
           IMAGE_TO_PATCH_NORMALIZED=${IMAGE_TO_PATCH//[:\/]/_}
           cd CVEs
           mkdir -p reports
-          make trivy-download-db
-          DOCKER_CONFIG="${DOCKER_CONFIG}" make patch DRY_RUN=0 IMAGE_TO_PATCH="${IMAGE_TO_PATCH}" PATCH_REPORT_OUTPUT_FILE="reports/${IMAGE_TO_PATCH_NORMALIZED}.patched.md"
-          
+          make patch DRY_RUN=0 \
+            IMAGE_TO_PATCH="${IMAGE_TO_PATCH}" \
+            PATCH_REPORT_OUTPUT_FILE="reports/${IMAGE_TO_PATCH_NORMALIZED}.patched.md"
           echo "IMAGE_TO_PATCH_NORMALIZED=${IMAGE_TO_PATCH_NORMALIZED}" >> "$GITHUB_OUTPUT"
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
       - name: publish CVE patching report for ${{ matrix.image_to_patch }}
         uses: actions/upload-artifact@v4
         with:
           name: patch-report-${{ steps.patching.outputs.IMAGE_TO_PATCH_NORMALIZED }}.md
           if-no-files-found: ignore
           path: |
-            CVEs/reports/*.patched.md
+            CVEs/reports/*.patched.*.md
   scan_post_patch:
     runs-on: ubuntu-latest
     needs:
+      - install_tools
+      - set_docker_config_env_var
       - fetch_kfd_versions
       - patch
     continue-on-error: true
@@ -163,25 +238,30 @@ jobs:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
           password: ${{ secrets.SIGHUP_REGISTRY_PASSWORD }}
-      - name: Install trivy
-        run: |
-          sudo apt-get install wget apt-transport-https gnupg
-          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | gpg --dearmor | sudo tee /usr/share/keyrings/trivy.gpg > /dev/null
-          echo "deb [signed-by=/usr/share/keyrings/trivy.gpg] https://aquasecurity.github.io/trivy-repo/deb generic main" | sudo tee -a /etc/apt/sources.list.d/trivy.list
-          sudo apt-get update
-          sudo apt-get install trivy
-          trivy --version
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
       - name: download CVE scan pre patch output files
         uses: actions/download-artifact@v4
         with:
           path: CVEs
+      - name: cache tools
+        uses: actions/cache@v4
+        with:
+          fail-on-cache-miss: true
+          path: |
+            /tmp/.cache/trivy
+            /usr/local/bin/trivy
+            /usr/local/bin/furyctl
+            /usr/local/bin/buildctl
+            /usr/local/bin/copa
+          key: cve-scan-patching-tools-${{ needs.fetch_kfd_versions.outputs.today_date }}
       - name: Execute CVEs scan post patch for KFD ${{ matrix.kfd_version }}
         id: scan_post_patch
         run: |
           cd CVEs
-          find .
-          make trivy-download-db
           make scan-post-patch KFD_VERSIONS="${{ matrix.kfd_version }}"
+        env:
+          DOCKER_CONFIG: ${{ needs.set_docker_config_env_var.outputs.docker_config}}
       - name: publish CVE scan post patch output files for KFD ${{ matrix.kfd_version }}
         uses: actions/upload-artifact@v4
         with:
@@ -207,7 +287,7 @@ jobs:
         with:
           name: cve-patch-reports-by-image
           path: |
-            reports/*.patched.md
+            reports/*.patched.*.md
       - uses: geekyeggo/delete-artifact@v5
         with:
           name: |

diff --git a/.github/workflows/dry.yml b/.github/workflows/dry.yml
@@ -2,19 +2,43 @@ name: "Dry run"
 
 on:
   push:
+    paths:
+      - '.github/workflows/dry.yml'
+      - '.github/workflows/sync.yml'
+      - 'modules/**'
+      - '!README.md'
+      - '!single_sync.sh'
+      - '!single_sync_v2.sh'
+      - 'single_sync_v3.sh'
 
 jobs:
+  fetch_modules_to_sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@master
+      - name: Set output
+        id: set_output
+        run: |
+          MODULES_TO_SYNC_JSON=$(find modules -type d -mindepth 1 -maxdepth 1 | cut -d/ -f2 | sort | jq -R | jq -cs . )
+          echo "MODULES_TO_SYNC_JSON=${MODULES_TO_SYNC_JSON}" >> $GITHUB_OUTPUT
+    outputs:
+      modules_to_sync: ${{ steps.set_output.outputs.MODULES_TO_SYNC_JSON }}
   sync:
     runs-on: ubuntu-latest
+    needs: fetch_modules_to_sync
+    strategy:
+      fail-fast: false
+      matrix:
+        module: ${{ fromJson(needs.fetch_modules_to_sync.outputs.modules_to_sync) }}
     steps:
       - uses: actions/checkout@master
       - name: Login to Docker Hub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
       - name: Login to SIGHUP new Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: registry.sighup.io
           username: ${{ secrets.SIGHUP_REGISTRY_USERNAME }}
@@ -28,4 +52,4 @@ jobs:
         run: |
           yq --version
           docker run --rm quay.io/skopeo/stable:v1.13 --version
-          ./sync.sh --dry-run
+          ./single_sync_v3.sh modules/${{ matrix.module }}/images.yml true
diff --git a/.github/workflows/sync-auth.yml b/.github/workflows/sync-auth.yml
diff --git a/.github/workflows/sync-aws.yml b/.github/workflows/sync-aws.yml