From 826cfd8d064d41b830e3b209c7af70ff01ab9390 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 18:44:08 -0800 Subject: [PATCH 01/15] ci: add support for arm64 Linux --- .github/workflows/CI.yml | 17 +++++++++++++++-- docker/alpine.dockerfile | 16 ++++++++++++++-- docker/debian.dockerfile | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 66 insertions(+), 4 deletions(-) create mode 100644 docker/debian.dockerfile diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 5b1b3262..56905b47 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -40,12 +40,24 @@ jobs: node_arch: arm64 cpp_arch: amd64_arm64 - # Alpine + # Musl Alpine - os: ubuntu-22.04 dockerfile: docker/alpine.dockerfile node_arch: x64 cpp_arch: x64 + # Debian Arm + - os: macos-14 + node_arch: arm64 + cpp_arch: amd64_arm64 + dockerfile: docker/alpine.dockerfile + + # Musl Alpine Arm + - os: macos-14 + node_arch: arm64 + cpp_arch: amd64_arm64 + dockerfile: docker/debian.dockerfile + env: npm_config_arch: ${{ matrix.node_arch }} npm_config_target_arch: ${{ matrix.node_arch }} @@ -137,7 +149,8 @@ jobs: run: | docker build -t zeromq -f ${{ matrix.dockerfile }} . docker create --name zeromq-temp zeromq - docker cp zeromq-temp:/app/build ./build + mkdir -p ./build + docker cp zeromq-temp:/app/build ./ docker rm -f zeromq-temp - name: Upload build diff --git a/docker/alpine.dockerfile b/docker/alpine.dockerfile index 21a99e5c..407e7a5e 100644 --- a/docker/alpine.dockerfile +++ b/docker/alpine.dockerfile @@ -1,9 +1,11 @@ -FROM node:18-alpine +FROM node:18-alpine AS builder WORKDIR /app COPY ./ ./ ENV VCPKG_FORCE_SYSTEM_BINARIES=1 -RUN apk add --no-cache \ +RUN \ + # system dependencies + apk add --no-cache \ bash \ build-base \ curl \ @@ -23,6 +25,16 @@ RUN apk add --no-cache \ autoconf \ libtool && \ cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja && \ + # build npm i -g pnpm && \ pnpm install && \ pnpm run build + +FROM node:18-alpine +WORKDIR /app + +COPY ./ ./ +COPY --from=builder /app/build ./build +COPY --from=builder /app/node_modules ./node_modules + + diff --git a/docker/debian.dockerfile b/docker/debian.dockerfile new file mode 100644 index 00000000..c351248b --- /dev/null +++ b/docker/debian.dockerfile @@ -0,0 +1,37 @@ +FROM node:18 AS builder + +WORKDIR /app +COPY ./ ./ +ENV VCPKG_FORCE_SYSTEM_BINARIES=1 +RUN \ + # system dependencies + apt-get update -y && \ + apt-get install --no-install-recommends -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool && \ + # build + npm i -g pnpm && \ + pnpm install && \ + pnpm run build + +FROM node:18 +WORKDIR /app + +COPY ./ ./ +COPY --from=builder /app/build ./build +COPY --from=builder /app/node_modules ./node_modules From f7e5d948febe402650a96f95cd51eb6ec4f68773 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:02:31 -0800 Subject: [PATCH 02/15] ci: build Linux arm64 on Ubuntu --- .github/workflows/CI.yml | 95 +++++++++++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 17 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 56905b47..a889e1d2 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,6 +9,7 @@ jobs: Build: if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} runs-on: ${{ matrix.os }} + name: ${{ matrix.os }}-${{ matrix.node_arch }}-${{ matrix.dockerfile }}-${{ matrix.distro }} strategy: fail-fast: false matrix: @@ -47,16 +48,56 @@ jobs: cpp_arch: x64 # Debian Arm - - os: macos-14 + - os: ubuntu-22.04 node_arch: arm64 cpp_arch: amd64_arm64 - dockerfile: docker/alpine.dockerfile + distro: node:18 + install: | + apt-get update -y && \ + apt-get install --no-install-recommends -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool # Musl Alpine Arm - - os: macos-14 + - os: ubuntu-22.04 node_arch: arm64 cpp_arch: amd64_arm64 - dockerfile: docker/debian.dockerfile + install: | + apk add --no-cache \ + bash \ + build-base \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkgconfig \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja \ + musl-dev \ + automake \ + autoconf \ + libtool && \ + cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja env: npm_config_arch: ${{ matrix.node_arch }} @@ -85,7 +126,7 @@ jobs: shell: bash - name: Setup Cpp - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} uses: aminya/setup-cpp@v1 with: vcvarsall: ${{ contains(matrix.os, 'windows') }} @@ -101,34 +142,34 @@ jobs: brew install gnutls autoconf automake libtool - uses: pnpm/action-setup@v4 - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} with: version: 9 - name: Install Node 20 - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} uses: actions/setup-node@v4 with: node-version: 20 architecture: ${{ env.setup_node_arch }} - name: Install and Build Native - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} run: pnpm install - name: Build JavaScript - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} run: pnpm run build.js - name: Install Node 10 - if: ${{ !matrix.dockerfile && matrix.os != 'macos-14' }} + if: ${{ !matrix.dockerfile && !matrix.install && matrix.os != 'macos-14' }} uses: actions/setup-node@v4 with: node-version: 10 architecture: ${{ env.setup_node_arch }} - + - name: Build Native - if: ${{ !matrix.dockerfile && matrix.node_arch != 'ia32' }} + if: ${{ !matrix.dockerfile && !matrix.install && matrix.node_arch != 'ia32' }} run: npm run build.native - name: Build Native Windows 32 @@ -138,7 +179,7 @@ jobs: windows-x86 - name: Use Node 20 - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} uses: actions/setup-node@v4 with: node-version: 20 @@ -153,6 +194,26 @@ jobs: docker cp zeromq-temp:/app/build ./ docker rm -f zeromq-temp + - name: Build Linux Arm64 + if: ${{ matrix.install }} + uses: uraimo/run-on-arch-action@v2 + with: + arch: aarch64 + distro: ${{ matrix.distro }} + githubToken: ${{ github.token }} + setup: | + mkdir -p "${PWD}/build" + dockerRunArgs: | + --volume "${PWD}/build:/build" + env: | + VCPKG_FORCE_SYSTEM_BINARIES: 1 + install: | + ${{ matrix.install }} + run: | + npm i -g pnpm && \ + pnpm install && \ + pnpm run build + - name: Upload build uses: actions/upload-artifact@v4 with: @@ -161,11 +222,11 @@ jobs: overwrite: true - name: Lint - if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile }}" + if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" run: pnpm run lint-test - name: Test - if: ${{ !matrix.dockerfile }} + if: ${{ !matrix.dockerfile && !matrix.install }} uses: nick-fields/retry@v3 with: timeout_minutes: 5 @@ -176,7 +237,7 @@ jobs: rm -rf ./tmp && mkdir -p ./tmp - name: Test Electron Windows/MacOS - if: "${{ !contains(matrix.os, 'ubuntu') && !matrix.dockerfile }}" + if: "${{ !contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" uses: nick-fields/retry@v3 with: timeout_minutes: 5 @@ -186,7 +247,7 @@ jobs: continue-on-error: true - name: Test Electron Linux - if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile }}" + if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" uses: nick-fields/retry@v3 with: timeout_minutes: 5 From a800adc73a40b722694385efebc3bfb068562c58 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:14:29 -0800 Subject: [PATCH 03/15] ci: fix installation of distros for Linux arm64 --- .github/workflows/CI.yml | 159 ++++++++++++++++++++++++--------------- 1 file changed, 99 insertions(+), 60 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a889e1d2..a82a9115 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -9,6 +9,7 @@ jobs: Build: if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }} runs-on: ${{ matrix.os }} + # prettier-ignore name: ${{ matrix.os }}-${{ matrix.node_arch }}-${{ matrix.dockerfile }}-${{ matrix.distro }} strategy: fail-fast: false @@ -22,11 +23,16 @@ jobs: - x64 dockerfile: - "" + distro: + - "" + native: + - true include: - os: windows-2019 node_arch: ia32 cpp_arch: amd64_x86 + native: true # - os: windows-2022 # node_arch: x64 @@ -36,68 +42,33 @@ jobs: - os: macos-13 node_arch: x64 cpp_arch: x64 + native: true - os: macos-14 node_arch: arm64 cpp_arch: amd64_arm64 + native: true # Musl Alpine - os: ubuntu-22.04 dockerfile: docker/alpine.dockerfile node_arch: x64 cpp_arch: x64 + native: false # Debian Arm - os: ubuntu-22.04 node_arch: arm64 cpp_arch: amd64_arm64 - distro: node:18 - install: | - apt-get update -y && \ - apt-get install --no-install-recommends -y \ - bash \ - build-essential \ - curl \ - git \ - g++ \ - make \ - ninja-build \ - pkg-config \ - unzip \ - zip \ - python3 \ - tar \ - cmake \ - ninja-build \ - automake \ - autoconf \ - libtool + distro: bullseye + native: false # Musl Alpine Arm - os: ubuntu-22.04 node_arch: arm64 cpp_arch: amd64_arm64 - install: | - apk add --no-cache \ - bash \ - build-base \ - curl \ - git \ - g++ \ - make \ - ninja-build \ - pkgconfig \ - unzip \ - zip \ - python3 \ - tar \ - cmake \ - ninja \ - musl-dev \ - automake \ - autoconf \ - libtool && \ - cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja + distro: alpine_latest + native: false env: npm_config_arch: ${{ matrix.node_arch }} @@ -112,9 +83,8 @@ jobs: path: | ./node_modules/ ~/vcpkg - key: - # prettier-ignore - "cache-OS:${{ matrix.os }}-arch:${{ matrix.node_arch}}-${{hashFiles('./pnpm-lock.yaml', './vcpkg.json', './CMakeLists.txt') }}" + # prettier-ignore + key: "cache-OS:${{ matrix.os }}-arch:${{ matrix.node_arch}}-${{hashFiles('./pnpm-lock.yaml', './vcpkg.json', './CMakeLists.txt') }}" restore-keys: | "cache-OS:${{ matrix.os }}-arch:${{ matrix.node_arch }}-" @@ -126,7 +96,7 @@ jobs: shell: bash - name: Setup Cpp - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} uses: aminya/setup-cpp@v1 with: vcvarsall: ${{ contains(matrix.os, 'windows') }} @@ -142,34 +112,34 @@ jobs: brew install gnutls autoconf automake libtool - uses: pnpm/action-setup@v4 - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} with: version: 9 - name: Install Node 20 - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} uses: actions/setup-node@v4 with: node-version: 20 architecture: ${{ env.setup_node_arch }} - name: Install and Build Native - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} run: pnpm install - name: Build JavaScript - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} run: pnpm run build.js - name: Install Node 10 - if: ${{ !matrix.dockerfile && !matrix.install && matrix.os != 'macos-14' }} + if: ${{ matrix.native && matrix.os != 'macos-14' }} uses: actions/setup-node@v4 with: node-version: 10 architecture: ${{ env.setup_node_arch }} - + - name: Build Native - if: ${{ !matrix.dockerfile && !matrix.install && matrix.node_arch != 'ia32' }} + if: ${{ matrix.native && matrix.node_arch != 'ia32' }} run: npm run build.native - name: Build Native Windows 32 @@ -179,7 +149,7 @@ jobs: windows-x86 - name: Use Node 20 - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} uses: actions/setup-node@v4 with: node-version: 20 @@ -195,7 +165,7 @@ jobs: docker rm -f zeromq-temp - name: Build Linux Arm64 - if: ${{ matrix.install }} + if: ${{ matrix.distro }} uses: uraimo/run-on-arch-action@v2 with: arch: aarch64 @@ -208,7 +178,76 @@ jobs: env: | VCPKG_FORCE_SYSTEM_BINARIES: 1 install: | - ${{ matrix.install }} + case "${{ matrix.distro }}" in + ubuntu*|jessie|stretch|buster|bullseye) + apt-get update -q -y + apt-get install --no-install-recommends -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool \ + nodejs + ;; + fedora*) + dnf update -q -y + dnf install -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool \ + nodejs + ;; + alpine*) + apk add --no-cache \ + bash \ + build-base \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkgconfig \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja \ + musl-dev \ + automake \ + autoconf \ + libtool \ + nodejs && \ + cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja + ;; + esac + run: | npm i -g pnpm && \ pnpm install && \ @@ -222,11 +261,11 @@ jobs: overwrite: true - name: Lint - if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" + if: "${{ contains(matrix.os, 'ubuntu') && matrix.native }}" run: pnpm run lint-test - name: Test - if: ${{ !matrix.dockerfile && !matrix.install }} + if: ${{ matrix.native }} uses: nick-fields/retry@v3 with: timeout_minutes: 5 @@ -237,7 +276,7 @@ jobs: rm -rf ./tmp && mkdir -p ./tmp - name: Test Electron Windows/MacOS - if: "${{ !contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" + if: "${{ !contains(matrix.os, 'ubuntu') && matrix.native }}" uses: nick-fields/retry@v3 with: timeout_minutes: 5 @@ -247,7 +286,7 @@ jobs: continue-on-error: true - name: Test Electron Linux - if: "${{ contains(matrix.os, 'ubuntu') && !matrix.dockerfile && !matrix.install }}" + if: "${{ contains(matrix.os, 'ubuntu') && matrix.native }}" uses: nick-fields/retry@v3 with: timeout_minutes: 5 From 039acc98063e2bd9343d5b09d09c325435a423b2 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:20:40 -0800 Subject: [PATCH 04/15] ci: use ubuntu 24 for non-native builds --- .github/workflows/CI.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a82a9115..1fb14812 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -50,21 +50,21 @@ jobs: native: true # Musl Alpine - - os: ubuntu-22.04 + - os: ubuntu-24.04 dockerfile: docker/alpine.dockerfile node_arch: x64 cpp_arch: x64 native: false # Debian Arm - - os: ubuntu-22.04 + - os: ubuntu-24.04 node_arch: arm64 cpp_arch: amd64_arm64 distro: bullseye native: false # Musl Alpine Arm - - os: ubuntu-22.04 + - os: ubuntu-24.04 node_arch: arm64 cpp_arch: amd64_arm64 distro: alpine_latest @@ -166,7 +166,7 @@ jobs: - name: Build Linux Arm64 if: ${{ matrix.distro }} - uses: uraimo/run-on-arch-action@v2 + uses: uraimo/run-on-arch-action@v2.8.1 with: arch: aarch64 distro: ${{ matrix.distro }} From d8912ee4dd43f50fe730c903e453fe499bcebea5 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:25:45 -0800 Subject: [PATCH 05/15] ci: fix dependency names for Linux Arm installation --- .github/workflows/CI.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 1fb14812..30814157 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -199,7 +199,8 @@ jobs: automake \ autoconf \ libtool \ - nodejs + nodejs \ + npm ;; fedora*) dnf update -q -y @@ -238,18 +239,18 @@ jobs: python3 \ tar \ cmake \ - ninja \ musl-dev \ automake \ autoconf \ libtool \ - nodejs && \ + nodejs \ + npm && \ cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja ;; esac + npm i -g pnpm run: | - npm i -g pnpm && \ pnpm install && \ pnpm run build From eaece875c0370df6a62dcc4bf9b968bdaf1b4bc3 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:31:52 -0800 Subject: [PATCH 06/15] ci: use bookworm for Linux Arm + apk update --- .github/workflows/CI.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 30814157..3e0387c5 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -60,7 +60,7 @@ jobs: - os: ubuntu-24.04 node_arch: arm64 cpp_arch: amd64_arm64 - distro: bullseye + distro: bookworm native: false # Musl Alpine Arm @@ -178,6 +178,7 @@ jobs: env: | VCPKG_FORCE_SYSTEM_BINARIES: 1 install: | + set -x case "${{ matrix.distro }}" in ubuntu*|jessie|stretch|buster|bullseye) apt-get update -q -y @@ -225,6 +226,7 @@ jobs: nodejs ;; alpine*) + apk update apk add --no-cache \ bash \ build-base \ From 3335f180e9b23d498fd65c9bd781f50e60a1ab86 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:35:56 -0800 Subject: [PATCH 07/15] ci: fix debian selection + apk install --- .github/workflows/CI.yml | 48 +++++++++++++--------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 3e0387c5..db4070f0 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -180,9 +180,9 @@ jobs: install: | set -x case "${{ matrix.distro }}" in - ubuntu*|jessie|stretch|buster|bullseye) - apt-get update -q -y - apt-get install --no-install-recommends -y \ + fedora*) + dnf update -q -y + dnf install -y \ bash \ build-essential \ curl \ @@ -200,12 +200,17 @@ jobs: automake \ autoconf \ libtool \ - nodejs \ - npm + nodejs ;; - fedora*) - dnf update -q -y - dnf install -y \ + alpine*) + apk update + apk add --no-cache bash build-base curl git g++ make ninja-build pkgconfig unzip zip python3 tar cmake musl-dev automake autoconf libtool nodejs npm + cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja + ;; + *) + # Default case (Ubuntu/Debian) + apt-get update -q -y + apt-get install --no-install-recommends -y \ bash \ build-essential \ curl \ @@ -223,31 +228,8 @@ jobs: automake \ autoconf \ libtool \ - nodejs - ;; - alpine*) - apk update - apk add --no-cache \ - bash \ - build-base \ - curl \ - git \ - g++ \ - make \ - ninja-build \ - pkgconfig \ - unzip \ - zip \ - python3 \ - tar \ - cmake \ - musl-dev \ - automake \ - autoconf \ - libtool \ - nodejs \ - npm && \ - cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja + nodejs \ + npm ;; esac npm i -g pnpm From d20d90f95cd9f081f99c520618453a8ace0efcdf Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Tue, 31 Dec 2024 19:53:53 -0800 Subject: [PATCH 08/15] ci: cache and install vcpkg and zeromq for LinuxArm --- .github/workflows/CI.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index db4070f0..d7b62ded 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -233,6 +233,8 @@ jobs: ;; esac npm i -g pnpm + npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" + ~/vcpkg/vcpkg install 'zeromq[draft,curve,sodium]' run: | pnpm install && \ From 08840f43106b726785e382f71226c88beb1ea08b Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 2 Jan 2025 10:27:57 -0800 Subject: [PATCH 09/15] ci: add install-deps script --- .github/workflows/CI.yml | 58 +------------------------------ script/install-deps.bash | 74 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 57 deletions(-) create mode 100755 script/install-deps.bash diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d7b62ded..47c21f73 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -178,63 +178,7 @@ jobs: env: | VCPKG_FORCE_SYSTEM_BINARIES: 1 install: | - set -x - case "${{ matrix.distro }}" in - fedora*) - dnf update -q -y - dnf install -y \ - bash \ - build-essential \ - curl \ - git \ - g++ \ - make \ - ninja-build \ - pkg-config \ - unzip \ - zip \ - python3 \ - tar \ - cmake \ - ninja-build \ - automake \ - autoconf \ - libtool \ - nodejs - ;; - alpine*) - apk update - apk add --no-cache bash build-base curl git g++ make ninja-build pkgconfig unzip zip python3 tar cmake musl-dev automake autoconf libtool nodejs npm - cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja - ;; - *) - # Default case (Ubuntu/Debian) - apt-get update -q -y - apt-get install --no-install-recommends -y \ - bash \ - build-essential \ - curl \ - git \ - g++ \ - make \ - ninja-build \ - pkg-config \ - unzip \ - zip \ - python3 \ - tar \ - cmake \ - ninja-build \ - automake \ - autoconf \ - libtool \ - nodejs \ - npm - ;; - esac - npm i -g pnpm - npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" - ~/vcpkg/vcpkg install 'zeromq[draft,curve,sodium]' + ./script/install-deps.bash run: | pnpm install && \ diff --git a/script/install-deps.bash b/script/install-deps.bash new file mode 100755 index 00000000..58ed60c5 --- /dev/null +++ b/script/install-deps.bash @@ -0,0 +1,74 @@ +#!/bin/bash + +set -x + +# Ubuntu/Debian +apt=$(command -v apt-get || true) +if [[ -n $apt ]]; then + apt-get update -q -y + apt-get install --no-install-recommends -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool \ + nodejs \ + npm +fi + +# Alpine Linux +apk=$(command -v apk || true) +if [[ -n $apk ]]; then + apk update + apk add --no-cache bash build-base curl git g++ make ninja-build pkgconfig unzip zip python3 tar cmake musl-dev automake autoconf libtool nodejs npm + cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja +fi + +# Fedora/RHEL +dnf=$(command -v dnf || true) +if [[ -n $dnf ]]; then + dnf update -q -y + dnf install -y \ + bash \ + build-essential \ + curl \ + git \ + g++ \ + make \ + ninja-build \ + pkg-config \ + unzip \ + zip \ + python3 \ + tar \ + cmake \ + ninja-build \ + automake \ + autoconf \ + libtool \ + nodejs +fi + +# pnpm +npm i -g pnpm + +# vcpkg +npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" + +# zeromq +export VCPKG_FORCE_SYSTEM_BINARIES=1 +cd ~/vcpkg || exit 1 +~/vcpkg/vcpkg install 'zeromq[draft,curve,sodium]' || (cd - || exit 1) +cd - || exit 1 From 33e79aacf2a3621c74b554597fbd2a77081bee9a Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 2 Jan 2025 14:31:57 -0800 Subject: [PATCH 10/15] ci: read/interpolate the install script --- .github/workflows/CI.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 47c21f73..a4280197 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -164,6 +164,12 @@ jobs: docker cp zeromq-temp:/app/build ./ docker rm -f zeromq-temp + - name: Read Installer Script + if: ${{ matrix.distro }} + id: read-installer-script + run: | + echo "install_deps=$(cat ./script/install-deps.bash)" >> $GITHUB_OUTPUT + - name: Build Linux Arm64 if: ${{ matrix.distro }} uses: uraimo/run-on-arch-action@v2.8.1 @@ -178,7 +184,7 @@ jobs: env: | VCPKG_FORCE_SYSTEM_BINARIES: 1 install: | - ./script/install-deps.bash + ${{ steps.read-installer-script.outputs.install_deps }} run: | pnpm install && \ From 33d51d94afa7601d6871469ce8c8bd8549965ab3 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 2 Jan 2025 23:05:38 -0800 Subject: [PATCH 11/15] ci: read the multi-line script --- .github/workflows/CI.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a4280197..fd394c22 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -168,7 +168,10 @@ jobs: if: ${{ matrix.distro }} id: read-installer-script run: | - echo "install_deps=$(cat ./script/install-deps.bash)" >> $GITHUB_OUTPUT + EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) + echo "install_deps<<$EOF" >> $GITHUB_OUTPUT + cat ./script/install-deps.bash >> $GITHUB_OUTPUT + echo "$EOF" >> $GITHUB_OUTPUT - name: Build Linux Arm64 if: ${{ matrix.distro }} From e99c46561fd928f0d3b77db3d2a025b75fd5586e Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 2 Jan 2025 23:11:20 -0800 Subject: [PATCH 12/15] fix: use sh for the installer script --- .github/workflows/CI.yml | 2 +- script/{install-deps.bash => install-deps.sh} | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) rename script/{install-deps.bash => install-deps.sh} (94%) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index fd394c22..a9c86ae6 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -170,7 +170,7 @@ jobs: run: | EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64) echo "install_deps<<$EOF" >> $GITHUB_OUTPUT - cat ./script/install-deps.bash >> $GITHUB_OUTPUT + cat ./script/install-deps.sh >> $GITHUB_OUTPUT echo "$EOF" >> $GITHUB_OUTPUT - name: Build Linux Arm64 diff --git a/script/install-deps.bash b/script/install-deps.sh similarity index 94% rename from script/install-deps.bash rename to script/install-deps.sh index 58ed60c5..202e0295 100755 --- a/script/install-deps.bash +++ b/script/install-deps.sh @@ -1,10 +1,10 @@ -#!/bin/bash +#!/bin/sh set -x # Ubuntu/Debian apt=$(command -v apt-get || true) -if [[ -n $apt ]]; then +if [ -n "$apt" ]; then apt-get update -q -y apt-get install --no-install-recommends -y \ bash \ @@ -30,7 +30,7 @@ fi # Alpine Linux apk=$(command -v apk || true) -if [[ -n $apk ]]; then +if [ -n "$apk" ]; then apk update apk add --no-cache bash build-base curl git g++ make ninja-build pkgconfig unzip zip python3 tar cmake musl-dev automake autoconf libtool nodejs npm cp /usr/lib/ninja-build/bin/ninja /usr/bin/ninja @@ -38,7 +38,7 @@ fi # Fedora/RHEL dnf=$(command -v dnf || true) -if [[ -n $dnf ]]; then +if [ -n "$dnf" ]; then dnf update -q -y dnf install -y \ bash \ From bad314851a821861e9d0e2fafe34bbe563beacca Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Thu, 2 Jan 2025 23:56:44 -0800 Subject: [PATCH 13/15] fix: force vcpkg system binaries before install --- script/install-deps.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/install-deps.sh b/script/install-deps.sh index 202e0295..e93e55bd 100755 --- a/script/install-deps.sh +++ b/script/install-deps.sh @@ -64,11 +64,12 @@ fi # pnpm npm i -g pnpm +export VCPKG_FORCE_SYSTEM_BINARIES=1 + # vcpkg npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" # zeromq -export VCPKG_FORCE_SYSTEM_BINARIES=1 cd ~/vcpkg || exit 1 ~/vcpkg/vcpkg install 'zeromq[draft,curve,sodium]' || (cd - || exit 1) cd - || exit 1 From 7ff9b45dff9cc41c1abb777f0542a20651f6adb5 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Fri, 3 Jan 2025 00:47:25 -0800 Subject: [PATCH 14/15] ci: increase the installation timeout to 1 hour --- script/install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/install-deps.sh b/script/install-deps.sh index e93e55bd..aedd2273 100755 --- a/script/install-deps.sh +++ b/script/install-deps.sh @@ -67,7 +67,7 @@ npm i -g pnpm export VCPKG_FORCE_SYSTEM_BINARIES=1 # vcpkg -npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" +npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" --timeout 3600000 # zeromq cd ~/vcpkg || exit 1 From 65c8e087da3e80809ef6f9a40279938ab59d2e05 Mon Sep 17 00:00:00 2001 From: Amin Yahyaabadi Date: Sun, 5 Jan 2025 02:39:43 -0800 Subject: [PATCH 15/15] ci: clone/build vcpkg directly --- script/install-deps.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/script/install-deps.sh b/script/install-deps.sh index aedd2273..1e9cc4dd 100755 --- a/script/install-deps.sh +++ b/script/install-deps.sh @@ -67,7 +67,11 @@ npm i -g pnpm export VCPKG_FORCE_SYSTEM_BINARIES=1 # vcpkg -npx -y setup-cpp --vcpkg "ee2d2a100103e0f3613c60655dcf15be7d5157b8" --timeout 3600000 +git clone https://github.com/microsoft/vcpkg.git ~/vcpkg +cd ~/vcpkg || exit 1 +git checkout "ee2d2a100103e0f3613c60655dcf15be7d5157b8" +~/vcpkg/bootstrap-vcpkg.sh +cd - || exit 1 # zeromq cd ~/vcpkg || exit 1