From 7af911250b3096c6e5c90bdfe11589976a0f274f Mon Sep 17 00:00:00 2001 From: avifenesh Date: Fri, 3 May 2024 11:45:02 +0000 Subject: [PATCH] Adding support for node:alpine --- .../workflows/build-node-wrapper/action.yml | 3 +- .../install-rust-and-protoc/action.yml | 32 ++++ .../install-shared-dependencies/action.yml | 30 ++-- .../node-create-package-file/action.yml | 8 +- .github/workflows/node.yml | 81 +++++++++- .github/workflows/npm-cd.yml | 151 +++++++++++++++++- node/npm/glide/index.ts | 25 ++- node/npm/glide/package.json | 7 +- 8 files changed, 317 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/install-rust-and-protoc/action.yml diff --git a/.github/workflows/build-node-wrapper/action.yml b/.github/workflows/build-node-wrapper/action.yml index be57e1351f..6a1cd8fd55 100644 --- a/.github/workflows/build-node-wrapper/action.yml +++ b/.github/workflows/build-node-wrapper/action.yml @@ -65,6 +65,7 @@ runs: named_os: ${{ inputs.named_os }} arch: ${{ inputs.arch }} npm_scope: ${{ inputs.npm_scope }} + target: ${{ inputs.target }} - name: npm install shell: bash @@ -72,7 +73,7 @@ runs: run: | rm -rf node_modules && npm install --frozen-lockfile cd rust-client - npm install + npm install --frozen-lockfile - name: Build shell: bash diff --git a/.github/workflows/install-rust-and-protoc/action.yml b/.github/workflows/install-rust-and-protoc/action.yml new file mode 100644 index 0000000000..18588ca9f7 --- /dev/null +++ b/.github/workflows/install-rust-and-protoc/action.yml @@ -0,0 +1,32 @@ +name: Install Rust tool chain and protoc + +inputs: + target: + description: "Specified target for rust toolchain, ex. x86_64-apple-darwin" + type: string + required: false + defalt: "x86_64-unknown-linux-gnu" + options: + - x86_64-unknown-linux-gnu + - aarch64-unknown-linux-gnu + - x86_64-apple-darwin + - aarch64-apple-darwin + - aarch64-unknown-linux-musl + - x86_64-unknown-linux-musl + + +runs: + using: "composite" + steps: + - name: Install Rust toolchain + if: "${{ inputs.target != 'x86_64-unknown-linux-musl' && inputs.target != 'aarch64-unknown-linux-musl' }}" + uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ inputs.target }} + + - name: Install protoc (protobuf) + if: "${{ inputs.target != 'x86_64-unknown-linux-musl' && inputs.target != 'aarch64-unknown-linux-musl' }}" + uses: arduino/setup-protoc@v3 + with: + version: "25.1" + repo-token: ${{ inputs.github-token }} diff --git a/.github/workflows/install-shared-dependencies/action.yml b/.github/workflows/install-shared-dependencies/action.yml index 66dde73a34..af142cbefe 100644 --- a/.github/workflows/install-shared-dependencies/action.yml +++ b/.github/workflows/install-shared-dependencies/action.yml @@ -19,11 +19,15 @@ inputs: - aarch64-unknown-linux-gnu - x86_64-apple-darwin - aarch64-apple-darwin + - aarch64-unknown-linux-musl + - x86_64-unknown-linux-musl github-token: description: "GITHUB_TOKEN, GitHub App installation access token" required: true type: string + + runs: using: "composite" steps: @@ -35,12 +39,21 @@ runs: brew upgrade || true brew install git gcc pkgconfig openssl redis coreutils - - name: Install software dependencies for Ubuntu + - name: Install software dependencies for Ubuntu GNU shell: bash - if: "${{ inputs.os == 'ubuntu' }}" + if: "${{ inputs.os == 'ubuntu' && inputs.target != 'x86_64-unknown-linux-musl' && inputs.target != 'aarch64-unknown-linux-musl'}}" run: | sudo apt update -y sudo apt install -y git gcc pkg-config openssl libssl-dev + + - name: Install software dependencies for Ubuntu MUSL + shell: bash + if: "${{ inputs.target == 'x86_64-unknown-linux-musl' || inputs.target == 'aarch64-unknown-linux-musl'}}" + run: | + apk update + wget -O - https://sh.rustup.rs | sh -s -- -y + source "$HOME/.cargo/env" + apk add protobuf-dev musl-dev make gcc redis - name: Install software dependencies for Amazon-Linux shell: bash @@ -48,13 +61,8 @@ runs: run: | yum install -y gcc pkgconfig openssl openssl-devel which curl redis6 gettext --allowerasing - - name: Install Rust toolchain - uses: dtolnay/rust-toolchain@stable - with: - targets: ${{ inputs.target }} - - - name: Install protoc (protobuf) - uses: arduino/setup-protoc@v3 + - name: Install Rust toolchain and protoc + if: "${{ inputs.target != 'x86_64-unknown-linux-musl' && inputs.target != 'aarch64-unknown-linux-musl' }}" + uses: ./.github/workflows/install-rust-and-protoc with: - version: "25.1" - repo-token: ${{ inputs.github-token }} + target: ${{ inputs.target }} diff --git a/.github/workflows/node-create-package-file/action.yml b/.github/workflows/node-create-package-file/action.yml index 2d70b8ae33..e3009df725 100644 --- a/.github/workflows/node-create-package-file/action.yml +++ b/.github/workflows/node-create-package-file/action.yml @@ -34,6 +34,10 @@ inputs: required: false type: string default: "@aws" + target: + description: "Specified target for rust toolchain, ex. x86_64-apple-darwin" + type: string + required: true runs: using: "composite" @@ -42,6 +46,8 @@ runs: shell: bash working-directory: ./node run: | + # echo -musl if inputs.target is musl + export MUSL_FLAG=`if [[ "${{ inputs.target }}" =~ .*"musl".* ]]; then echo "-musl"; fi` # set the package name name="glide-for-redis" # derive the OS and architecture from the inputs @@ -50,7 +56,7 @@ runs: # set the version export package_version="${{ inputs.release_version }}" # set the package name - export pkg_name="${name}-${node_os}-${node_arch}" + export pkg_name="${name}-${node_os}-${node_arch}${MUSL_FLAG}" # set the scope export scope=`if [ "${{ inputs.npm_scope }}" != '' ]; then echo "${{ inputs.npm_scope }}/"; fi` # set the registry scope diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index be56f3a23d..8e98b4fa63 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -177,4 +177,83 @@ jobs: - name: Test compatibility run: npm test -- -t "set and get flow works" - working-directory: ./node + working-directory: ./node + + start-self-hosted-runner: + if: github.repository_owner == 'aws' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start self hosted EC2 runner + uses: ./.github/workflows/start-self-hosted-runner + with: + aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} + aws-region: ${{ secrets.AWS_REGION }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + + set-up-gitconfig-for-cotntainer: + if: github.repository_owner == 'aws' + needs: start-self-hosted-runner + runs-on: [self-hosted, Linux, ARM64] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup self-hosted runner access + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/glide-for-redis + + - name: Set up Git config for build-linux-musl-latest + run: | + mkdir -p $HOME/musl-test-workspace + cp $GITHUB_WORKSPACE/* $HOME/musl-test-workspace + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + build-linux-musl-latest: + needs: set-up-gitconfig-for-cotntainer + if: github.repository_owner == 'aws' + name: Build and test Node wrapper on Linux musl + runs-on: [self-hosted, Linux, ARM64] + container: + image: node:alpine + options: --user root + + strategy: + fail-fast: false + timeout-minutes: 15 + steps: + + - name: Install dependencies + run: | + apk update + apk add bash git envsubst + + # Currently it is not possible to use the checkout action with self-hosted runners on alpine so we need to do it manually + - name: Checkout repository + shell: bash + run: | + git config --global safe.directory "$HOME/musl-test-workspace" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Downgrade npm major version to 8 + run: | + npm i -g npm@8 + + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper + with: + os: ubuntu + named_os: linux + arch: arm64 + target: aarch64-unknown-linux-musl + github-token: ${{ secrets.GITHUB_TOKEN }} + publish: true + + - name: Test compatibility + run: npm test -- -t "set and get flow works" + working-directory: ./node + diff --git a/.github/workflows/npm-cd.yml b/.github/workflows/npm-cd.yml index e6a098ff66..cb8e47bf64 100644 --- a/.github/workflows/npm-cd.yml +++ b/.github/workflows/npm-cd.yml @@ -17,19 +17,21 @@ concurrency: cancel-in-progress: true jobs: + start-self-hosted-runner: - if: github.repository_owner == 'aws' + if: github.repository_owner == 'aws' runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 + - name: Start self hosted EC2 runner uses: ./.github/workflows/start-self-hosted-runner with: aws-access-key-id: ${{ secrets.AWS_EC2_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.AWS_EC2_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_REGION }} - ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} + ec2-instance-id: ${{ secrets.AWS_EC2_INSTANCE_ID }} publish-binaries: needs: start-self-hosted-runner @@ -100,7 +102,7 @@ jobs: with: folder_path: "${{ github.workspace }}/node/rust-client/.cargo" named_os: ${{ matrix.build.NAMED_OS }} - + - name: Build Node wrapper uses: ./.github/workflows/build-node-wrapper with: @@ -152,6 +154,149 @@ jobs: path: ./node/bin if-no-files-found: error + set-up-gitconfig-for-cotntainer: + if: github.repository_owner == 'aws' + needs: start-self-hosted-runner + runs-on: [self-hosted, Linux, ARM64] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup self-hosted runner access + run: sudo chown -R $USER:$USER /home/ubuntu/actions-runner/_work/glide-for-redis + + - name: Set up Git config for publish-musl-binaries + run: | + mkdir -p $HOME/musl-build-workspace + cp $GITHUB_WORKSPACE/* $HOME/musl-build-workspace + env: + GITHUB_WORKSPACE: ${{ github.workspace }} + + publish-musl-binaries: + needs: set-up-gitconfig-for-cotntainer + if: github.repository_owner == 'aws' + name: Publish packages to NPM + runs-on: ${{ matrix.build.RUNNER }} + container: + image: node:alpine + options: --user root + strategy: + fail-fast: false + matrix: + build: + - { + OS: ubuntu, + NAMED_OS: linux, + RUNNER: ubuntu-latest, + ARCH: x64, + TARGET: x86_64-unknown-linux-musl, + } + - { + OS: ubuntu, + NAMED_OS: linux, + RUNNER: [self-hosted, Linux, ARM64], + ARCH: arm64, + TARGET: aarch64-unknown-linux-musl, + CONTAINER: "2_28", + } + steps: + + - name: Install dependencies + run: | + apk update + apk add bash envsubst git sed + + # Currently it is not possible to use the checkout action with self-hosted runners on alpine so we need to do it manually + - name: Checkout repository for ARM64 + if: ${{ contains(matrix.build.RUNNER, 'self-hosted') }} + shell: bash + run: | + git config --global safe.directory "$HOME/musl-build-workspace" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout repository for x64 + if: ${{ !contains(matrix.build.RUNNER, 'self-hosted') }} + uses: actions/checkout@v4 + with: + submodules: "true" + + + - name: Set the release version + shell: bash + run: | + export version=`if ${{ github.event_name == 'pull_request' }}; then echo '255.255.255'; else echo ${GITHUB_REF:11}; fi` + echo "RELEASE_VERSION=${version}" >> $GITHUB_ENV + + - name: Set the package name + shell: bash + run: | + export pkg_name=`if [ "${{ matrix.build.ARCH }}" == 'x64' ]; then echo "glide-for-redis-linux-musl-x64"; else echo "glide-for-redis-linux-musl-arm64"; fi` + echo "PKG_NAME=${pkg_name}" >> $GITHUB_ENV + + - name: Update package version in config.toml + uses: ./.github/workflows/update-glide-version + with: + folder_path: "${{ github.workspace }}/node/rust-client/.cargo" + named_os: ${{ matrix.build.NAMED_OS }} + + - name: Build Node wrapper + uses: ./.github/workflows/build-node-wrapper + with: + os: ${{ matrix.build.OS }} + named_os: ${{ matrix.build.NAMED_OS }} + arch: ${{ matrix.build.ARCH }} + target: ${{ matrix.build.TARGET }} + npm_scope: ${{ vars.NPM_SCOPE }} + publish: "true" + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Setup node + working-directory: ./node + run: | + npm config set registry https://registry.npmjs.org/ + npm config set '//registry.npmjs.org/:_authToken' ${{ secrets.NPM_AUTH_TOKEN }} + npm config set scope ${{ vars.NPM_SCOPE }} + + - name: Publish to NPM + if: github.event_name != 'pull_request' + shell: bash + working-directory: ./node + run: | + set +e + # Redirect only stderr + { npm_publish_err=$(npm publish --access public 2>&1 >&3 3>&-); } 3>&1 + if [[ "$npm_publish_err" == *"You cannot publish over the previously published versions"* ]] + then + echo "Skipping publishing, package already published" + elif [[ ! -z "$npm_publish_err" ]] + then + echo "Failed to publish with error: ${npm_publish_err}" + exit 1 + fi + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + + - name: Pack the Node package + shell: bash + working-directory: ./node + run: | + # Remove the "cpu" and "os" fileds so the base package would be able to install it on ubuntu + sed -i '/"\/\/\/cpu": \[/,/]/d' ./package.json && sed -i '/"\/\/\/os": \[/,/]/d' ./package.json + mkdir -p bin + npm pack --pack-destination ./bin + ls ./bin + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_AUTH_TOKEN }} + + - name: Upload the Node package + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.build.TARGET }} + path: ./node/bin + if-no-files-found: error + publish-base-to-npm: if: github.event_name != 'pull_request' name: Publish the base NPM package diff --git a/node/npm/glide/index.ts b/node/npm/glide/index.ts index 4891f8d224..3f5fd1c597 100644 --- a/node/npm/glide/index.ts +++ b/node/npm/glide/index.ts @@ -4,6 +4,7 @@ * Copyright GLIDE-for-Redis Project Contributors - SPDX Identifier: Apache-2.0 */ +import { GLIBC, MUSL, familySync } from "detect-libc"; import { arch, platform } from "process"; let globalObject = global as unknown; @@ -14,10 +15,30 @@ function loadNativeBinding() { case "linux": switch (arch) { case "x64": - nativeBinding = require("@scope/glide-for-redis-linux-x64"); + switch (familySync()) { + case GLIBC: + nativeBinding = require("@scope/glide-for-redis-linux-x64"); + break; + case MUSL: + nativeBinding = require("@scope/glide-for-redis-linux-musl-x64"); + break; + default: + nativeBinding = require("@scope/glide-for-redis-linux-x64"); + break; + } break; case "arm64": - nativeBinding = require("@scope/glide-for-redis-linux-arm64"); + switch (familySync()) { + case GLIBC: + nativeBinding = require("@scope/glide-for-redis-linux-arm64"); + break; + case MUSL: + nativeBinding = require("@scope/glide-for-redis-linux-musl-arm64"); + break; + default: + nativeBinding = require("@scope/glide-for-redis-linux-arm64"); + break; + } break; default: throw new Error( diff --git a/node/npm/glide/package.json b/node/npm/glide/package.json index b6ded308e1..fadbc33ddd 100644 --- a/node/npm/glide/package.json +++ b/node/npm/glide/package.json @@ -43,7 +43,9 @@ "${scope}glide-for-redis-darwin-arm64": "${package_version}", "${scope}glide-for-redis-darwin-x64": "${package_version}", "${scope}glide-for-redis-linux-arm64": "${package_version}", - "${scope}glide-for-redis-linux-x64": "${package_version}" + "${scope}glide-for-redis-linux-x64": "${package_version}", + "${scope}glide-for-redis-linux-musl-arm64": "${package_version}", + "${scope}glide-for-redis-linux-musl-x64": "${package_version}" }, "eslintConfig": { "extends": [ @@ -58,5 +60,8 @@ "build-ts/*" ], "root": true + }, + "dependencies": { + "detect-libc": "^2.0.3" } }