From ab21f79b060c8320040b46b85c85170261a67616 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Thu, 19 Sep 2024 20:57:51 +0100 Subject: [PATCH] ci: nightly releases The nightly release will build the binary set from the `main` branch every night at midnight UTC. The workflow: * Builds the binaries with the `nightly` feature, which versions them using the current date, rather than a Semantic Version. * Uploads packaged binaries to S3, again with the date for the version. * Also uploads packaged binaries to S3 with 'nightly' as the version, which will always be the latest. The previous 'nightly' version will be deleted from the bucket. * Creates a Github Release and uploads archives with the binaries packaged by architecture, as per the stable release. To prevent our releases page being flooded, the previous nightly release will be deleted before the current one is created. --- .github/workflows/nightly-release.yml | 256 ++++++++++++++++++++++++++ Justfile | 109 ++++++++--- 2 files changed, 343 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/nightly-release.yml diff --git a/.github/workflows/nightly-release.yml b/.github/workflows/nightly-release.yml new file mode 100644 index 0000000000..4c7b41f429 --- /dev/null +++ b/.github/workflows/nightly-release.yml @@ -0,0 +1,256 @@ +name: nightly release + +# on: +# schedule: +# - cron: '0 0 * * *' # Run every night at midnight UTC +# workflow_dispatch: # This also allows the workflow to be triggered manually + +on: + push: + branches: + - ci-nightly_releases + +env: + WORKFLOW_URL: https://github.com/maidsafe/safe_network/actions/runs + +jobs: + build: + if: ${{ github.repository_owner == 'maidsafe' }} + name: build + environment: stable + env: + FOUNDATION_PK: ${{ vars.FOUNDATION_PK }} + GENESIS_PK: ${{ vars.GENESIS_PK }} + GENESIS_SK: ${{ secrets.GENESIS_SK }} + NETWORK_ROYALTIES_PK: ${{ vars.NETWORK_ROYALTIES_PK }} + PAYMENT_FORWARD_PK: ${{ vars.PAYMENT_FORWARD_PK }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - os: windows-latest + target: x86_64-pc-windows-msvc + - os: macos-latest + target: x86_64-apple-darwin + - os: macos-latest + target: aarch64-apple-darwin + - os: ubuntu-latest + target: x86_64-unknown-linux-musl + - os: ubuntu-latest + target: arm-unknown-linux-musleabi + - os: ubuntu-latest + target: armv7-unknown-linux-musleabihf + - os: ubuntu-latest + target: aarch64-unknown-linux-musl + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: cargo-bins/cargo-binstall@main + - shell: bash + run: cargo binstall --no-confirm just + + - name: build nightly release artifacts + shell: bash + run: | + just build-release-artifacts "${{ matrix.target }}" "true" + + - uses: actions/upload-artifact@main + with: + name: safe_network-${{ matrix.target }} + path: | + artifacts + !artifacts/.cargo-lock + + - name: post notification to slack on failure + if: ${{ failure() }} + uses: bryannice/gitactions-slack-notification@2.0.0 + env: + SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + SLACK_TITLE: "Release Failed" + + s3-release: + if: ${{ github.repository_owner == 'maidsafe' }} + name: s3 release + runs-on: ubuntu-latest + needs: [build] + env: + AWS_ACCESS_KEY_ID: ${{ secrets.S3_DEPLOY_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_DEPLOY_AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: eu-west-2 + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-pc-windows-msvc + path: artifacts/x86_64-pc-windows-msvc/release + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-unknown-linux-musl + path: artifacts/x86_64-unknown-linux-musl/release + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-apple-darwin + path: artifacts/x86_64-apple-darwin/release + - uses: actions/download-artifact@master + with: + name: safe_network-aarch64-apple-darwin + path: artifacts/aarch64-apple-darwin/release + - uses: actions/download-artifact@master + with: + name: safe_network-arm-unknown-linux-musleabi + path: artifacts/arm-unknown-linux-musleabi/release + - uses: actions/download-artifact@master + with: + name: safe_network-armv7-unknown-linux-musleabihf + path: artifacts/armv7-unknown-linux-musleabihf/release + - uses: actions/download-artifact@master + with: + name: safe_network-aarch64-unknown-linux-musl + path: artifacts/aarch64-unknown-linux-musl/release + + - uses: cargo-bins/cargo-binstall@main + - name: install just + shell: bash + run: cargo binstall --no-confirm just + + - name: remove latest nightly release + shell: bash + run: | + just delete-s3-bin "faucet" "nightly" + just delete-s3-bin "nat-detection" "nightly" + just delete-s3-bin "node-launchpad" "nightly" + just delete-s3-bin "safe" "nightly" + just delete-s3-bin "safenode" "nightly" + just delete-s3-bin "safenode_rpc_client" "nightly" + just delete-s3-bin "safenode-manager" "nightly" + just delete-s3-bin "safenodemand" "nightly" + just delete-s3-bin "sn_auditor" "nightly" + + - name: upload binaries to S3 + shell: bash + run: | + version=$(date +"%Y.%m.%d") + just package-bin "faucet" "$version" + just package-bin "nat-detection" "$version" + just package-bin "node-launchpad" "$version" + just package-bin "safe" "$version" + just package-bin "safenode" "$version" + just package-bin "safenode_rpc_client" "$version" + just package-bin "safenode-manager" "$version" + just package-bin "safenodemand" "$version" + just package-bin "sn_auditor" "$version" + just upload-all-packaged-bins-to-s3 + + rm -rf packaged_bins + just package-bin "faucet" "nightly" + just package-bin "nat-detection" "nightly" + just package-bin "node-launchpad" "nightly" + just package-bin "safe" "nightly" + just package-bin "safenode" "nightly" + just package-bin "safenode_rpc_client" "nightly" + just package-bin "safenode-manager" "nightly" + just package-bin "safenodemand" "nightly" + just package-bin "sn_auditor" "nightly" + just upload-all-packaged-bins-to-s3 + + github-release: + if: ${{ github.repository_owner == 'maidsafe' }} + name: github release + runs-on: ubuntu-latest + needs: [s3-release] + steps: + - uses: actions/checkout@v4 + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-pc-windows-msvc + path: artifacts/x86_64-pc-windows-msvc/release + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-unknown-linux-musl + path: artifacts/x86_64-unknown-linux-musl/release + - uses: actions/download-artifact@master + with: + name: safe_network-x86_64-apple-darwin + path: artifacts/x86_64-apple-darwin/release + - uses: actions/download-artifact@master + with: + name: safe_network-aarch64-apple-darwin + path: artifacts/aarch64-apple-darwin/release + - uses: actions/download-artifact@master + with: + name: safe_network-arm-unknown-linux-musleabi + path: artifacts/arm-unknown-linux-musleabi/release + - uses: actions/download-artifact@master + with: + name: safe_network-armv7-unknown-linux-musleabihf + path: artifacts/armv7-unknown-linux-musleabihf/release + - uses: actions/download-artifact@master + with: + name: safe_network-aarch64-unknown-linux-musl + path: artifacts/aarch64-unknown-linux-musl/release + + - uses: cargo-bins/cargo-binstall@main + - name: install just + shell: bash + run: cargo binstall --no-confirm just + + - name: set package version + shell: bash + run: | + version=$(date +"%Y.%m.%d") + echo "PACKAGE_VERSION=$version" >> $GITHUB_ENV + + - name: package release artifacts + shell: bash + run: just package-all-architectures + + - name: delete existing nightly release + env: + GITHUB_TOKEN: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} + run: | + releases=$(gh api repos/${{ github.repository }}/releases --paginate) + echo "$releases" | jq -c '.[]' | while read release; do + tag_name=$(echo $release | jq -r '.tag_name') + release_id=$(echo $release | jq -r '.id') + + if [[ $tag_name == nightly* ]]; then + echo "deleting nightly release $tag_name" + gh api -X DELETE repos/${{ github.repository }}/releases/$release_id + exit 0 + fi + done + + - name: create new nightly release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} + with: + tag_name: nightly-${{ env.PACKAGE_VERSION }} + release_name: "${{ env.PACKAGE_VERSION }} Nightly Release" + body: | + Nightly release of the Autonomi binary set, built from the `main` branch. + + These binaries should be compatible with the stable network, but they should be considered experimental. + + For the most reliable experience, prefer the latest stable release. + draft: false + prerelease: true + + - name: upload artifacts as assets + env: + GITHUB_TOKEN: ${{ secrets.VERSION_BUMP_COMMIT_PAT }} + shell: bash + run: | + ( + cd packaged_architectures + ls | xargs gh release upload nightly-${{ env.PACKAGE_VERSION }} + ) + + - name: post notification to slack on failure + if: ${{ failure() }} + uses: bryannice/gitactions-slack-notification@2.0.0 + env: + SLACK_INCOMING_WEBHOOK: ${{ secrets.SLACK_GH_ACTIONS_WEBHOOK_URL }} + SLACK_MESSAGE: "Please check the logs for the run at ${{ env.WORKFLOW_URL }}/${{ github.run_id }}" + SLACK_TITLE: "Nightly Release Failed" \ No newline at end of file diff --git a/Justfile b/Justfile index 8e260b9804..693929fcaf 100644 --- a/Justfile +++ b/Justfile @@ -65,11 +65,12 @@ kill-testbed: doctl compute droplet delete $droplet_id fi -build-release-artifacts arch: +build-release-artifacts arch nightly="false": #!/usr/bin/env bash set -e arch="{{arch}}" + nightly="{{nightly}}" supported_archs=( "x86_64-pc-windows-msvc" "x86_64-apple-darwin" @@ -107,9 +108,9 @@ build-release-artifacts arch: mkdir artifacts cargo clean - echo "===============" - echo "= Public Keys =" - echo "===============" + echo "================" + echo "= Network Keys =" + echo "================" echo "FOUNDATION_PK: $FOUNDATION_PK" echo "GENESIS_PK: $GENESIS_PK" echo "NETWORK_ROYALTIES_PK: $NETWORK_ROYALTIES_PK" @@ -118,28 +119,33 @@ build-release-artifacts arch: cross_container_opts="--env \"GENESIS_PK=$GENESIS_PK\" --env \"GENESIS_SK=$GENESIS_SK\" --env \"FOUNDATION_PK=$FOUNDATION_PK\" --env \"NETWORK_ROYALTIES_PK=$NETWORK_ROYALTIES_PK\" --env \"PAYMENT_FORWARD_PK=$PAYMENT_FORWARD_PK\"" export CROSS_CONTAINER_OPTS=$cross_container_opts + nightly_feature="" + if [[ "$nightly" == "true" ]]; then + nightly_feature="--features nightly" + fi + if [[ $arch == arm* || $arch == armv7* || $arch == aarch64* ]]; then echo "Passing to cross CROSS_CONTAINER_OPTS=$CROSS_CONTAINER_OPTS" cargo binstall --no-confirm cross - cross build --release --target $arch --bin faucet --features=distribution - cross build --release --target $arch --bin nat-detection - cross build --release --target $arch --bin node-launchpad - cross build --release --features="network-contacts,distribution" --target $arch --bin safe - cross build --release --features=network-contacts --target $arch --bin safenode - cross build --release --target $arch --bin safenode-manager - cross build --release --target $arch --bin safenodemand - cross build --release --target $arch --bin safenode_rpc_client - cross build --release --target $arch --bin sn_auditor + cross build --release --target $arch --bin faucet --features=distribution $nightly_feature + cross build --release --target $arch --bin nat-detection $nightly_feature + cross build --release --target $arch --bin node-launchpad $nightly_feature + cross build --release --features="network-contacts,distribution" --target $arch --bin safe $nightly_feature + cross build --release --features=network-contacts --target $arch --bin safenode $nightly_feature + cross build --release --target $arch --bin safenode-manager $nightly_feature + cross build --release --target $arch --bin safenodemand $nightly_feature + cross build --release --target $arch --bin safenode_rpc_client $nightly_feature + cross build --release --target $arch --bin sn_auditor $nightly_feature else - cargo build --release --target $arch --bin faucet --features=distribution - cargo build --release --target $arch --bin nat-detection - cargo build --release --target $arch --bin node-launchpad - cargo build --release --features="network-contacts,distribution" --target $arch --bin safe - cargo build --release --features=network-contacts --target $arch --bin safenode - cargo build --release --target $arch --bin safenode-manager - cargo build --release --target $arch --bin safenodemand - cargo build --release --target $arch --bin safenode_rpc_client - cargo build --release --target $arch --bin sn_auditor + cargo build --release --target $arch --bin faucet --features=distribution $nightly_feature + cargo build --release --target $arch --bin nat-detection $nightly_feature + cargo build --release --target $arch --bin node-launchpad $nightly_feature + cargo build --release --features="network-contacts,distribution" --target $arch --bin safe $nightly_feature + cargo build --release --features=network-contacts --target $arch --bin safenode $nightly_feature + cargo build --release --target $arch --bin safenode-manager $nightly_feature + cargo build --release --target $arch --bin safenodemand $nightly_feature + cargo build --release --target $arch --bin safenode_rpc_client $nightly_feature + cargo build --release --target $arch --bin sn_auditor $nightly_feature fi find target/$arch/release -maxdepth 1 -type f -exec cp '{}' artifacts \; @@ -347,6 +353,65 @@ upload-packaged-bin-to-s3 bin_name: fi done +delete-s3-bin bin_name version: + #!/usr/bin/env bash + set -e + + case "{{bin_name}}" in + faucet) + bucket="sn-faucet" + ;; + nat-detection) + bucket="nat-detection" + ;; + node-launchpad) + bucket="node-launchpad" + ;; + safe) + bucket="sn-cli" + ;; + safenode) + bucket="sn-node" + ;; + safenode-manager) + bucket="sn-node-manager" + ;; + safenodemand) + bucket="sn-node-manager" + ;; + safenode_rpc_client) + bucket="sn-node-rpc-client" + ;; + sn_auditor) + bucket="sn-auditor" + ;; + *) + echo "The {{bin_name}} binary is not supported" + exit 1 + ;; + esac + + architectures=( + "x86_64-pc-windows-msvc" + "x86_64-apple-darwin" + "aarch64-apple-darwin" + "x86_64-unknown-linux-musl" + "arm-unknown-linux-musleabi" + "armv7-unknown-linux-musleabihf" + "aarch64-unknown-linux-musl" + ) + + for arch in "${architectures[@]}"; do + zip_filename="{{bin_name}}-{{version}}-${arch}.zip" + tar_filename="{{bin_name}}-{{version}}-${arch}.tar.gz" + s3_zip_path="s3://$bucket/$zip_filename" + s3_tar_path="s3://$bucket/$tar_filename" + aws s3 rm "$s3_zip_path" + echo "deleted $s3_zip_path" + aws s3 rm "$s3_tar_path" + echo "deleted $s3_tar_path" + done + package-all-architectures: #!/usr/bin/env bash set -e