From 236d634db2096f2588e15a9f9096ac79ab54cd4d Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Wed, 27 Nov 2024 13:26:59 +0100 Subject: [PATCH] test download from s3 and release upload --- .github/scripts/common/lib.sh | 25 +- .../release-30_publish_release_draft.yml | 399 +++++++++++------- .github/workflows/release-build-rc.yml | 2 +- 3 files changed, 259 insertions(+), 167 deletions(-) diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh index 6b8f70a26d7e..7373dc6dd4ba 100755 --- a/.github/scripts/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -270,20 +270,19 @@ fetch_debian_package_from_s3() { } # Fetch the release artifacts like binary and signatures from S3. Assumes the ENV are set: -# - RELEASE_ID -# - GITHUB_TOKEN -# - REPO in the form paritytech/polkadot +# inputs: binary (polkadot), target(aarch64-apple-darwin) fetch_release_artifacts_from_s3() { BINARY=$1 - OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${BINARY}"} + TARGET=$2 + OUTPUT_DIR=${OUTPUT_DIR:-"./release-artifacts/${TARGET}/${BINARY}"} echo "OUTPUT_DIR : $OUTPUT_DIR" URL_BASE=$(get_s3_url_base $BINARY) echo "URL_BASE=$URL_BASE" - URL_BINARY=$URL_BASE/$VERSION/$BINARY - URL_SHA=$URL_BASE/$VERSION/$BINARY.sha256 - URL_ASC=$URL_BASE/$VERSION/$BINARY.asc + URL_BINARY=$URL_BASE/$VERSION/$TARGET/$BINARY + URL_SHA=$URL_BASE/$VERSION/$TARGET/$BINARY.sha256 + URL_ASC=$URL_BASE/$VERSION/$TARGET/$BINARY.asc # Fetch artifacts mkdir -p "$OUTPUT_DIR" @@ -306,15 +305,19 @@ fetch_release_artifacts_from_s3() { function get_s3_url_base() { name=$1 case $name in - polkadot | polkadot-execute-worker | polkadot-prepare-worker | staking-miner) + polkadot | polkadot-execute-worker | polkadot-prepare-worker ) printf "https://releases.parity.io/polkadot" ;; - polkadot-parachain) - printf "https://releases.parity.io/cumulus" + polkadot-parachain) + printf "https://releases.parity.io/polkadot-parachain" + ;; + + polkadot-omni-node) + printf "https://releases.parity.io/polkadot-omni-node" ;; - *) + *) printf "UNSUPPORTED BINARY $name" exit 1 ;; diff --git a/.github/workflows/release-30_publish_release_draft.yml b/.github/workflows/release-30_publish_release_draft.yml index 19850d825461..d782cbfb4704 100644 --- a/.github/workflows/release-30_publish_release_draft.yml +++ b/.github/workflows/release-30_publish_release_draft.yml @@ -9,11 +9,36 @@ on: workflow_dispatch: inputs: - version: - description: Current release/rc version + release_tag: + description: Tag matching the actual release candidate with the format polkadot-stableYYMM(-X)-rcX or polkadot-stableYYMM(-X) + required: true + type: string jobs: + check-synchronization: + uses: paritytech-release/sync-workflows/.github/workflows/check-syncronization.yml@main + + validate-inputs: + needs: [check-synchronization] + if: ${{ needs.check-synchronization.outputs.checks_passed }} == 'true' + runs-on: ubuntu-latest + outputs: + release_tag: ${{ steps.validate_inputs.outputs.release_tag }} + + steps: + - name: Checkout sources + uses: actions/checkout@d632683dd7b4114ad314bca15554477dd762a938 # v4.2.0 + + - name: Validate inputs + id: validate_inputs + run: | + . ./.github/scripts/common/lib.sh + + RELEASE_TAG=$(validate_stable_tag ${{ inputs.release_tag }}) + echo "release_tag=${RELEASE_TAG}" >> $GITHUB_OUTPUT + get-rust-versions: + needs: [validate-inputs] runs-on: ubuntu-latest outputs: rustc-stable: ${{ steps.get-rust-versions.outputs.stable }} @@ -23,181 +48,245 @@ jobs: RUST_STABLE_VERSION=$(curl -sS https://raw.githubusercontent.com/paritytech/scripts/master/dockerfiles/ci-unified/Dockerfile | grep -oP 'ARG RUST_STABLE_VERSION=\K[^ ]+') echo "stable=$RUST_STABLE_VERSION" >> $GITHUB_OUTPUT - # replace this job with fertch-runtimes-from-s3 - build-runtimes: - uses: "./.github/workflows/release-srtool.yml" - with: - excluded_runtimes: "asset-hub-rococo bridge-hub-rococo contracts-rococo coretime-rococo people-rococo rococo rococo-parachain substrate-test bp cumulus-test kitchensink minimal-template parachain-template penpal polkadot-test seedling shell frame-try sp solochain-template polkadot-sdk-docs-first" - build_opts: "--features on-chain-release-build" + # build-runtimes: + # needs: [validate-inputs] + # uses: "./.github/workflows/release-srtool.yml" + # with: + # excluded_runtimes: "asset-hub-rococo bridge-hub-rococo contracts-rococo coretime-rococo people-rococo rococo rococo-parachain substrate-test bp cumulus-test kitchensink minimal-template parachain-template penpal polkadot-test seedling shell frame-try sp solochain-template polkadot-sdk-docs-first" + # build_opts: "--features on-chain-release-build" - # replace this job with fetch-binaries-from-s3 - build-binaries: - runs-on: ubuntu-latest - strategy: - matrix: - # Tuples of [package, binary-name] - binary: [ [frame-omni-bencher, frame-omni-bencher], [staging-chain-spec-builder, chain-spec-builder] ] - steps: - - name: Checkout sources - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 + # # replace this job with fetch-binaries-from-s3 + # build-binaries: + # needs: [validate-inputs] + # runs-on: ubuntu-latest + # strategy: + # matrix: + # # Tuples of [package, binary-name] + # binary: [ [frame-omni-bencher, frame-omni-bencher], [staging-chain-spec-builder, chain-spec-builder] ] + # steps: + # - name: Checkout sources + # uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 - - name: Install protobuf-compiler - run: | - sudo apt update - sudo apt install -y protobuf-compiler + # - name: Install protobuf-compiler + # run: | + # sudo apt update + # sudo apt install -y protobuf-compiler - - name: Build ${{ matrix.binary[1] }} binary - run: | - cargo build --locked --profile=production -p ${{ matrix.binary[0] }} --bin ${{ matrix.binary[1] }} - target/production/${{ matrix.binary[1] }} --version + # - name: Build ${{ matrix.binary[1] }} binary + # run: | + # cargo build --locked --profile=production -p ${{ matrix.binary[0] }} --bin ${{ matrix.binary[1] }} + # target/production/${{ matrix.binary[1] }} --version - - name: Upload ${{ matrix.binary[1] }} binary - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - name: ${{ matrix.binary[1] }} - path: target/production/${{ matrix.binary[1] }} + # - name: Upload ${{ matrix.binary[1] }} binary + # uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + # with: + # name: ${{ matrix.binary[1] }} + # path: target/production/${{ matrix.binary[1] }} + # publish-release-draft: + # runs-on: ubuntu-latest + # needs: [ get-rust-versions, build-runtimes ] + # outputs: + # release_url: ${{ steps.create-release.outputs.html_url }} + # asset_upload_url: ${{ steps.create-release.outputs.upload_url }} + # steps: + # - name: Checkout + # uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 - publish-release-draft: - runs-on: ubuntu-latest - needs: [ get-rust-versions, build-runtimes ] - outputs: - release_url: ${{ steps.create-release.outputs.html_url }} - asset_upload_url: ${{ steps.create-release.outputs.upload_url }} - steps: - - name: Checkout - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 + # - name: Download artifacts + # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - - name: Download artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # - name: Prepare tooling + # run: | + # URL=https://github.com/chevdor/tera-cli/releases/download/v0.4.0/tera-cli_linux_amd64.deb + # wget $URL -O tera.deb + # sudo dpkg -i tera.deb - - name: Prepare tooling - run: | - URL=https://github.com/chevdor/tera-cli/releases/download/v0.4.0/tera-cli_linux_amd64.deb - wget $URL -O tera.deb - sudo dpkg -i tera.deb + # - name: Prepare draft + # id: draft + # env: + # RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json + # BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json + # COLLECTIVES_WESTEND_DIGEST: ${{ github.workspace}}/collectives-westend-runtime/collectives-westend-srtool-digest.json + # CORETIME_WESTEND_DIGEST: ${{ github.workspace}}/coretime-westend-runtime/coretime-westend-srtool-digest.json + # GLUTTON_WESTEND_DIGEST: ${{ github.workspace}}/glutton-westend-runtime/glutton-westend-srtool-digest.json + # PEOPLE_WESTEND_DIGEST: ${{ github.workspace}}/people-westend-runtime/people-westend-srtool-digest.json + # WESTEND_DIGEST: ${{ github.workspace}}/westend-runtime/westend-srtool-digest.json + # RELEASE_TAG: ${{ needs.validate-inputs.outputs.release_tag }} + # shell: bash + # run: | + # . ./.github/scripts/common/lib.sh - - name: Prepare draft - id: draft - env: - RUSTC_STABLE: ${{ needs.get-rust-versions.outputs.rustc-stable }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ASSET_HUB_WESTEND_DIGEST: ${{ github.workspace}}/asset-hub-westend-runtime/asset-hub-westend-srtool-digest.json - BRIDGE_HUB_WESTEND_DIGEST: ${{ github.workspace}}/bridge-hub-westend-runtime/bridge-hub-westend-srtool-digest.json - COLLECTIVES_WESTEND_DIGEST: ${{ github.workspace}}/collectives-westend-runtime/collectives-westend-srtool-digest.json - CORETIME_WESTEND_DIGEST: ${{ github.workspace}}/coretime-westend-runtime/coretime-westend-srtool-digest.json - GLUTTON_WESTEND_DIGEST: ${{ github.workspace}}/glutton-westend-runtime/glutton-westend-srtool-digest.json - PEOPLE_WESTEND_DIGEST: ${{ github.workspace}}/people-westend-runtime/people-westend-srtool-digest.json - WESTEND_DIGEST: ${{ github.workspace}}/westend-runtime/westend-srtool-digest.json - shell: bash - run: | - . ./.github/scripts/common/lib.sh + # export REF1=$(get_latest_release_tag) + # if [[ -z "$RELEASE_TAG" ]]; then + # export REF2="${{ github.ref_name }}" + # echo "REF2: ${REF2}" + # else + # export REF2="$RELEASE_TAG" + # echo "REF2: ${REF2}" + # fi + # echo "REL_TAG=$REF2" >> $GITHUB_ENV + # export VERSION=$(echo "$REF2" | sed -E 's/.*(stable[0-9]+).*$/\1/') - export REF1=$(get_latest_release_tag) - if [[ -z "${{ inputs.version }}" ]]; then - export REF2="${{ github.ref_name }}" - echo "REF2: ${REF2}" - else - export REF2="${{ inputs.version }}" - echo "REF2: ${REF2}" - fi - echo "REL_TAG=$REF2" >> $GITHUB_ENV - export VERSION=$(echo "$REF2" | sed -E 's/.*(stable[0-9]+).*$/\1/') - - ./scripts/release/build-changelogs.sh - - - name: Archive artifact context.json - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 - with: - name: release-notes-context - path: | - scripts/release/context.json - **/*-srtool-digest.json - - - name: Create draft release - id: create-release - uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ env.REL_TAG }} - release_name: Polkadot ${{ env.REL_TAG }} - body_path: ${{ github.workspace}}/scripts/release/RELEASE_DRAFT.md - draft: true - - publish-runtimes: - needs: [ build-runtimes, publish-release-draft ] - continue-on-error: true - runs-on: ubuntu-latest - strategy: - matrix: ${{ fromJSON(needs.build-runtimes.outputs.published_runtimes) }} + # ./scripts/release/build-changelogs.sh - steps: - - name: Checkout sources - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 + # - name: Archive artifact context.json + # uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + # with: + # name: release-notes-context + # path: | + # scripts/release/context.json + # **/*-srtool-digest.json - - name: Download artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # - name: Generate content write token for the release automation + # id: generate_write_token + # uses: actions/create-github-app-token@v1 + # with: + # app-id: ${{ vars.POLKADOT_SDK_RELEASE_RW_APP_ID }} + # private-key: ${{ secrets.POLKADOT_SDK_RELEASE_RW_APP_KEY }} + # owner: paritytech - - name: Get runtime info - env: - JSON: release-notes-context/${{ matrix.chain }}-runtime/${{ matrix.chain }}-srtool-digest.json - run: | - >>$GITHUB_ENV echo ASSET=$(find ${{ matrix.chain }}-runtime -name '*.compact.compressed.wasm') - >>$GITHUB_ENV echo SPEC=$(<${JSON} jq -r .runtimes.compact.subwasm.core_version.specVersion) + # - name: Create draft release + # id: create-release + # uses: actions/create-release@0cb9c9b65d5d1901c1f53e5e66eaf4afd303e70e # v1.1.4 + # env: + # # GHITHUB_TOKEN: ${{ steps.generate_write_token.outputs.token }} + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # with: + # tag_name: ${{ env.REL_TAG }} + # release_name: Polkadot ${{ env.REL_TAG }} + # body_path: ${{ github.workspace}}/scripts/release/RELEASE_DRAFT.md + # draft: true + # # owner: paritytech + # # repo: paritytech/polkadot-sdk - - name: Upload compressed ${{ matrix.chain }} v${{ env.SPEC }} wasm - if: ${{ matrix.chain != 'rococo-parachain' }} - uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }} - asset_path: ${{ env.ASSET }} - asset_name: ${{ matrix.chain }}_runtime-v${{ env.SPEC }}.compact.compressed.wasm - asset_content_type: application/wasm - - publish-binaries: - needs: [ publish-release-draft, build-binaries ] + # publish-runtimes: + # needs: [ build-runtimes, publish-release-draft ] + # continue-on-error: true + # runs-on: ubuntu-latest + # strategy: + # matrix: ${{ fromJSON(needs.build-runtimes.outputs.published_runtimes) }} + + # steps: + # - name: Checkout sources + # uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.0.0 + + # - name: Download artifacts + # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + + # - name: Get runtime info + # env: + # JSON: release-notes-context/${{ matrix.chain }}-runtime/${{ matrix.chain }}-srtool-digest.json + # run: | + # >>$GITHUB_ENV echo ASSET=$(find ${{ matrix.chain }}-runtime -name '*.compact.compressed.wasm') + # >>$GITHUB_ENV echo SPEC=$(<${JSON} jq -r .runtimes.compact.subwasm.core_version.specVersion) + + # - name: Generate content write token for the release automation + # id: generate_write_token + # uses: actions/create-github-app-token@v1 + # with: + # app-id: ${{ vars.POLKADOT_SDK_RELEASE_RW_APP_ID }} + # private-key: ${{ secrets.POLKADOT_SDK_RELEASE_RW_APP_KEY }} + # owner: paritytech + + # - name: Upload compressed ${{ matrix.chain }} v${{ env.SPEC }} wasm + # if: ${{ matrix.chain != 'rococo-parachain' }} + # uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # # GHITHUB_TOKEN: ${{ steps.generate_write_token.outputs.token }} + # with: + # upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }} + # asset_path: ${{ env.ASSET }} + # asset_name: ${{ matrix.chain }}_runtime-v${{ env.SPEC }}.compact.compressed.wasm + # asset_content_type: application/wasm + + # publish-binaries: + # needs: [ publish-release-draft, build-binaries ] + # continue-on-error: true + # runs-on: ubuntu-latest + # strategy: + # matrix: + # binary: [frame-omni-bencher, chain-spec-builder] + + # steps: + # - name: Download artifacts + # uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + # with: + # name: ${{ matrix.binary }} + + # - name: Generate content write token for the release automation + # id: generate_write_token + # uses: actions/create-github-app-token@v1 + # with: + # app-id: ${{ vars.POLKADOT_SDK_RELEASE_RW_APP_ID }} + # private-key: ${{ secrets.POLKADOT_SDK_RELEASE_RW_APP_KEY }} + # owner: paritytech + + # - name: Upload ${{ matrix.binary }} binary + # uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # # GHITHUB_TOKEN: ${{ steps.generate_write_token.outputs.token }} + # with: + # upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }} + # asset_path: ${{ github.workspace}}/${{ matrix.binary }} + # asset_name: ${{ matrix.binary }} + # asset_content_type: application/octet-stream + + publish-polkadot-artifacts: + # needs: [ publish-release-draft ] continue-on-error: true runs-on: ubuntu-latest strategy: matrix: - binary: [frame-omni-bencher, chain-spec-builder] + binary: [ polkadot, polkadot-parachain, polkadot-omni-node ] + target: [ x86_64-unknown-linux-gnu, aarch64-apple-darwin ] steps: - - name: Download artifacts - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: ${{ matrix.binary }} + - name: Fetch binaries from s3 based on version + run: | + . ./.github/scripts/common/lib.sh - - name: Upload ${{ matrix.binary }} binary - uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2 + VERSION="${{ needs.validate-inputs.outputs.release_tag }}" + fetch_release_artifacts_from_s3 ${{ matrix.binary }} ${{ matrix.target }} + + - name: Upload ${{ matrix.binary }} binary to release draft + # uses: actions/upload-release-asset@e8f9f06c4b078e705bd2ea027f0926603fc9b4d5 #v1.0.2 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }} - asset_path: ${{ github.workspace}}/${{ matrix.binary }} - asset_name: ${{ matrix.binary }} - asset_content_type: application/octet-stream + working-directory: ${{ github.workspace}}/release-artifacts/${{ matrix.target }}/${{ matrix.binary }} + run: | + for file in ${{ matrix.binary }} ${{ matrix.binary }}.asc ${{ matrix.binary }}.sha256; do + echo "Uploading $file to GH" + gh release upload ${{ needs.validate-inputs.outputs.release_tag }} $file + done - post_to_matrix: - runs-on: ubuntu-latest - needs: publish-release-draft - environment: release - strategy: - matrix: - channel: - - name: "Team: RelEng Internal" - room: '!GvAyzgCDgaVrvibaAF:parity.io' + # with: + # upload_url: ${{ needs.publish-release-draft.outputs.asset_upload_url }} + # asset_path: ${{ github.workspace}}/${{ matrix.binary }} + # asset_name: ${{ matrix.binary }} + # asset_content_type: application/octet-stream - steps: - - name: Send Matrix message to ${{ matrix.channel.name }} - uses: s3krit/matrix-message-action@70ad3fb812ee0e45ff8999d6af11cafad11a6ecf # v0.0.3 - with: - room_id: ${{ matrix.channel.room }} - access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }} - server: m.parity.io - message: | - **New version of polkadot tagged**: ${{ github.ref_name }}
- Draft release created: ${{ needs.publish-release-draft.outputs.release_url }} + # post_to_matrix: + # runs-on: ubuntu-latest + # needs: publish-release-draft + # environment: release + # strategy: + # matrix: + # channel: + # - name: "Team: RelEng Internal" + # room: '!GvAyzgCDgaVrvibaAF:parity.io' + + # steps: + # - name: Send Matrix message to ${{ matrix.channel.name }} + # uses: s3krit/matrix-message-action@70ad3fb812ee0e45ff8999d6af11cafad11a6ecf # v0.0.3 + # with: + # room_id: ${{ matrix.channel.room }} + # access_token: ${{ secrets.RELEASENOTES_MATRIX_V2_ACCESS_TOKEN }} + # server: m.parity.io + # message: | + # **New version of polkadot tagged**: ${{ github.ref_name }}
+ # Draft release created: ${{ needs.publish-release-draft.outputs.release_url }} diff --git a/.github/workflows/release-build-rc.yml b/.github/workflows/release-build-rc.yml index a43c2b282a8d..d5cf28379f51 100644 --- a/.github/workflows/release-build-rc.yml +++ b/.github/workflows/release-build-rc.yml @@ -14,7 +14,7 @@ on: - all release_tag: - description: Tag matching the actual release candidate with the format stableYYMM-rcX or stableYYMM + description: Tag matching the actual release candidate with the format polkadot-stableYYMM(-X)-rcX or polkadot-stableYYMM(-X) type: string jobs: