From bb2bf8387e2f29251c2e54453ec44c43c5e21eae Mon Sep 17 00:00:00 2001 From: Dejan Zele Pejchev Date: Thu, 21 Sep 2023 17:28:03 +0200 Subject: [PATCH] feat: separate release CI (#2966) * feat: separate release CI * rename prepare-release to build * Update ci.yml --------- Co-authored-by: Mohamed Abdelfatah <39927413+Mo-Fatah@users.noreply.github.com> Co-authored-by: Adam McArthur <46480158+Sharpz7@users.noreply.github.com> --- .github/workflows/build.yml | 56 +++++++++++++++++++++ .github/workflows/ci.yml | 5 ++ .github/workflows/release-rc.yml | 25 ++-------- .github/workflows/test.yml | 2 +- scripts/common.sh | 23 +++++++++ scripts/docker-push.sh | 83 +++++++++++++++++++++----------- scripts/docker-save.sh | 45 +++++++++++++++++ 7 files changed, 188 insertions(+), 51 deletions(-) create mode 100644 .github/workflows/build.yml create mode 100755 scripts/common.sh create mode 100755 scripts/docker-save.sh diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..20d2e7ff19f --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,56 @@ +name: Build + +on: + workflow_call: + +jobs: + prepare: + runs-on: ubuntu-22.04 + if: github.repository_owner == 'armadaproject' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go with Cache + uses: actions/setup-go@v4 + with: + go-version: '1.20' + + - name: Cache GOBIN + uses: actions/cache@v3 + with: + path: /home/runner/go/bin + key: ${{ runner.os }}-gobin-${{ hashFiles('**/tools.yaml') }} + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v2 + + - uses: goreleaser/goreleaser-action@v3 + with: + distribution: goreleaser + version: v1.20.0 + args: release --snapshot --skip-sbom --skip-sign --clean + env: + DOCKER_REPO: "gresearch" + GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" + DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}" + DOCKER_BUILDX_CACHE_FROM: "type=gha" + DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + + - name: Output full commit sha + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + run: echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_ENV + + - name: Save Docker image tarballs + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + run: | + scripts/docker-save.sh -t ${{ env.sha_full }} -o /tmp/imgs + + - name: Save Docker image tarballs as artifacts + if: github.event_name == 'push' && github.ref == 'refs/heads/master' + uses: actions/upload-artifact@v3 + with: + name: armada-image-tarballs + path: /tmp/imgs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9851e9028e9..0b5ac50bd82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,9 @@ jobs: test: if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id uses: ./.github/workflows/test.yml + build: + if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id + uses: ./.github/workflows/build.yml # Virtual job that can be configured as a required check before a PR can be merged. all-required-checks-done: name: All required checks done @@ -39,6 +42,7 @@ jobs: needs: - lint - test + - build runs-on: ubuntu-22.04 steps: - uses: actions/github-script@v6 @@ -50,3 +54,4 @@ jobs: } else { core.setFailed('Some required checks failed'); } + diff --git a/.github/workflows/release-rc.yml b/.github/workflows/release-rc.yml index de70a33b22b..f220c7b2bf2 100644 --- a/.github/workflows/release-rc.yml +++ b/.github/workflows/release-rc.yml @@ -49,33 +49,16 @@ jobs: with: fetch-depth: 0 - - name: Setup Golang with Cache - uses: magnetikonline/action-golang-cache@v4 - with: - go-version: "1.20" - - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - - name: "Docker login" uses: "docker/login-action@v2" with: username: "${{ secrets.DOCKERHUB_USER }}" password: "${{ secrets.DOCKERHUB_PASS }}" - - name: "Run GoReleaser" - uses: "goreleaser/goreleaser-action@v4" - with: - distribution: "goreleaser" - version: v1.19.2 - args: "-f ./.goreleaser.yml release --snapshot --skip-sbom --skip-sign --clean" + - name: Download Docker image tarballs artifact + run: gh run download ${{ github.event.workflow_run.id }} --name armada-image-tarballs env: - DOCKER_REPO: "gresearch" - GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}" - DOCKER_BUILDX_CACHE_FROM: "type=gha" - DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max" + GH_TOKEN: ${{ github.token }} - name: Run Docker push script - run: ./scripts/docker-push.sh -t '${{ github.event.workflow_run.head_sha }}' + run: ./scripts/docker-push.sh --tag '${{ github.event.workflow_run.head_sha }}' --images-dir . --use-tarballs "true" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 887c3658836..3db81155cef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: Code Build and Tests +name: Tests on: workflow_call: diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100755 index 00000000000..1a83d8311b4 --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script defines common variables and functions for the other scripts. + +export docker_registry="gresearch" +export image_names=( + "armada-bundle" + "armada-lookout-bundle" + "armada-full-bundle" + "armada-server" + "armada-executor" + "armada-fakeexecutor" + "armada-lookout-ingester" + "armada-lookout-ingester-v2" + "armada-lookout" + "armada-lookout-v2" + "armada-event-ingester" + "armada-scheduler" + "armada-scheduler-ingester" + "armada-binoculars" + "armada-jobservice" + "armadactl" +) diff --git a/scripts/docker-push.sh b/scripts/docker-push.sh index 920c6384d19..7756f79954c 100755 --- a/scripts/docker-push.sh +++ b/scripts/docker-push.sh @@ -1,38 +1,34 @@ #!/bin/bash -docker_registry="gresearch" + +if ! source "$(dirname "$0")/common.sh"; then + echo "::error ::failed to source common.sh" + exit 1 +fi + docker_tag="" -image_names=( - "armada-bundle" - "armada-lookout-bundle" - "armada-full-bundle" - "armada-server" - "armada-executor" - "armada-fakeexecutor" - "armada-lookout-ingester" - "armada-lookout-ingester-v2" - "armada-lookout" - "armada-lookout-v2" - "armada-event-ingester" - "armada-scheduler" - "armada-scheduler-ingester" - "armada-binoculars" - "armada-jobservice" - "armadactl" -) +use_tarballs=false print_usage() { - echo "Usage: $0 [-t|--tag ] [-r|--registry ]" + echo "Usage: $0 [-t|--tag ] [-r|--registry -i|--images ]" echo "" echo "Options:" - echo " -t|--tag Docker tag (required)" - echo " -r|--registry Docker registry (default: 'gresearch')" - echo " -h|--help Display this help message" + echo " -u|--use-tarballs Directory with image tarballs to push" + echo " -i|--images-dir Directory with image tarballs to push" + echo " -t|--tag Docker tag (required)" + echo " -r|--registry Docker registry (default: '$docker_registry')" + echo " -h|--help Display this help message" } # parse command-line arguments while [[ $# -gt 0 ]]; do case "$1" in + -i|--images-dir) + images_dir=$2 + images_dir=${images_dir%/} + shift + shift + ;; -t|--tag) docker_tag=$2 shift @@ -43,6 +39,11 @@ while [[ $# -gt 0 ]]; do shift shift ;; + -u|--use-tarballs) + use_tarballs=$2 + shift + shift + ;; -h|--help) print_usage exit 0 @@ -60,14 +61,31 @@ if [ -z "$docker_tag" ]; then exit 1 fi +if [ "$use_tarballs" = true ]; then + if [ -z "$images_dir" ]; then + echo "::error ::tarball images dir must be provided with -i|--images-dir option" + exit 1 + fi +fi + # iterate over image names, check existence and push them for image_name in "${image_names[@]}"; do - full_image_name="${docker_registry}/${image_name}:${docker_tag}" - echo "checking existence of $full_image_name..." - # Check if the image with the tag exists - if ! docker image inspect "$full_image_name" > /dev/null 2>&1; then - echo "::error ::image $full_image_name does not exist locally" - exit 1 + echo "::group::validating $image_name..." + if [ "$use_tarballs" = true ]; then + tarball_image="${images_dir}/${image_name}.tar" + if [ ! -f "$tarball_image" ]; then + echo "::error ::image $tarball_image does not exist" + exit 1 + fi + else + full_image_name="${docker_registry}/${image_name}:${docker_tag}" + echo "checking existence of $full_image_name..." + # Check if the image with the tag exists + if ! docker image inspect "$full_image_name" > /dev/null 2>&1; then + echo "::error ::image $full_image_name does not exist locally" + exit 1 + fi + echo "::endgroup::" fi done @@ -75,11 +93,18 @@ echo "pushing Armada images to $docker_registry with tag $docker_tag..." # iterate over image names and push them for image_name in "${image_names[@]}"; do + echo "::group::pushing $image_name..." full_image_name="${docker_registry}/${image_name}:${docker_tag}" + if [ "$use_tarballs" = true ]; then + tarball_image="${images_dir}/${image_name}.tar" + echo "loading tarball image $tarball_image..." + docker load --input "$tarball_image" + fi echo "pushing $full_image_name..." docker push $full_image_name if [ $? -ne 0 ]; then echo "::error ::failed to push $full_image_name" exit 1 fi + echo "::endgroup::" done diff --git a/scripts/docker-save.sh b/scripts/docker-save.sh new file mode 100755 index 00000000000..056bd0a342d --- /dev/null +++ b/scripts/docker-save.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +if ! source "$(dirname "$0")/common.sh"; then + echo "::error ::failed to source common.sh" + exit 1 +fi + +docker_tag="" +output_dir="." + +while [[ $# -gt 0 ]]; do + case "$1" in + -t|--tag) + docker_tag=$2 + shift + shift + ;; + -o|--output) + output_dir=$2 + shift + shift + ;; + esac +done + +# validate that docker_tag is provided +if [ -z "$docker_tag" ]; then + echo "::error ::docker tag is must be provided with -t|--tag option" + exit 1 +fi + +# Check if output directory exists, if not create it +if [[ ! -d $output_dir ]]; then + if ! mkdir -p "$output_dir"; then + echo "::error ::failed to create output directory $output_dir" + exit 1 + fi +fi + +for image_name in "${image_names[@]}"; do + output_tarball="$output_dir/$image_name.tar" + echo "::group::saving $docker_registry/$image_name:$docker_tag to $output_tarball" + docker save -o "$output_tarball" "$docker_registry/$image_name:$docker_tag" + echo "::endgroup::" +done