Merge branch 'master' into gang-min-cardinality

.github/workflows/build.yml

@@ -0,0 +1,56 @@
+name: Build
+
+on:
+  workflow_call:
+
+jobs:
+  prepare:
+    runs-on: ubuntu-22.04
+    if: github.repository_owner == 'armadaproject'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Go with Cache
+        uses: actions/setup-go@v4
+        with:
+         go-version: '1.20'
+
+      - name: Cache GOBIN
+        uses: actions/cache@v3
+        with:
+          path: /home/runner/go/bin
+          key: ${{ runner.os }}-gobin-${{ hashFiles('**/tools.yaml') }}
+
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+
+      - uses: goreleaser/goreleaser-action@v3
+        with:
+          distribution: goreleaser
+          version: v1.20.0
+          args: release  --snapshot --skip-sbom --skip-sign --clean
+        env:
+          DOCKER_REPO: "gresearch"
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
+          DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
+          DOCKER_BUILDX_CACHE_FROM: "type=gha"
+          DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max"
+
+      - name: Output full commit sha
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        run: echo "sha_full=$(git rev-parse HEAD)" >> $GITHUB_ENV
+
+      - name: Save Docker image tarballs
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        run: |
+          scripts/docker-save.sh -t ${{ env.sha_full }} -o /tmp/imgs
+
+      - name: Save Docker image tarballs as artifacts
+        if: github.event_name == 'push' && github.ref == 'refs/heads/master'
+        uses: actions/upload-artifact@v3
+        with:
+          name: armada-image-tarballs
+          path: /tmp/imgs

.github/workflows/ci.yml

@@ -32,13 +32,17 @@ jobs:
   test:
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     uses: ./.github/workflows/test.yml
+  build:
+    if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
+    uses: ./.github/workflows/build.yml
   # Virtual job that can be configured as a required check before a PR can be merged.
   all-required-checks-done:
     name: All required checks done
     if: github.event_name == 'schedule' || github.event_name == 'push' || github.event.pull_request.head.repo.id != github.event.pull_request.base.repo.id
     needs:
       - lint
       - test
+      - build
     runs-on: ubuntu-22.04
     steps:
       - uses: actions/github-script@v6
@@ -50,3 +54,4 @@ jobs:
             } else {
               core.setFailed('Some required checks failed');
             }
+

.github/workflows/release-rc.yml

@@ -49,33 +49,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Setup Golang with Cache
-        uses: magnetikonline/action-golang-cache@v4
-        with:
-          go-version: "1.20"
-
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v2
-
       - name: "Docker login"
         uses: "docker/login-action@v2"
         with:
           username: "${{ secrets.DOCKERHUB_USER }}"
           password: "${{ secrets.DOCKERHUB_PASS }}"
 
-      - name: "Run GoReleaser"
-        uses: "goreleaser/goreleaser-action@v4"
-        with:
-          distribution: "goreleaser"
-          version: v1.19.2
-          args: "-f ./.goreleaser.yml release --snapshot --skip-sbom --skip-sign --clean"
+      - name: Download Docker image tarballs artifact
+        run: gh run download ${{ github.event.workflow_run.id }} --name armada-image-tarballs
         env:
-          DOCKER_REPO: "gresearch"
-          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-          DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
-          DOCKER_BUILDX_CACHE_FROM: "type=gha"
-          DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max"
+          GH_TOKEN: ${{ github.token }}
 
       - name: Run Docker push script
-        run: ./scripts/docker-push.sh -t '${{ github.event.workflow_run.head_sha }}'
+        run: ./scripts/docker-push.sh --tag '${{ github.event.workflow_run.head_sha }}' --images-dir . --use-tarballs "true"

.github/workflows/test.yml

@@ -1,4 +1,4 @@
-name: Code Build and Tests
+name: Tests
 
 on:
   workflow_call:

scripts/common.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# This script defines common variables and functions for the other scripts.
+
+export docker_registry="gresearch"
+export image_names=(
+    "armada-bundle"
+    "armada-lookout-bundle"
+    "armada-full-bundle"
+    "armada-server"
+    "armada-executor"
+    "armada-fakeexecutor"
+    "armada-lookout-ingester"
+    "armada-lookout-ingester-v2"
+    "armada-lookout"
+    "armada-lookout-v2"
+    "armada-event-ingester"
+    "armada-scheduler"
+    "armada-scheduler-ingester"
+    "armada-binoculars"
+    "armada-jobservice"
+    "armadactl"
+)

scripts/docker-push.sh

@@ -1,38 +1,34 @@
 #!/bin/bash
 
-docker_registry="gresearch"
+
+if ! source "$(dirname "$0")/common.sh"; then
+    echo "::error ::failed to source common.sh"
+    exit 1
+fi
+
 docker_tag=""
-image_names=(
-    "armada-bundle"
-    "armada-lookout-bundle"
-    "armada-full-bundle"
-    "armada-server"
-    "armada-executor"
-    "armada-fakeexecutor"
-    "armada-lookout-ingester"
-    "armada-lookout-ingester-v2"
-    "armada-lookout"
-    "armada-lookout-v2"
-    "armada-event-ingester"
-    "armada-scheduler"
-    "armada-scheduler-ingester"
-    "armada-binoculars"
-    "armada-jobservice"
-    "armadactl"
-)
+use_tarballs=false
 
 print_usage() {
-    echo "Usage: $0 [-t|--tag <tag>] [-r|--registry <registry>]"
+    echo "Usage: $0 [-t|--tag <tag>] [-r|--registry <registry> -i|--images <images>]"
     echo ""
     echo "Options:"
-    echo "    -t|--tag       Docker tag (required)"
-    echo "    -r|--registry  Docker registry (default: 'gresearch')"
-    echo "    -h|--help      Display this help message"
+    echo "    -u|--use-tarballs  Directory with image tarballs to push"
+    echo "    -i|--images-dir  Directory with image tarballs to push"
+    echo "    -t|--tag         Docker tag (required)"
+    echo "    -r|--registry    Docker registry (default: '$docker_registry')"
+    echo "    -h|--help        Display this help message"
 }
 
 # parse command-line arguments
 while [[ $# -gt 0 ]]; do
     case "$1" in
+        -i|--images-dir)
+            images_dir=$2
+            images_dir=${images_dir%/}
+            shift
+            shift
+            ;;
         -t|--tag)
             docker_tag=$2
             shift
@@ -43,6 +39,11 @@ while [[ $# -gt 0 ]]; do
             shift
             shift
             ;;
+        -u|--use-tarballs)
+            use_tarballs=$2
+            shift
+            shift
+            ;;
         -h|--help)
             print_usage
             exit 0
@@ -60,26 +61,50 @@ if [ -z "$docker_tag" ]; then
     exit 1
 fi
 
+if [ "$use_tarballs" = true ]; then
+    if [ -z "$images_dir" ]; then
+        echo "::error ::tarball images dir must be provided with -i|--images-dir option"
+        exit 1
+    fi
+fi
+
 # iterate over image names, check existence and push them
 for image_name in "${image_names[@]}"; do
-    full_image_name="${docker_registry}/${image_name}:${docker_tag}"
-    echo "checking existence of $full_image_name..."
-    # Check if the image with the tag exists
-    if ! docker image inspect "$full_image_name" > /dev/null 2>&1; then
-        echo "::error ::image $full_image_name does not exist locally"
-        exit 1
+    echo "::group::validating $image_name..."
+    if [ "$use_tarballs" = true ]; then
+        tarball_image="${images_dir}/${image_name}.tar"
+        if [ ! -f "$tarball_image" ]; then
+            echo "::error ::image $tarball_image does not exist"
+            exit 1
+        fi
+    else
+        full_image_name="${docker_registry}/${image_name}:${docker_tag}"
+        echo "checking existence of $full_image_name..."
+        # Check if the image with the tag exists
+        if ! docker image inspect "$full_image_name" > /dev/null 2>&1; then
+            echo "::error ::image $full_image_name does not exist locally"
+            exit 1
+        fi
+        echo "::endgroup::"
     fi
 done
 
 echo "pushing Armada images to $docker_registry with tag $docker_tag..."
 
 # iterate over image names and push them
 for image_name in "${image_names[@]}"; do
+    echo "::group::pushing $image_name..."
     full_image_name="${docker_registry}/${image_name}:${docker_tag}"
+    if [ "$use_tarballs" = true ]; then
+        tarball_image="${images_dir}/${image_name}.tar"
+        echo "loading tarball image $tarball_image..."
+        docker load --input "$tarball_image"
+    fi
     echo "pushing $full_image_name..."
     docker push $full_image_name
     if [ $? -ne 0 ]; then
         echo "::error ::failed to push $full_image_name"
         exit 1
     fi
+    echo "::endgroup::"
 done

scripts/docker-save.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+if ! source "$(dirname "$0")/common.sh"; then
+    echo "::error ::failed to source common.sh"
+    exit 1
+fi
+
+docker_tag=""
+output_dir="."
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -t|--tag)
+            docker_tag=$2
+            shift
+            shift
+            ;;
+        -o|--output)
+            output_dir=$2
+            shift
+            shift
+            ;;
+    esac
+done
+
+# validate that docker_tag is provided
+if [ -z "$docker_tag" ]; then
+    echo "::error ::docker tag is must be provided with -t|--tag option"
+    exit 1
+fi
+
+# Check if output directory exists, if not create it
+if [[ ! -d $output_dir ]]; then
+    if ! mkdir -p "$output_dir"; then
+        echo "::error ::failed to create output directory $output_dir"
+        exit 1
+    fi
+fi
+
+for image_name in "${image_names[@]}"; do
+    output_tarball="$output_dir/$image_name.tar"
+    echo "::group::saving $docker_registry/$image_name:$docker_tag to $output_tarball"
+    docker save -o "$output_tarball" "$docker_registry/$image_name:$docker_tag"
+    echo "::endgroup::"
+done