Build improvements and paying down related tech debt (#189)

* Fix CI cache config to fix slow runs

* Replace ABI build crates with cargo build script

* Replace Rust proto build crates with build script step

* Remove workspace, make steward crate the repo root

* Fix build caching, combine Rust CI steps into one job

* Reduce docker image size

* Speed up and consolodate proto documentation generation

* Move scripts into a designated folder

* Update README

* Add missed doc file

* Fix Makefile bugs after changes

* Fix linting issues with proto directory structure and linting warnings (ContractCall -> ContractCallService)

* Make code changes for the service name change

* Remove sommelier submodule. Include Go proto and API doc build in build.rs

* fmt

* Use steward image artifact instead of repo push in integration test CI

* Fix proto gen bug

* fmt

* Review items

* Add missed review item

* Add missing abi file

* Fix abi and proto gen inconsistencies

* Review items

* Move go proto build back into Makefile

* Remove API docs build from build.rs

* Clippy warning in build.rs

* Correction round after rebasing main

* Fix doc folder after rebase

* fmt

* Change rust CI cache key to only root Cargo.lock since no others exist

* Add PR number suffix to integration test CI artifact

* Bump proto to v4

* Include v4 proto binding as module

* Fix naming/pathing for artifact upload/download

* Make hardhat image build to artifact in CI. Fix path bug

* Fix another CI bug

* Fix image naming

* debugging ci

* Correct env var used for hardhat image in CI

* reference by tag

.github/workflows/automated_release.yml

@@ -24,18 +24,19 @@ jobs:
       - name: checkout-code
         uses: actions/checkout@v2
       - name: "Check version"
-        run: ./check_version.sh
+        run: scripts/check_version.sh
       # Rust Cache
       - name: rust-cache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
+        id: rust-cache
         with:
           path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            steward/target/
-          key: ${{ runner.os }}-cargo-${{ hashFiles('steward/Cargo.lock') }}
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
+          key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
       # Install OpenSSL dependencies
       - name: install-packages
         run: sudo apt-get install -y libssl-dev musl-tools

.github/workflows/integration_tests.yml

@@ -10,10 +10,16 @@ on:
 
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}-steward
+  IMAGE_NAME: steward-steward
+  PR_NUMBER: ${{ github.event.number }}
   HARDHAT_IMAGE: ghcr.io/peggyjv/steward-hardhat
+  HARDHAT_IMAGE_NAME: steward-hardhat
   ORCHESTRATOR_IMAGE_NAME: ghcr.io/peggyjv/gravity-bridge-orchestrator:main
   SOMMELIER_IMAGE: ghcr.io/peggyjv/sommelier-sommelier:main
+  STEWARD_ARTIFACT: steward_steward
+  HARDHAT_ARTIFACT: steward_hardhat
+  UPLOAD_DIR: /tmp/uploaded_artifacts
+  DOWNLOAD_DIR: /tmp/downloaded_artifacts
 
 jobs:
   hardhat-build:
@@ -46,17 +52,25 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.HARDHAT_IMAGE }}
+      - name: Make upload dir
+        run: mkdir -p ${{ env.UPLOAD_DIR }}
       - name: build-and-push
         uses: docker/build-push-action@v2
         with:
           context: integration_tests/ethereum
           file: integration_tests/ethereum/Dockerfile
-          push: true
+          push: false
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
           builder: ${{ steps.buildx.outputs.name }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+          outputs: type=docker,dest=${{ env.UPLOAD_DIR }}/${{ env.HARDHAT_ARTIFACT }}
+      - name: Make hardhat image artifact available to other jobs
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.HARDHAT_IMAGE_NAME }}
+          path: ${{ env.UPLOAD_DIR }}/${{ env.HARDHAT_ARTIFACT }}
+          retention-days: 1
       - name: Move Docker cache
         run: |
           rm -rf /tmp/.buildx-cache
@@ -81,27 +95,24 @@ jobs:
           key: ${{ runner.os }}-multi-buildx-${{ github.sha }}
           restore-keys: |
             ${{ runner.os }}-multi-buildx
-      - name: container-login
-        uses: docker/login-action@v1
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: labels
-        id: meta
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      - name: Make upload dir
+        run: mkdir -p ${{ env.UPLOAD_DIR }}
       - name: build-and-push
         uses: docker/build-push-action@v2
         with:
           file: Dockerfile
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
+          push: false
+          tags: steward:prebuilt
           builder: ${{ steps.buildx.outputs.name }}
           cache-from: type=local,src=/tmp/.buildx-cache
           cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new
+          outputs: type=docker,dest=${{ env.UPLOAD_DIR }}/${{ env.STEWARD_ARTIFACT }}
+      - name: Make Steward artifact available to other jobs
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.IMAGE_NAME }}
+          path: ${{ env.UPLOAD_DIR }}/${{ env.STEWARD_ARTIFACT }}
+          retention-days: 1
       - name: Move Docker cache
         run: |
           rm -rf /tmp/.buildx-cache
@@ -145,32 +156,34 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.SOMMELIER_IMAGE }}
-      - name: docker hardhat metadata
-        id: meta-hardhat
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.HARDHAT_IMAGE }}
-      - name: docker steward metadata
-        id: meta-steward
-        uses: docker/metadata-action@v3
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
       - name: pull sommelier image
         run: docker pull ${{ env.SOMMELIER_IMAGE }}
       - name: rename sommelier image
         run: docker tag ${{ env.SOMMELIER_IMAGE }} sommelier:prebuilt
-      - name: pull hardhat image
-        run: docker pull ${{ steps.meta-hardhat.outputs.tags }}
-      - name: rename hardhat image
-        run: docker tag ${{ steps.meta-hardhat.outputs.tags }} ethereum:prebuilt
       - name: pull orchestrator image
         run: docker pull ${{ env.ORCHESTRATOR_IMAGE_NAME }}
       - name: rename orchestrator image
         run: docker tag ${{ env.ORCHESTRATOR_IMAGE_NAME }} orchestrator:prebuilt
-      - name: pull steward image
-        run: docker pull ${{ steps.meta-steward.outputs.tags }}
-      - name: rename steward image
-        run: docker tag ${{ steps.meta-steward.outputs.tags }} steward:prebuilt
+      - name: Make download dir
+        run: mkdir -p ${{ env.DOWNLOAD_DIR }}
+      - name: Get hardhat artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.HARDHAT_IMAGE_NAME }}
+          path: ${{ env.DOWNLOAD_DIR }}/
+      # The image is already tagged at this point by the hardhat-build job
+      - name: Load hardhat image
+        run: docker load --input ${{ env.DOWNLOAD_DIR }}/${{ env.HARDHAT_ARTIFACT }}
+      - name: Rename hardhat image
+        run: docker tag ${{ env.HARDHAT_IMAGE }}:pr-${{ env.PR_NUMBER }} ethereum:prebuilt
+      - name: Get steward artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ env.IMAGE_NAME }}
+          path: ${{ env.DOWNLOAD_DIR }}/
+      # The image is already tagged at this point by the steward-build job
+      - name: Load steward image
+        run: docker load --input ${{ env.DOWNLOAD_DIR }}/${{ env.STEWARD_ARTIFACT }}
       - name: clean the slate
         run: make e2e_clean_slate
       - name: build go test binary
@@ -180,6 +193,6 @@ jobs:
         env:
           E2E_SKIP_CLEANUP: true
       - name: Clean up docker images
-        run: ./clean_slate.sh
+        run: scripts/clean_slate.sh
       - name: Remove testnet tempdir
         run: sudo rm -rf somm-e2e-testnet*

.github/workflows/rust.yml

@@ -14,17 +14,17 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Rust caches
         uses: actions/cache@v2
         id: rust-cache
         with:
           path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
           key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
       - name: Run steward acceptance tests
         run: cargo test --all --verbose
@@ -33,17 +33,17 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: Set up Rust caches
         uses: actions/cache@v2
         id: rust-cache
         with:
           path: |
-            ~/.cargo/bin/
-            ~/.cargo/registry/index/
-            ~/.cargo/registry/cache/
-            ~/.cargo/git/db/
-            target/
+            ~/.cargo/bin
+            ~/.cargo/registry/index
+            ~/.cargo/registry/cache
+            ~/.cargo/git/db
+            target
           key: ${{ runner.os }}-cargo-${{ hashFiles('Cargo.lock') }}
       - name: Run Cargo build
         run: cargo build
@@ -53,11 +53,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
+        uses: actions/checkout@v3
       - name: Add rustfmt
         run: rustup component add rustfmt
       - name: Run fmt
@@ -68,11 +64,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout branch
-        uses: actions/checkout@v2
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
+        uses: actions/checkout@v3
       - name: Add clippy
         run: rustup component add clippy
       - name: Run clippy