ref(workflow): move most scripts to their own executables (#8005)

* ref(workflow): move most scripts to their own executable

* debug: JSON value

* fix(scripts): move remaining script to its own file

* fix(script): revert to the correct disk search logic

* fix(scripts)

* fix(scripts): use correct NETWORK with lowercase

* fix: typo

* fix(script): wrong variable assignment

* fix(script): use correct return values inside a function

* fix(script): fix value assigment

* test: debug

* fix(script): make disk conditions simpler

* fix(script): export variables to the `shell` executing the script

* fix(script): do not fail on expected unbound variables

* test: output

* fix(scripts): do not `echo` a variable more than once

* fix(scripts): typo

* docs(workflow): adds a description at the top of each file (#8009)

Co-authored-by: Marek <mail@marek.onl>
Co-authored-by: teor <teor@riseup.net>

---------

Co-authored-by: teor <teor@riseup.net>
Co-authored-by: Alfredo Garcia <oxarbitrage@gmail.com>
Co-authored-by: Marek <mail@marek.onl>

.github/workflows/cd-deploy-nodes-gcp.yml

@@ -1,6 +1,16 @@
 # Google Cloud node deployments and tests that run when Rust code or dependencies are modified,
 # but only on PRs from the ZcashFoundation/zebra repository.
-# (External PRs are tested/deployed by mergify.)
+# (External PRs are tested/deployed by mergify.) 
+#
+# 1. `versioning`: Extracts the major version from the release semver. Useful for segregating instances based on major versions.
+# 2. `build`: Builds a Docker image named `zebrad` with the necessary tags derived from Git.
+# 3. `test-configuration-file`: Validates Zebra using the default config with the latest version.
+# 4. `test-configuration-file-testnet`: Tests the Docker image for the testnet configuration.
+# 5. `test-zebra-conf-path`: Verifies Zebra with a custom Docker config file.
+# 6. `deploy-nodes`: Deploys Managed Instance Groups (MiGs) for Mainnet and Testnet. If triggered by main branch pushes, it always replaces the MiG. For releases, MiGs are replaced only if deploying the same major version; otherwise, a new major version is deployed.
+# 7. `deploy-instance`: Deploys a single node in a specified GCP zone for testing specific commits. Instances from this job aren't auto-replaced or deleted.
+#
+# The overall goal is to ensure that Zebra nodes are consistently deployed, tested, and managed on GCP.
 name: Deploy Nodes to GCP
 
 # Ensures that only one workflow task will run at a time. Previous deployments, if

.github/workflows/chore-delete-gcp-resources.yml

@@ -1,4 +1,13 @@
-# TODO: rename this action name and filename to Delete infra resources
+# This workflow is designed to delete old Google Cloud Platform (GCP) resources to save on costs.
+#
+# 1. Deletes specific instances in GCP older than a defined number of days.
+# 2. Deletes instance templates older than a set number of days.
+# 3. Deletes older disks not currently in use, with certain ones prefixed by commit hashes or "zebrad-".
+# 4. Deletes cache images from GCP, retaining a specified number of the latest images for certain types like zebrad checkpoint cache, zebrad tip cache, and lightwalletd + zebrad tip cache.
+# 5. Deletes unused artifacts from Google Artifact Registry older than a defined number of hours while retaining the latest few.
+#
+# It uses the gcloud CLI for most of its operations and also leverages specific GitHub Actions like the gcr-cleaner for deleting old images from the Google Artifact Registry.
+# The workflow is scheduled to run daily at 0700 UTC.
 name: Delete GCP resources
 
 on:
@@ -56,29 +65,11 @@ jobs:
       # so it can't be shell-quoted.
       - name: Delete old instances
         run: |
-          DELETE_BEFORE_DATE=$(date --date="$DELETE_INSTANCE_DAYS days ago" '+%Y%m%d')
-
-          IFS=$'\n'
-          INSTANCES=$(gcloud compute instances list --sort-by=creationTimestamp --filter="name~-[0-9a-f]{7,}$ AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME,ZONE)' | \
-                      sed 's/\(.*\)\t\(.*\)/\1 --zone=\2/')
-
-          for INSTANCE_AND_ZONE in $INSTANCES
-          do
-            IFS=$' '
-            gcloud compute instances delete --verbosity=info ${INSTANCE_AND_ZONE} --delete-disks=all || continue
-            IFS=$'\n'
-          done
-
+          ./.github/workflows/scripts/gcp-delete-old-instances.sh
       # Deletes all the instance templates older than $DELETE_AGE_DAYS days.
       - name: Delete old instance templates
         run: |
-          DELETE_BEFORE_DATE=$(date --date="$DELETE_AGE_DAYS days ago" '+%Y%m%d')
-          TEMPLATES=$(gcloud compute instance-templates list --sort-by=creationTimestamp --filter="name~-[0-9a-f]{7,}$ AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-
-          for TEMPLATE in $TEMPLATES
-          do
-            gcloud compute instance-templates delete "${TEMPLATE}" || continue
-          done
+          ./.github/workflows/scripts/gcp-delete-old-templates.sh
 
       # Deletes all mainnet and testnet disks older than $DELETE_AGE_DAYS days.
       #
@@ -89,31 +80,7 @@ jobs:
       # so it can't be shell-quoted.
       - name: Delete old disks
         run: |
-          DELETE_BEFORE_DATE=$(date --date="$DELETE_AGE_DAYS days ago" '+%Y%m%d')
-
-          IFS=$'\n'
-          # Disks created by PR jobs, and other jobs that use a commit hash
-          COMMIT_DISKS=$(gcloud compute disks list --sort-by=creationTimestamp --filter="name~-[0-9a-f]{7,}$ AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME,LOCATION,LOCATION_SCOPE)' | \
-                         sed 's/\(.*\)\t\(.*\)\t\(.*\)/\1 --\3=\2/')
-
-          for DISK_AND_LOCATION in $COMMIT_DISKS
-          do
-            IFS=$' '
-            gcloud compute disks delete --verbosity=info ${DISK_AND_LOCATION} || continue
-            IFS=$'\n'
-          done
-
-          IFS=$'\n'
-          # Disks created by managed instance groups, and other jobs that start with "zebrad-"
-          ZEBRAD_DISKS=$(gcloud compute disks list --sort-by=creationTimestamp --filter="name~^zebrad- AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME,LOCATION,LOCATION_SCOPE)' | \
-                         sed 's/\(.*\)\t\(.*\)\t\(.*\)/\1 --\3=\2/')
-
-          for DISK_AND_LOCATION in $ZEBRAD_DISKS
-          do
-            IFS=$' '
-            gcloud compute disks delete --verbosity=info ${DISK_AND_LOCATION} || continue
-            IFS=$'\n'
-          done
+          ./.github/workflows/scripts/gcp-delete-old-disks.sh
 
       # Deletes mainnet and testnet cache images older than $DELETE_AGE_DAYS days.
       #
@@ -125,108 +92,9 @@ jobs:
       #
       # TODO:
       # - refactor out repeated shell script code
-      - name: Delete old cache disks
+      - name: Delete old cache images
         run: |
-          DELETE_BEFORE_DATE=$(date --date="$DELETE_AGE_DAYS days ago" '+%Y%m%d')
-
-          # As of April 2023, these disk names look like:
-          # zebrad-cache-6039-merge-62c8ecc-v25-mainnet-checkpoint-053559
-          #
-          # Mainnet zebrad checkpoint
-          ZEBRAD_MAINNET_CHECKPOINT_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^zebrad-cache-.*-mainnet-checkpoint AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $ZEBRAD_MAINNET_CHECKPOINT_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
-
-          # Testnet zebrad checkpoint
-          ZEBRAD_TESTNET_CHECKPOINT_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^zebrad-cache-.*-testnet-checkpoint AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $ZEBRAD_TESTNET_CHECKPOINT_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
-
-          # As of April 2023, these disk names look like:
-          # zebrad-cache-6556-merge-a2ca4de-v25-mainnet-tip(-u)?-140654
-          #
-          # Mainnet zebrad tip
-          ZEBRAD_MAINNET_TIP_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^zebrad-cache-.*-mainnet-tip AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $ZEBRAD_MAINNET_TIP_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
-
-          # Testnet zebrad tip
-          ZEBRAD_TESTNET_TIP_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^zebrad-cache-.*-testnet-tip AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $ZEBRAD_TESTNET_TIP_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
-
-          # As of April 2023, these disk names look like:
-          # lwd-cache-main-fb3fec0-v25-mainnet-tip(-u)?-061314
-          #
-          # Mainnet lightwalletd tip
-          LWD_MAINNET_TIP_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^lwd-cache-.*-mainnet-tip AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $LWD_MAINNET_TIP_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
-
-          # Testnet lightwalletd tip
-          LWD_TESTNET_TIP_IMAGES=$(gcloud compute images list --sort-by=~creationTimestamp --filter="name~^lwd-cache-.*-testnet-tip AND creationTimestamp < $DELETE_BEFORE_DATE" --format='value(NAME)')
-          KEPT_IMAGES=0
-          for IMAGE in $LWD_TESTNET_TIP_IMAGES
-          do
-            if [[ "$KEPT_IMAGES" -lt "$KEEP_LATEST_IMAGE_COUNT" ]];
-            then
-              KEPT_IMAGES=$((KEPT_IMAGES+1))
-              echo "Keeping image $KEPT_IMAGES named $IMAGE"
-              continue
-            fi
-
-            gcloud compute images delete "${IMAGE}" || continue
-          done
+          ./.github/workflows/scripts/gcp-delete-old-cache-images.sh
 
   # We're using a generic approach here, which allows multiple registries to be included,
   # even those not related to GCP. Enough reason to create a separate job.

.github/workflows/chore-project-management.yml

@@ -1,6 +1,12 @@
+# This workflow manages the automatic addition of new issues to specific GitHub projects.
+#
+# 1. Newly opened issues are added to the "Zebra Backlog" Github project.
+# 2. They are also added to the "ZF Engineering Backlog" Github project.
+#
+# The action makes use of the `add-to-project` action and requires a Github token
+# (currently sourced from secrets) to authenticate and perform the addition.
 name: Add new issues to GitHub projects
 
-# Configuration for automatically adding issues to various Github projects for Project Management purposes
 on:
   issues:
     types:

.github/workflows/ci-build-crates.yml

@@ -1,3 +1,13 @@
+# This workflow facilitates the individual building of Rust crates present in the repository.
+# 1. A matrix is generated dynamically to identify each crate in the repository.
+# 2. This matrix is checked for validity.
+# 3. Each identified crate undergoes three build processes:
+#    - With no features.
+#    - With the default features.
+#    - With all the features enabled.
+# 4. In case of build failures outside of pull requests, an issue is either opened or updated
+#    in the repository to report the failure.
+# Throughout the workflow, various setup steps ensure the correct environment and tools are present.
 name: Build crates individually
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/ci-coverage.yml

@@ -1,3 +1,11 @@
+# This workflow calculates the test coverage for the Rust codebase.
+# 1. The code is checked out.
+# 2. Rust with the stable toolchain, minimal profile, and llvm-tools-preview component is set up.
+# 3. Necessary tools like 'cargo-llvm-cov' are installed.
+# 4. Proptest is minimized for efficient coverage test runs.
+# 5. Tests are run without producing a report to gather coverage information.
+# 6. A coverage report (lcov format) is generated based on the gathered information.
+# 7. Finally, this report is uploaded to Codecov for visualization and analysis.
 name: Coverage
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/ci-integration-tests-gcp.yml

@@ -1,5 +1,8 @@
 # Google Cloud integration tests that run when Rust code or dependencies are modified,
 # but only on PRs from the ZcashFoundation/zebra repository. (External PRs are tested by mergify.)
+#
+# Specific conditions and dependencies are set for each job to ensure they are executed in the correct sequence and under the right circumstances.
+# Each test has a description of the conditions under which it runs.
 name: Integration Tests on GCP
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/ci-lint.yml

@@ -1,3 +1,10 @@
+# This workflow conducts various linting checks for a Rust-based project.
+# 1. Determines if Rust or workflow files have been modified.
+# 2. Runs the Clippy linter on Rust files, producing annotations and failing on warnings.
+# 3. Ensures Rust code formatting complies with 'rustfmt' standards.
+# 4. Lints GitHub Actions workflow files for common issues.
+# 5. Checks for common spelling errors in the codebase.
+# The workflow is designed to maintain code quality and consistency, running checks conditionally based on the changed files.
 name: Lint
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/ci-unit-tests-docker.yml

@@ -1,5 +1,16 @@
 # Google Cloud unit tests that run when Rust code or dependencies are modified,
 # but only on PRs from the ZcashFoundation/zebra repository. (External PRs are tested by mergify.)
+#
+# This workflow is designed for running various unit tests within Docker containers.
+# Jobs:
+# 1. Builds a Docker image for tests, adaptable to the specified network (Mainnet or Testnet).
+# 2. 'test-all': Executes all Zebra tests, including normally ignored ones, in a Docker environment.
+# 3. 'test-fake-activation-heights': Runs state tests with fake activation heights, isolating its build products.
+# 4. 'test-empty-sync': Tests Zebra's ability to sync and checkpoint from an empty state.
+# 5. 'test-lightwalletd-integration': Validates integration with 'lightwalletd' starting from an empty state.
+# 6. 'test-configuration-file': Assesses the default Docker configuration for Zebra.
+# 7. 'test-configuration-file-testnet': Checks the Docker image reconfiguration for the Testnet.
+# 8. 'test-zebra-conf-path': Tests Zebra using a custom Docker configuration.
 name: Docker Unit Tests
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/ci-unit-tests-os.yml

@@ -1,3 +1,9 @@
+# This workflow performs unit tests across different operating systems and Rust versions. It includes steps for:
+# - Testing on Ubuntu and macOS with stable and beta Rust toolchains.
+# - Installing Zebra from the lockfile without cache on Ubuntu.
+# - Verifying that Cargo.lock is up-to-date with Cargo.toml changes.
+# - Running cargo-deny checks for dependencies.
+# - Checking for unused dependencies in the code.
 name: Multi-OS Unit Tests
 
 # Ensures that only one workflow task will run at a time. Previous builds, if

.github/workflows/docs-deploy-firebase.yml

@@ -1,5 +1,9 @@
 # Google Cloud docs updates that run when docs, Rust code, or dependencies are modified,
 # but only on PRs from the ZcashFoundation/zebra repository. (External PRs are deployed by mergify.)
+
+# - Builds and deploys Zebra Book Docs using mdBook, setting up necessary tools and deploying to Firebase.
+# - Compiles and deploys external documentation, setting up Rust with the beta toolchain and default profile, building the docs, and deploying them to Firebase.
+# - Assembles and deploys internal documentation with similar steps, including private items in the documentation, and deploys to Firebase.
 name: Docs
 
 # Ensures that only one workflow task will run at a time. Previous deployments, if

.github/workflows/manual-zcashd-deploy.yml

@@ -1,3 +1,10 @@
+# This workflow is designed for manually deploying zcashd nodes to Google Cloud Platform (GCP) based on user inputs.
+# - Allows selection of network type (Mainnet or Testnet) and instance group size.
+# - Converts network name to lowercase to comply with GCP labeling requirements.
+# - Authenticates with Google Cloud using provided credentials.
+# - Creates a GCP instance template from a container image of zcashd.
+# - Checks if the specified instance group already exists.
+# - Depending on the existence check, either creates a new managed instance group or updates the existing one with the new template.
 name: Zcashd Manual Deploy
 
 on:

.github/workflows/release-crates-io.yml

@@ -93,26 +93,9 @@ jobs:
       #
       # These steps should be kept up to date with the release checklist.
       #
-      # TODO: move these steps into a script which is run in the release checklist and CI
       - name: Crate release dry run
         run: |
-          set -ex
-          git config --global user.email "release-tests-no-reply@zfnd.org"
-          git config --global user.name "Automated Release Test"
-          # This script must be the same as:
-          # https://github.com/ZcashFoundation/zebra/blob/main/.github/PULL_REQUEST_TEMPLATE/release-checklist.md#update-crate-versions
-          # with an extra `--no-confirm` argument for non-interactive testing.
-          cargo release version --verbose --execute --no-confirm --allow-branch '*' --workspace --exclude zebrad beta
-          cargo release version --verbose --execute --no-confirm --allow-branch '*' --package zebrad patch
-          cargo release replace --verbose --execute --no-confirm --allow-branch '*' --package zebrad
-          cargo release commit --verbose --execute --no-confirm --allow-branch '*'
-          # Check the release will work using a dry run
-          #
-          # Workaround unpublished dependency version errors by skipping those crates:
-          # https://github.com/crate-ci/cargo-release/issues/691
-          #
-          # TODO: check all crates after fixing these errors
-          cargo release publish --verbose --dry-run --allow-branch '*' --workspace --exclude zebra-consensus --exclude zebra-utils --exclude zebrad
+          ./.github/workflows/scripts/release-crates-dry-run.sh
 
   # TODO: actually do the release here
   #release-crates:

.github/workflows/release-drafter.yml

@@ -1,6 +1,10 @@
-# Creates a draft release with all the PR names since the last release.
+# This workflow automates the creation and updating of draft releases. It compiles PR titles into the draft release notes.
 # https://github.com/ZcashFoundation/zebra/releases
 #
+# - Updates the draft release upon each merge into 'main'.
+# - Utilizes the release-drafter GitHub Action to accumulate PR titles since the last release into a draft release note.
+# - Suitable permissions are set for creating releases and handling pull requests.
+#
 # Workflow is based on:
 # https://github.com/marketplace/actions/release-drafter#usage
 name: Release Drafter