[.github] - refacto: specify deploy region (#9276)

* [k8s] - feature: support regional specification for cloud builds

 - Added a --region flag to specify the region during cloud builds
 - Cloud build configuration now requires the --region parameter to operate correctly
 - Docker image tags are now constructed using the provided region value

* [.github] - feature: allow regional deployment options for front edge workflow

 - Added input fields for specifying deployment regions with default region as 'us-central1'
 - Split the build-and-deploy job into separate notify, build, and deploy jobs for better modularity
 - Implemented matrix strategy for build job to handle multiple regions
 - Adjusted notify and deployment steps to use dynamic region input and pass messages correctly with thread timestamps

* [github/workflows] - feature: specify deployment region for services

 - Add the `--region` parameter with `us-central1` to deployment commands across multiple GitHub Actions workflows
 - Ensure consistency and correct regional deployment for the services `alerting-temporal`, `connectors`, `core`, `dante`, `front-qa`, `front`, `nginx`, `oauth`, `prodbox`, and `viz`

* [.github] - refactor: streamline deployment workflow for front-edge

 - Removed regional deployment options in favor of a single fixed region deployment
 - Merged build and deployment steps into a single job to simplify the workflow
 - Updated Slack notification steps to reflect the changes in job structure
 - Set the deployment region to 'us-central1' as a default without options
 - Removed redundant checks-outs and extractions of the git short SHA
 - Fixed Slack notification to use the thread timestamp from the current job

* [.github/workflows] - feature: refactor deploy-prodbox workflow with new regions and notification steps

 - Add input for regions with a choice type enabling selection of deployment regions
 - Implement notification step that outputs thread timestamp for Slack notification
 - Separate build and deployment jobs for better modularity
 - Introduce a condition to set regions based on user input, supporting 'all' option
 - Modify build step to iterate over regions and include region in build step name
 - Update slack-notify step to use thread timestamp from notify-start job
 - Streamline input of regions to the trigger-component-deploy step

* [.github] - feature: specify deployment region in deploy-front-edge workflow

 - Added the `--region` flag to deployment command to target `us-central1` for deployment operations

* [.github/workflows] - refactor: update deployment workflows for front-edge and prodbox

 - Modify the deploy-front-edge.yml workflow to support multiple regions and notify separately for build start
 - Adjust the deploy-front-edge.yml to use a matrix strategy for building images across specified regions
 - Simplify the deploy-prodbox.yml by removing the region input and hardcoding the region for deployment
 - Consolidate steps in deploy-prodbox.yml, removing the separate notifications for build start and merging build and deploy steps

.github/workflows/deploy-alerting-temporal.yml

@@ -44,7 +44,8 @@ jobs:
             --image-name=alerting-temporal \
             --dockerfile-path=./Dockerfile \
             --working-dir=./alerting/temporal/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-connectors.yml

@@ -67,7 +67,8 @@ jobs:
             --image-name=connectors \
             --dockerfile-path=./connectors/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-core.yml

@@ -67,7 +67,8 @@ jobs:
             --image-name=core \
             --dockerfile-path=./Dockerfile \
             --working-dir=./core/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-dante.yml

@@ -54,7 +54,8 @@ jobs:
             --dockerfile-path=./dante.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=../.gcloudignore-dante \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
           cd ..
 
       - name: Generate a token

.github/workflows/deploy-front-edge.yml

@@ -3,6 +3,15 @@ name: Deploy Front Edge
 on:
   workflow_dispatch:
     inputs:
+      regions:
+        description: "Regions to deploy to"
+        required: true
+        default: "us-central1"
+        type: choice
+        options:
+          - "us-central1"
+          - "europe-west1"
+          - "us-central1,europe-west1"
       check_deployment_blocked:
         description: "Check #deployment locks or force deploy"
         required: true
@@ -21,13 +30,12 @@ env:
   IMAGE_NAME: front-edge
 
 jobs:
-  build-and-deploy:
+  notify-start:
     runs-on: ubuntu-latest
-
+    outputs:
+      thread_ts: ${{ steps.build_message.outputs.thread_ts }}
     steps:
-      - name: Checkout code
-        uses: actions/checkout@v3
-
+      - uses: actions/checkout@v3
       - name: Get short sha
         id: short_sha
         run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
@@ -51,20 +59,42 @@ jobs:
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
 
+  build:
+    needs: notify-start
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        region: ${{ fromJson(format('[{0}]', join(split(github.event.inputs.regions, ','), ','))) }}
+      fail-fast: true
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: "Authenticate with Google Cloud"
         uses: "google-github-actions/auth@v1"
         with:
           credentials_json: "${{ secrets.GCLOUD_SA_KEY }}"
 
-      - name: Build the image on Cloud Build
+      - name: Build image for ${{ matrix.region }}
         run: |
           chmod +x ./k8s/cloud-build.sh
           ./k8s/cloud-build.sh \
             --image-name=$IMAGE_NAME \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://front-edge.dust.tt
+            --dust-client-facing-url=https://front-edge.dust.tt \
+            --region=${{ matrix.region }}
 
+  deploy:
+    needs: [notify-start, build]
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Get short sha
+        id: short_sha
+        run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
       - name: Generate a token
         id: generate-token
         uses: actions/create-github-app-token@v1
@@ -85,10 +115,10 @@ jobs:
               repo: 'dust-infra',
               event_type: 'trigger-component-deploy',
               client_payload: {
-                regions: 'us-central1',
+                regions: '${{ github.event.inputs.regions }}',
                 component: 'front-edge',
                 image_tag: '${{ steps.short_sha.outputs.short_sha }}',
-                slack_thread_ts: "${{ steps.build_message.outputs.thread_ts }}",
+                slack_thread_ts: "${{ needs.notify-start.outputs.thread_ts }}",
                 slack_channel: '${{ secrets.SLACK_CHANNEL_ID }}'
               }
             });
@@ -103,4 +133,4 @@ jobs:
           image_tag: ${{ steps.short_sha.outputs.short_sha }}
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
-          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
+          thread_ts: "${{ needs.notify-start.outputs.thread_ts }}"

.github/workflows/deploy-front-qa.yml

@@ -45,7 +45,8 @@ jobs:
             --image-name=front-qa \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://front-qa.dust.tt
+            --dust-client-facing-url=https://front-qa.dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-front.yml

@@ -64,7 +64,8 @@ jobs:
             --image-name=front \
             --dockerfile-path=./front/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-nginx.yml

@@ -51,7 +51,8 @@ jobs:
             --dockerfile-path=./nginx.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=../.gcloudignore-nginx \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
           cd ..
 
       - name: Generate a token

.github/workflows/deploy-oauth.yml

@@ -64,7 +64,8 @@ jobs:
             --image-name=oauth \
             --dockerfile-path=./oauth.Dockerfile \
             --working-dir=./core/ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

.github/workflows/deploy-prodbox.yml

@@ -69,7 +69,8 @@ jobs:
             --dockerfile-path=./prodbox.Dockerfile \
             --working-dir=./ \
             --gcloud-ignore-file=.gcloudignore-prodbox \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token
@@ -109,4 +110,4 @@ jobs:
           image_tag: ${{ steps.short_sha.outputs.short_sha }}
           channel: ${{ secrets.SLACK_CHANNEL_ID }}
           slack_token: ${{ secrets.SLACK_BOT_TOKEN }}
-          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"
+          thread_ts: "${{ steps.build_message.outputs.thread_ts }}"

.github/workflows/deploy-viz.yml

@@ -67,7 +67,8 @@ jobs:
             --image-name=viz \
             --dockerfile-path=./viz/Dockerfile \
             --working-dir=./ \
-            --dust-client-facing-url=https://dust.tt
+            --dust-client-facing-url=https://dust.tt \ 
+            --region=us-central1
 
       - name: Generate a token
         id: generate-token

k8s/cloud-build.sh

@@ -10,6 +10,7 @@ GCLOUD_IGNORE_FILE=""
 IMAGE_NAME=""
 DOCKERFILE_PATH=""
 DUST_CLIENT_FACING_URL=""
+REGION=""
 
 # parse command-line arguments
 while [[ $# -gt 0 ]]; do
@@ -34,6 +35,10 @@ while [[ $# -gt 0 ]]; do
       DUST_CLIENT_FACING_URL="${1#*=}"
       shift
       ;;
+    --region=*)
+      REGION="${1#*=}"
+      shift
+      ;;
     *)
       echo "unknown argument: $1"
       exit 1
@@ -42,8 +47,8 @@ while [[ $# -gt 0 ]]; do
 done
 
 # check required arguments
-if [ -z "$WORKING_DIR" ] || [ -z "$IMAGE_NAME" ] || [ -z "$DOCKERFILE_PATH" ]; then
-  echo "error: --working-dir, --image-name, and --dockerfile-path are required"
+if [ -z "$WORKING_DIR" ] || [ -z "$IMAGE_NAME" ] || [ -z "$DOCKERFILE_PATH" ] || [ -z "$REGION" ]; then
+  echo "error: --working-dir, --image-name, --region and --dockerfile-path are required"
   exit 1
 fi
 
@@ -54,13 +59,13 @@ cd "$WORKING_DIR"
 echo "current working directory is $(pwd)"
 
 # prepare substitutions
-SUBSTITUTIONS="SHORT_SHA=$(git rev-parse --short HEAD),_IMAGE_NAME=$IMAGE_NAME,_DOCKERFILE_PATH=$DOCKERFILE_PATH"
+SUBSTITUTIONS="SHORT_SHA=$(git rev-parse --short HEAD),_IMAGE_NAME=$IMAGE_NAME,_DOCKERFILE_PATH=$DOCKERFILE_PATH,_REGION=$REGION"
 if [ -n "$DUST_CLIENT_FACING_URL" ]; then
     SUBSTITUTIONS="$SUBSTITUTIONS,_DUST_CLIENT_FACING_URL=$DUST_CLIENT_FACING_URL"
 fi
 
 # prepare the gcloud command
-GCLOUD_CMD=(gcloud builds submit --quiet --config "${SCRIPT_DIR}/cloudbuild.yaml")
+GCLOUD_CMD=(gcloud builds submit --quiet --config "${SCRIPT_DIR}/cloudbuild.yaml" --region="$REGION")
 
 # add ignore file argument if specified
 if [ -n "$GCLOUD_IGNORE_FILE" ]; then

k8s/cloudbuild.yaml

@@ -5,8 +5,8 @@ steps:
       depot build \
         --project 3vz0lnf16v \
         --provenance=false \
-        -t us-central1-docker.pkg.dev/${PROJECT_ID}/dust-images/${_IMAGE_NAME}:${SHORT_SHA} \
-        -t us-central1-docker.pkg.dev/${PROJECT_ID}/dust-images/${_IMAGE_NAME}:latest \
+        -t ${_REGION}-docker.pkg.dev/${PROJECT_ID}/dust-images/${_IMAGE_NAME}:${SHORT_SHA} \
+        -t ${_REGION}-docker.pkg.dev/${PROJECT_ID}/dust-images/${_IMAGE_NAME}:latest \
         -f ${_DOCKERFILE_PATH} \
         --build-arg COMMIT_HASH=${SHORT_SHA} \
         --build-arg NEXT_PUBLIC_VIZ_URL=https://viz.dust.tt \