From 3be13ac01cbdc058697039dd5f581e33026af12f Mon Sep 17 00:00:00 2001 From: JulesBelveze Date: Fri, 15 Nov 2024 11:00:20 +0100 Subject: [PATCH 1/4] [.github] - feature: add manual workflow dispatch for deploying infrastructure - Introduce workflow_dispatch trigger with a configurable input for deploying to the 'us-central1' region - Implement concurrency control to manage deployment processes and prevent collisions - Authenticate with Google Cloud and set up the Cloud SDK for deployment tasks - Build a Docker image using Cloud Build and a custom script, with parameters for image name and Dockerfile path - Generate a GitHub App token dynamically for use in the workflow - Enable triggering of a downstream repository's workflow using a repository dispatch event with a custom payload including the region and image tag --- .github/workflows/deploy-connectors-infra.yml | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 .github/workflows/deploy-connectors-infra.yml diff --git a/.github/workflows/deploy-connectors-infra.yml b/.github/workflows/deploy-connectors-infra.yml new file mode 100644 index 000000000000..24c70eaf4d75 --- /dev/null +++ b/.github/workflows/deploy-connectors-infra.yml @@ -0,0 +1,73 @@ +name: Deploy Infra + +on: + workflow_dispatch: + inputs: + us-central1: + description: "Deploy to us-central1" + type: boolean + default: true + +concurrency: + group: deploy_infra + cancel-in-progress: false + +env: + GCLOUD_PROJECT_ID: ${{ secrets.GCLOUD_PROJECT_ID }} + +jobs: + build-and-deploy: + runs-on: ubuntu-latest + + if: github.ref == 'refs/heads/main' + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Get short sha + id: short_sha + run: echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: "Authenticate with Google Cloud" + uses: "google-github-actions/auth@v1" + with: + credentials_json: "${{ secrets.GCLOUD_SA_KEY }}" + + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v1" + + - name: Build the image on Cloud Build + run: | + chmod +x ./k8s/cloud-build.sh + ./k8s/cloud-build.sh \ + --image-name=connectors \ + --dockerfile-path=./connectors/Dockerfile \ + --working-dir=. \ + --dust-client-facing-url=https://dust.tt + + - name: Generate a token + id: generate-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ vars.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + + - name: Trigger dust-infra workflow + uses: actions/github-script@v6 + env: + GH_TOKEN: ${{ steps.generate-token.outputs.token }} + with: + github-token: ${{ secrets.GH_TOKEN }} + script: | + await github.rest.repos.createDispatchEvent({ + owner: 'dust-tt', + repo: 'dust-infra', + event_type: 'trigger-component-deploy', + client_payload: { + us_central1: ${{ inputs.us-central1 }}, + component: 'connectors', + image_tag: '${{ steps.short_sha.outputs.short_sha }}' + } + }); \ No newline at end of file From 34b35d151601130985ad17b9c7dc4fc5795b9fef Mon Sep 17 00:00:00 2001 From: JulesBelveze Date: Fri, 15 Nov 2024 11:06:49 +0100 Subject: [PATCH 2/4] [.github] - fix: update secrets and app ID for connector infra deployment - Switch to using specific app ID and private key for infra deployment - Correct the environment variable used for the GitHub token in the dispatch event trigger --- .github/workflows/deploy-connectors-infra.yml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-connectors-infra.yml b/.github/workflows/deploy-connectors-infra.yml index 24c70eaf4d75..611d274f8f2a 100644 --- a/.github/workflows/deploy-connectors-infra.yml +++ b/.github/workflows/deploy-connectors-infra.yml @@ -50,16 +50,15 @@ jobs: id: generate-token uses: actions/create-github-app-token@v1 with: - app-id: ${{ vars.APP_ID }} - private-key: ${{ secrets.APP_PRIVATE_KEY }} - + app-id: ${{ vars.INFRA_DISPATCH_APP_ID }} + private-key: ${{ secrets.INFRA_DISPATCH_APP_PRIVATE_KEY }} - name: Trigger dust-infra workflow uses: actions/github-script@v6 env: GH_TOKEN: ${{ steps.generate-token.outputs.token }} with: - github-token: ${{ secrets.GH_TOKEN }} + github-token: ${{ env.GH_TOKEN }} script: | await github.rest.repos.createDispatchEvent({ owner: 'dust-tt', From c583ac6dd5959e70259b0af20d86e605c4034972 Mon Sep 17 00:00:00 2001 From: JulesBelveze Date: Fri, 15 Nov 2024 11:11:00 +0100 Subject: [PATCH 3/4] [.github] - fix: correct working directory path in GitHub Actions config - Ensure the `cloud-build.sh` script uses the correct relative working directory by adding a leading `./` to the path configuration --- .github/workflows/deploy-connectors-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-connectors-infra.yml b/.github/workflows/deploy-connectors-infra.yml index 611d274f8f2a..a967e1b31b6f 100644 --- a/.github/workflows/deploy-connectors-infra.yml +++ b/.github/workflows/deploy-connectors-infra.yml @@ -43,7 +43,7 @@ jobs: ./k8s/cloud-build.sh \ --image-name=connectors \ --dockerfile-path=./connectors/Dockerfile \ - --working-dir=. \ + --working-dir=./ \ --dust-client-facing-url=https://dust.tt - name: Generate a token From a7b2e7ef6dffcfe8a1f92cb9abbe4e69ea376c1a Mon Sep 17 00:00:00 2001 From: JulesBelveze Date: Fri, 15 Nov 2024 11:11:26 +0100 Subject: [PATCH 4/4] [.github] - fix: use secret for INFRA_DISPATCH_APP_ID in GitHub Actions - Changed the GitHub App ID reference to use secrets for enhanced security and better management of sensitive data - This update ensures that the App ID is not exposed in the workflow file, aligning with best practices for credential storage --- .github/workflows/deploy-connectors-infra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy-connectors-infra.yml b/.github/workflows/deploy-connectors-infra.yml index a967e1b31b6f..2d2fe1f5df51 100644 --- a/.github/workflows/deploy-connectors-infra.yml +++ b/.github/workflows/deploy-connectors-infra.yml @@ -50,7 +50,7 @@ jobs: id: generate-token uses: actions/create-github-app-token@v1 with: - app-id: ${{ vars.INFRA_DISPATCH_APP_ID }} + app-id: ${{ secrets.INFRA_DISPATCH_APP_ID }} private-key: ${{ secrets.INFRA_DISPATCH_APP_PRIVATE_KEY }} - name: Trigger dust-infra workflow