From 6697801680c37dc2d43beaba27372340b19de89b Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 08:35:06 -0400 Subject: [PATCH 01/10] introduce script and workflow job to update deployment info --- .../workflows/deploy-multiple-accounts.yml | 30 ++++++++- update_matrix.py | 67 +++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) create mode 100644 update_matrix.py diff --git a/.github/workflows/deploy-multiple-accounts.yml b/.github/workflows/deploy-multiple-accounts.yml index 3a3770dc..71f12894 100644 --- a/.github/workflows/deploy-multiple-accounts.yml +++ b/.github/workflows/deploy-multiple-accounts.yml @@ -66,7 +66,6 @@ jobs: aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} aws-region: ${{ secrets.AWS_DEFAULT_REGION }} # Get AWS parameters to setup environment variables for Serverless function - # Get AWS parameters to setup environment variables for Serverless function - name: Set Twilio Account SID uses: marvinpinto/action-inject-ssm-secrets@latest with: @@ -219,7 +218,6 @@ jobs: with: ssm_parameter: 'ASELO_DEPLOYS_CHANNEL_ID' env_variable_name: 'ASELO_DEPLOYS_CHANNEL_ID' - - name: Slack Aselo channel id: slack uses: slackapi/slack-github-action@v1.26.0 @@ -228,3 +226,31 @@ jobs: slack-message: '`[SERVERLESS PARALLEL DEPLOYMENT]` Serverless helplines ${{ inputs.helplines }} successfully deployed to the following environments: ${{ inputs.environments }} from ${{ github.ref_type }} `${{ github.ref_name }}` requested by `${{ github.triggering_actor }}` with SHA ${{ github.sha }} :rocket:.' env: SLACK_BOT_TOKEN: ${{ env.GITHUB_ACTIONS_SLACK_BOT_TOKEN }} + update-sheet: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread + - name: Update Google Sheets + env: + GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} + helpline: ${{ inputs.helplines }} + environments: ${{ inputs.environments }} + environment: ${{ env.environment}} + github_ref: ${{ github.ref }} + github_sha: ${{ github.sha }} + github_actor: ${{ github.triggering_actor }} + github_branch: ${{ github.ref_name }} + aws_region: ${{ secrets.AWS_DEFAULT_REGION }} + run: | + echo "${GOOGLE_SHEETS_CREDENTIALS}" > credentials.json + python update_matrix.py diff --git a/update_matrix.py b/update_matrix.py new file mode 100644 index 00000000..707afe2a --- /dev/null +++ b/update_matrix.py @@ -0,0 +1,67 @@ +import os +import json +import sys +import gspread +from google.oauth2.service_account import Credentials +from google.auth.exceptions import GoogleAuthError +from datetime import datetime + +def main(): + try: + # Load credentials from the environment variable + credentials_json = os.getenv('GOOGLE_SHEETS_CREDENTIALS') + sheet_id = os.getenv('GOOGLE_SHEET_ID') + + if not credentials_json: + raise ValueError("Credentials JSON not found in environment variable.") + if not sheet_id: + raise ValueError("Sheet ID not found in environment variable.") + + credentials_data = json.loads(credentials_json) + print("Credentials loaded successfully.") + + # Define the scope and credentials + scopes = [ + 'https://www.googleapis.com/auth/spreadsheets', + 'https://www.googleapis.com/auth/drive.file' + ] + credentials = Credentials.from_service_account_info(credentials_data, scopes=scopes) + print("Credentials initialized successfully.") + + client = gspread.authorize(credentials) + print("Google Sheets client authorized successfully.") + + sheet = client.open_by_key(sheet_id).worksheet("Deploys") + print("Google Sheet opened successfully.") + + current_date = datetime.now().strftime("%m/%d/%Y") + current_time = datetime.now().strftime("%H:%M:%S") + + helpline = os.getenv('helpline') + environments = os.getenv('environments') + environment = os.getenv('environment') + github_ref = os.getenv('github_ref') + github_sha = os.getenv('github_sha') + github_actor = os.getenv('github_actor') + github_branch = os.getenv('github_branch') + aws_region = os.getenv('aws_region') + + print("Environment variables loaded successfully.") + + hl_env = helpline + "_" + environment + + new_row = [current_date, current_time, "serverless", hl_env , environments, github_ref, aws_region, github_actor, github_branch, github_sha] + append_row = sheet.append_row(new_row) + print("Row added to Deploys sheet.") + + if not append_row: + raise RuntimeError("Failed to add row to Deploys sheet.") + + print("Google Sheet updated successfully.") + + except (GoogleAuthError, ValueError, RuntimeError, json.JSONDecodeError) as e: + print(f"Error: {e}") + sys.exit(1) + +if __name__ == '__main__': + main() From a485dcd0a39bc8a07025e098e8a9229d0c51d559 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 08:38:14 -0400 Subject: [PATCH 02/10] fix: license header --- update_matrix.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/update_matrix.py b/update_matrix.py index 707afe2a..f4072b2f 100644 --- a/update_matrix.py +++ b/update_matrix.py @@ -1,3 +1,17 @@ +# Copyright (C) 2021-2023 Technology Matters +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see https://www.gnu.org/licenses/. + import os import json import sys From e93bde12822d64b45e0ff76fcb40909717d0072c Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 11:59:35 -0400 Subject: [PATCH 03/10] move update sheet logic to main action --- .github/actions/main-action/action.yml | 28 +++++++++++++++++++ .../workflows/deploy-multiple-accounts.yml | 28 ------------------- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index 60a5dedd..cd66ad8a 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -184,3 +184,31 @@ runs: slack-message: "`[Serverless]` Deployment to `${{ env.helpline-name }}` from ${{ github.ref_type }} `${{ github.ref_name }}` requested by `${{ github.triggering_actor }}` completed using workflow '${{ github.workflow }}' with SHA ${{ github.sha }} :rocket:." env: SLACK_BOT_TOKEN: ${{ env.GITHUB_ACTIONS_SLACK_BOT_TOKEN }} + # Update Google Sheets + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Python + uses: actions/setup-python@v3 + with: + python-version: '3.8' + - name: Install dependencies + shell: bash + run: | + python -m pip install --upgrade pip + pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread + - name: Update Google Sheets + shell: bash + env: + GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} + helpline: ${{ inputs.helpline-code }} + environments: ${{ inputs.environment-code }} + environment: ${{ inputs.environment }} + github_ref: ${{ github.ref }} + github_sha: ${{ github.sha }} + github_actor: ${{ github.triggering_actor }} + github_branch: ${{ github.ref_name }} + aws_region: ${{ secrets.AWS_DEFAULT_REGION }} + run: | + echo "${GOOGLE_SHEETS_CREDENTIALS}" > credentials.json + python update_matrix \ No newline at end of file diff --git a/.github/workflows/deploy-multiple-accounts.yml b/.github/workflows/deploy-multiple-accounts.yml index 71f12894..d4fd623b 100644 --- a/.github/workflows/deploy-multiple-accounts.yml +++ b/.github/workflows/deploy-multiple-accounts.yml @@ -226,31 +226,3 @@ jobs: slack-message: '`[SERVERLESS PARALLEL DEPLOYMENT]` Serverless helplines ${{ inputs.helplines }} successfully deployed to the following environments: ${{ inputs.environments }} from ${{ github.ref_type }} `${{ github.ref_name }}` requested by `${{ github.triggering_actor }}` with SHA ${{ github.sha }} :rocket:.' env: SLACK_BOT_TOKEN: ${{ env.GITHUB_ACTIONS_SLACK_BOT_TOKEN }} - update-sheet: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v3 - with: - python-version: '3.8' - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread - - name: Update Google Sheets - env: - GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} - GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} - helpline: ${{ inputs.helplines }} - environments: ${{ inputs.environments }} - environment: ${{ env.environment}} - github_ref: ${{ github.ref }} - github_sha: ${{ github.sha }} - github_actor: ${{ github.triggering_actor }} - github_branch: ${{ github.ref_name }} - aws_region: ${{ secrets.AWS_DEFAULT_REGION }} - run: | - echo "${GOOGLE_SHEETS_CREDENTIALS}" > credentials.json - python update_matrix.py From 795d9908bd212a0b05659f96f80af257505ccf78 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 12:00:35 -0400 Subject: [PATCH 04/10] supress slack --- .github/actions/main-action/action.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index cd66ad8a..1c7cdeca 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -174,16 +174,16 @@ runs: working-directory: ${{ github.action_path }} shell: bash - # Send Slack notifying success - - name: Slack Aselo channel - id: slack-parallel - uses: slackapi/slack-github-action@v1.14.0 - if: ${{ inputs.send-slack-message != 'false' }} - with: - channel-id: ${{ env.ASELO_DEPLOYS_CHANNEL_ID }} - slack-message: "`[Serverless]` Deployment to `${{ env.helpline-name }}` from ${{ github.ref_type }} `${{ github.ref_name }}` requested by `${{ github.triggering_actor }}` completed using workflow '${{ github.workflow }}' with SHA ${{ github.sha }} :rocket:." - env: - SLACK_BOT_TOKEN: ${{ env.GITHUB_ACTIONS_SLACK_BOT_TOKEN }} + # # Send Slack notifying success + # - name: Slack Aselo channel + # id: slack-parallel + # uses: slackapi/slack-github-action@v1.14.0 + # if: ${{ inputs.send-slack-message != 'false' }} + # with: + # channel-id: ${{ env.ASELO_DEPLOYS_CHANNEL_ID }} + # slack-message: "`[Serverless]` Deployment to `${{ env.helpline-name }}` from ${{ github.ref_type }} `${{ github.ref_name }}` requested by `${{ github.triggering_actor }}` completed using workflow '${{ github.workflow }}' with SHA ${{ github.sha }} :rocket:." + # env: + # SLACK_BOT_TOKEN: ${{ env.GITHUB_ACTIONS_SLACK_BOT_TOKEN }} # Update Google Sheets - name: Checkout uses: actions/checkout@v4 From 37dc9525d8411af394be14e6adcb8fc7f8cb9b12 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 12:28:23 -0400 Subject: [PATCH 05/10] remove / replace secrets call --- .github/actions/main-action/action.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index 1c7cdeca..34ec9101 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -126,6 +126,8 @@ runs: ASELO_APP_SECRET_KEY=${{ inputs.aselo-app-secret-key }} S3_BUCKET=${{ inputs.s3-bucket }} AWS_REGION=${{ inputs.aws-region }} + GOOGLE_SHEETS_CREDENTIALS=${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID=${{ secrets.GOOGLE_SHEET_ID }} TWILIO_SERVERLESS_API_CONCURRENCY=1 EOT shell: bash @@ -199,8 +201,8 @@ runs: - name: Update Google Sheets shell: bash env: - GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} - GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} + # GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + # GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} helpline: ${{ inputs.helpline-code }} environments: ${{ inputs.environment-code }} environment: ${{ inputs.environment }} @@ -208,7 +210,7 @@ runs: github_sha: ${{ github.sha }} github_actor: ${{ github.triggering_actor }} github_branch: ${{ github.ref_name }} - aws_region: ${{ secrets.AWS_DEFAULT_REGION }} + # aws_region: ${{ secrets.AWS_DEFAULT_REGION }} run: | echo "${GOOGLE_SHEETS_CREDENTIALS}" > credentials.json python update_matrix \ No newline at end of file From f086aa9b32e77f827539042ae7b2a0cb33ca691c Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 15:05:57 -0400 Subject: [PATCH 06/10] remove shell for running script --- .github/actions/main-action/action.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index 34ec9101..fcac94e8 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -199,7 +199,6 @@ runs: python -m pip install --upgrade pip pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread - name: Update Google Sheets - shell: bash env: # GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} # GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} @@ -213,4 +212,4 @@ runs: # aws_region: ${{ secrets.AWS_DEFAULT_REGION }} run: | echo "${GOOGLE_SHEETS_CREDENTIALS}" > credentials.json - python update_matrix \ No newline at end of file + python update_matrix.py \ No newline at end of file From a75a50d1750a2aac5f117a1e7e2bff647a246af7 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 15:14:31 -0400 Subject: [PATCH 07/10] refactor script --- .github/actions/main-action/action.yml | 4 ++-- update_matrix.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index fcac94e8..9cc93db0 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -200,8 +200,8 @@ runs: pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread - name: Update Google Sheets env: - # GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} - # GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} + GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} helpline: ${{ inputs.helpline-code }} environments: ${{ inputs.environment-code }} environment: ${{ inputs.environment }} diff --git a/update_matrix.py b/update_matrix.py index f4072b2f..772a6c3c 100644 --- a/update_matrix.py +++ b/update_matrix.py @@ -58,13 +58,13 @@ def main(): github_sha = os.getenv('github_sha') github_actor = os.getenv('github_actor') github_branch = os.getenv('github_branch') - aws_region = os.getenv('aws_region') + # aws_region = os.getenv('aws_region') print("Environment variables loaded successfully.") hl_env = helpline + "_" + environment - new_row = [current_date, current_time, "serverless", hl_env , environments, github_ref, aws_region, github_actor, github_branch, github_sha] + new_row = [current_date, current_time, "serverless", hl_env , environments, github_ref, github_actor, github_branch, github_sha] append_row = sheet.append_row(new_row) print("Row added to Deploys sheet.") From 5056fa829c0cf6bd58fdd218b1064befaa2fcf6c Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 15:22:10 -0400 Subject: [PATCH 08/10] refactor yml with inputs --- .github/actions/main-action/action.yml | 8 ++++---- .github/workflows/deploy-multiple-accounts.yml | 2 ++ 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index 9cc93db0..d7760195 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -126,9 +126,9 @@ runs: ASELO_APP_SECRET_KEY=${{ inputs.aselo-app-secret-key }} S3_BUCKET=${{ inputs.s3-bucket }} AWS_REGION=${{ inputs.aws-region }} - GOOGLE_SHEETS_CREDENTIALS=${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} - GOOGLE_SHEET_ID=${{ secrets.GOOGLE_SHEET_ID }} TWILIO_SERVERLESS_API_CONCURRENCY=1 + GOOGLE_SHEETS_CREDENTIALS=${{ inputs.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID=${{ inputs.GOOGLE_SHEET_ID }} EOT shell: bash # Can be overridden in custom action @@ -200,8 +200,8 @@ runs: pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread - name: Update Google Sheets env: - GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} - GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} + GOOGLE_SHEETS_CREDENTIALS: ${{ inputs.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID: ${{ inputs.GOOGLE_SHEET_ID }} helpline: ${{ inputs.helpline-code }} environments: ${{ inputs.environment-code }} environment: ${{ inputs.environment }} diff --git a/.github/workflows/deploy-multiple-accounts.yml b/.github/workflows/deploy-multiple-accounts.yml index d4fd623b..dab18b65 100644 --- a/.github/workflows/deploy-multiple-accounts.yml +++ b/.github/workflows/deploy-multiple-accounts.yml @@ -191,6 +191,8 @@ jobs: # Set 'false' if the target environment is production OR the force_enable_operating_hours override option is checked - otherwise 'true' disable-operating-hours: ${{ (inputs.force_enable_operating_hours == 'true' || matrix.environment_code == 'PROD') && 'false' || 'true' }} send-slack-message: ${{ inputs.send_slack_message_per_deploy }} + GOOGLE_SHEETS_CREDENTIALS: ${{ secrets.GOOGLE_SHEETS_CREDENTIALS }} + GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} # Send Slack notifying success send-slack-message: # The type of runner that the job will run on From c871886c364a9c058e47036c2910a16c9180fbc0 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 15:26:26 -0400 Subject: [PATCH 09/10] refactor yml with inputs --- .github/actions/main-action/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/main-action/action.yml b/.github/actions/main-action/action.yml index d7760195..2a841148 100644 --- a/.github/actions/main-action/action.yml +++ b/.github/actions/main-action/action.yml @@ -199,6 +199,7 @@ runs: python -m pip install --upgrade pip pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client gspread - name: Update Google Sheets + shell: bash env: GOOGLE_SHEETS_CREDENTIALS: ${{ inputs.GOOGLE_SHEETS_CREDENTIALS }} GOOGLE_SHEET_ID: ${{ inputs.GOOGLE_SHEET_ID }} From 6b008d0173e0f30ea69333fd4c7cdf5738315117 Mon Sep 17 00:00:00 2001 From: mythilytm Date: Wed, 24 Jul 2024 15:31:11 -0400 Subject: [PATCH 10/10] change script --- update_matrix.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/update_matrix.py b/update_matrix.py index 772a6c3c..6bfa821c 100644 --- a/update_matrix.py +++ b/update_matrix.py @@ -52,9 +52,7 @@ def main(): current_time = datetime.now().strftime("%H:%M:%S") helpline = os.getenv('helpline') - environments = os.getenv('environments') environment = os.getenv('environment') - github_ref = os.getenv('github_ref') github_sha = os.getenv('github_sha') github_actor = os.getenv('github_actor') github_branch = os.getenv('github_branch') @@ -64,7 +62,7 @@ def main(): hl_env = helpline + "_" + environment - new_row = [current_date, current_time, "serverless", hl_env , environments, github_ref, github_actor, github_branch, github_sha] + new_row = [current_date, current_time, "serverless", hl_env, github_branch, github_actor, github_sha] append_row = sheet.append_row(new_row) print("Row added to Deploys sheet.")