From 764ee61c012de35e0f7f57ec00e1a231817203bc Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 11 Sep 2023 16:28:13 -0500 Subject: [PATCH 1/5] Create workflow for VZ API and adapt helper script --- .github/workflows/aws-vz-api-helper.sh | 45 +++++++++++++ .github/workflows/deploy_vz_api.yml | 88 ++++++++++++++++++++++++++ 2 files changed, 133 insertions(+) create mode 100644 .github/workflows/aws-vz-api-helper.sh create mode 100644 .github/workflows/deploy_vz_api.yml diff --git a/.github/workflows/aws-vz-api-helper.sh b/.github/workflows/aws-vz-api-helper.sh new file mode 100644 index 000000000..08f5fc035 --- /dev/null +++ b/.github/workflows/aws-vz-api-helper.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +# Determine work branch +case "${BRANCH_NAME}" in + "production") + export WORKING_STAGE="production"; + ;; + + "master") + export WORKING_STAGE="staging"; + ;; + + *) + export WORKING_STAGE="pr"; + ;; +esac + +# Deploys API to AWS +function deploy_aws_lambda { + if [[ "${WORKING_STAGE}" == "pr" ]]; then + echo "PRs are not currently being deployed to the API"; + exit 0; + fi; + + if [[ "${WORKING_STAGE}" == "" ]]; then + echo "No working stage could be determined." + exit 1; + fi; + + if [[ "${AWS_ACCESS_KEY_ID}" == "" ]] || [[ "${AWS_SECRET_ACCESS_KEY}" = "" ]]; then + echo "The AWS keys are not set" + exit 1; + fi; + + # Check ATD CR3 API + cd "atd-cr3-api"; + + python3 -m venv venv; + source venv/bin/activate; + pip install -r requirements.txt + + # Run zappa + echo $ZAPPA_SETTINGS > zappa_settings.json; + zappa update $WORKING_STAGE; +} diff --git a/.github/workflows/deploy_vz_api.yml b/.github/workflows/deploy_vz_api.yml new file mode 100644 index 000000000..dd0154c12 --- /dev/null +++ b/.github/workflows/deploy_vz_api.yml @@ -0,0 +1,88 @@ +# +# Builds Moped API +# +name: "Build & Publish Moped API" + +on: + push: + branches: + - main + - production + paths: + - ".github/workflows/deploy_vz_api.yml" + - ".github/workflows/aws-vz-api-helper.sh" + - "atd-cr3-api/**" + workflow_dispatch: + inputs: + description: + default: "" +jobs: + build: + name: Build + runs-on: ubuntu-20.04 + steps: + - uses: actions/setup-python@v4 + with: + python-version: "3.8" + architecture: "x64" + # Get the code first + - name: "Checkout" + uses: actions/checkout@v3 + # Then install the AWC CLI tools & boto3 + - name: "Install AWS Cli" + run: | + sudo apt-get install python3-setuptools + pip3 install awscli boto3 zappa virtualenv + # Run the shell commands using the AWS environment variables + - name: "Build" + env: + AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE: ${{ secrets.AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE }} + run: | + export BRANCH_NAME=${GITHUB_REF##*/} + echo "SHA: ${GITHUB_SHA}" + echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" + echo "GR: ${GITHUB_REF}" + echo "PWD: $(pwd)" + source $(pwd)/.github/workflows/aws-moped-api-helper.sh + deploy_moped_api + +# +# CircleCI Builder for Vision Zero +# + +version: 2.1 + +jobs: + deploy: + docker: + - image: circleci/python:3.7 + steps: + - checkout + - setup_remote_docker: + version: 18.09.3 + - run: + # Builds master and production branches only + name: "Building Docker" + command: | + source .circleci/docker-helper.sh + build_containers + - run: + # Builds master and production branches only + name: "Build VZE API" + command: | + source .circleci/aws-helper.sh + deploy_aws_lambda + +workflows: + version: 2.1 + build-and-deploy: + jobs: + - deploy: + filters: + branches: + only: + - master + - production From b72dd6faa8247365ad02654ca44f4ec99c0c6491 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 11 Sep 2023 16:38:17 -0500 Subject: [PATCH 2/5] Remove moped copy/paste stuff, remove lib installs that are covered in API reqs file --- .github/workflows/deploy_vz_api.yml | 53 ++++------------------------- 1 file changed, 6 insertions(+), 47 deletions(-) diff --git a/.github/workflows/deploy_vz_api.yml b/.github/workflows/deploy_vz_api.yml index dd0154c12..ff70ca711 100644 --- a/.github/workflows/deploy_vz_api.yml +++ b/.github/workflows/deploy_vz_api.yml @@ -1,12 +1,9 @@ -# -# Builds Moped API -# -name: "Build & Publish Moped API" +name: "Build & deploy the VZ API" on: push: branches: - - main + - master - production paths: - ".github/workflows/deploy_vz_api.yml" @@ -32,57 +29,19 @@ jobs: - name: "Install AWS Cli" run: | sudo apt-get install python3-setuptools - pip3 install awscli boto3 zappa virtualenv + pip3 install awscli virtualenv # Run the shell commands using the AWS environment variables - name: "Build" env: AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE: ${{ secrets.AWS_MOPED_API_ZAPPA_CONFIGURAITON_FILE }} + ZAPPA_SETTINGS: ${{ secrets.ZAPPA_SETTINGS }} run: | export BRANCH_NAME=${GITHUB_REF##*/} echo "SHA: ${GITHUB_SHA}" echo "ACTION/BRANCH_NAME: ${BRANCH_NAME}" echo "GR: ${GITHUB_REF}" echo "PWD: $(pwd)" - source $(pwd)/.github/workflows/aws-moped-api-helper.sh - deploy_moped_api - -# -# CircleCI Builder for Vision Zero -# - -version: 2.1 - -jobs: - deploy: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - setup_remote_docker: - version: 18.09.3 - - run: - # Builds master and production branches only - name: "Building Docker" - command: | - source .circleci/docker-helper.sh - build_containers - - run: - # Builds master and production branches only - name: "Build VZE API" - command: | - source .circleci/aws-helper.sh - deploy_aws_lambda - -workflows: - version: 2.1 - build-and-deploy: - jobs: - - deploy: - filters: - branches: - only: - - master - - production + source $(pwd)/.github/workflows/aws-vz-api-helper.sh + deploy_aws_lambda From 71ceb2cbc1bd7312c74e3e8fc92f73430422d54e Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 11 Sep 2023 16:46:41 -0500 Subject: [PATCH 3/5] Remove circleci jobs folder --- .circleci/aws-helper.sh | 45 ------------------------------- .circleci/config.yml | 37 ------------------------- .circleci/docker-helper.sh | 55 -------------------------------------- 3 files changed, 137 deletions(-) delete mode 100644 .circleci/aws-helper.sh delete mode 100644 .circleci/config.yml delete mode 100644 .circleci/docker-helper.sh diff --git a/.circleci/aws-helper.sh b/.circleci/aws-helper.sh deleted file mode 100644 index 92c8639b4..000000000 --- a/.circleci/aws-helper.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env bash - -# Determine work branch -case "${CIRCLE_BRANCH}" in - "production") - export WORKING_STAGE="production"; - ;; - - "master") - export WORKING_STAGE="staging"; - ;; - - *) - export WORKING_STAGE="pr"; - ;; -esac - -# Deploys API to AWS -function deploy_aws_lambda { - if [[ "${WORKING_STAGE}" == "pr" ]]; then - echo "PRs are not currently being deployed to the API"; - exit 0; - fi; - - if [[ "${WORKING_STAGE}" == "" ]]; then - echo "No working stage could be determined." - exit 1; - fi; - - if [[ "${AWS_ACCESS_KEY_ID}" == "" ]] || [[ "${AWS_SECRET_ACCESS_KEY}" = "" ]]; then - echo "The AWS keys are not set" - exit 1; - fi; - - # Check ATD CR3 API - cd "atd-cr3-api"; - - python3 -m venv venv; - source venv/bin/activate; - pip install -r requirements.txt - - # Run zappa - echo $ZAPPA_SETTINGS > zappa_settings.json; - zappa update $WORKING_STAGE; -} diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 217b5c40c..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,37 +0,0 @@ -# -# CircleCI Builder for Vision Zero -# - -version: 2.1 - -jobs: - deploy: - docker: - - image: circleci/python:3.7 - steps: - - checkout - - setup_remote_docker: - version: 18.09.3 - - run: - # Builds master and production branches only - name: "Building Docker" - command: | - source .circleci/docker-helper.sh - build_containers - - run: - # Builds master and production branches only - name: "Build VZE API" - command: | - source .circleci/aws-helper.sh - deploy_aws_lambda - -workflows: - version: 2.1 - build-and-deploy: - jobs: - - deploy: - filters: - branches: - only: - - master - - production diff --git a/.circleci/docker-helper.sh b/.circleci/docker-helper.sh deleted file mode 100644 index 899939b6e..000000000 --- a/.circleci/docker-helper.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash - -# Determine work branch -case "${CIRCLE_BRANCH}" in - "production") - export WORKING_STAGE="production"; - ;; - - "master") - export WORKING_STAGE="staging"; - ;; - - *) - export WORKING_STAGE="pr"; - ;; -esac - -export ATD_IMAGE="atddocker/atd-vz-etl"; -export ATD_IMAGE_AGOL="atddocker/atd-vz-etl-agol" -# -# We need to assign the name of the branch as the tag to be deployed -# -export ATD_TAG="${WORKING_STAGE}"; - - -function build_containers { - - if [[ "${WORKING_STAGE}" == "pr" ]]; then - echo "PRs are not currently deployed to the docker-hub."; - exit 0; - fi; - - echo "Logging in to Docker hub" - docker login -u $ATD_DOCKER_USER -p $ATD_DOCKER_PASS - - # First build, tag and push the regular ETL image - echo "docker build -f Dockerfile -t $ATD_IMAGE:$ATD_TAG ."; - docker build -f atd-etl/Dockerfile -t $ATD_IMAGE:$ATD_TAG ./atd-etl - - echo "docker tag $ATD_IMAGE:$ATD_TAG $ATD_IMAGE:$ATD_TAG;"; - docker tag $ATD_IMAGE:$ATD_TAG $ATD_IMAGE:$ATD_TAG; - - echo "docker push $ATD_IMAGE:$ATD_TAG"; - docker push $ATD_IMAGE:$ATD_TAG; - - # Then build, tag and push the Agol-containing Image - echo "docker build -f Dockerfile.agol -t $ATD_IMAGE_AGOL:$ATD_TAG ."; - docker build -f atd-etl/Dockerfile.agol -t $ATD_IMAGE_AGOL:$ATD_TAG ./atd-etl - - echo "docker tag $ATD_IMAGE_AGOL:$ATD_TAG $ATD_IMAGE_AGOL:$ATD_TAG;"; - docker tag $ATD_IMAGE_AGOL:$ATD_TAG $ATD_IMAGE_AGOL:$ATD_TAG; - - echo "docker push $ATD_IMAGE_AGOL:$ATD_TAG"; - docker push $ATD_IMAGE_AGOL:$ATD_TAG; -} From 614f6ca24eb6e3f3448fa91698213747189299a5 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 11 Sep 2023 17:03:11 -0500 Subject: [PATCH 4/5] Remove venv lib since the helper script uses python3 venv --- .github/workflows/deploy_vz_api.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/deploy_vz_api.yml b/.github/workflows/deploy_vz_api.yml index ff70ca711..af1775c1a 100644 --- a/.github/workflows/deploy_vz_api.yml +++ b/.github/workflows/deploy_vz_api.yml @@ -25,11 +25,11 @@ jobs: # Get the code first - name: "Checkout" uses: actions/checkout@v3 - # Then install the AWC CLI tools & boto3 + # Then install the AWC CLI tools - name: "Install AWS Cli" run: | sudo apt-get install python3-setuptools - pip3 install awscli virtualenv + pip3 install awscli # Run the shell commands using the AWS environment variables - name: "Build" env: From a59e9fb02943f140afb0ec89dc6ed2ee95285121 Mon Sep 17 00:00:00 2001 From: Mike Date: Mon, 11 Sep 2023 17:14:10 -0500 Subject: [PATCH 5/5] Remove awscli install since zappa doesn't need it --- .github/workflows/deploy_vz_api.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/deploy_vz_api.yml b/.github/workflows/deploy_vz_api.yml index af1775c1a..b37dafdee 100644 --- a/.github/workflows/deploy_vz_api.yml +++ b/.github/workflows/deploy_vz_api.yml @@ -25,11 +25,6 @@ jobs: # Get the code first - name: "Checkout" uses: actions/checkout@v3 - # Then install the AWC CLI tools - - name: "Install AWS Cli" - run: | - sudo apt-get install python3-setuptools - pip3 install awscli # Run the shell commands using the AWS environment variables - name: "Build" env: