Merge pull request #198 from NASA-IMPACT/dev

Preview of `dev` merge to `main` 😬

.coveragerc

@@ -0,0 +1,2 @@
+[run]
+omit = **/tests/**

.env.example

@@ -1,12 +1,20 @@
 STAGE=<Fill Me>
-PREFIX=<Fill Me>-${STAGE}
+APP_NAME=<Fill Me>
+PREFIX=${APP_NAME}-${STAGE}
 AWS_REGION=us-west-2
+BACKEND_VPC_ID=<Fill Me>
 VPC_ID=<Fill Me>
 AWS_PROFILE=<Fill Me>
+SUBNET_IDS='["<Private Subnet 1>", "<Private Subnet 2>"]'
 SUBNET_TAGNAME=<Fill Me>
 STATE_BUCKET_NAME=<Fill Me>
 STATE_BUCKET_KEY=<Fill Me>
 STATE_DYNAMO_TABLE=<Fill Me>
 ASSUME_ROLE_ARNS='["<Read role>", "<Write role>"]'
-COGNITO_APP_SECRET=<Fill Me>
-STAC_INGESTOR_API_URL=<Fill Me>
+VEDA_WORKFLOWS_CLIENT_SECRET_ID=<Fill Me>
+VEDA_PROGRAMMATIC_CLIENT_SECRET_ID=<Fill Me>
+VEDA_STAC_INGESTOR_API_URL=<Fill Me>
+VEDA_RASTER_URL=<Fill Me>
+VEDA_DATA_ACCESS_ROLE_ARN=<Fill Me>
+VEDA_STAC_URL=<Fill Me>
+WORKFLOW_ROOT_PATH=<Fill Me>

.env_dev

@@ -1,5 +1,6 @@
 STAGE=dev
-PREFIX=veda-pipeline-${STAGE}
+APP_NAME=veda-pipeline
+PREFIX=${APP_NAME}-${STAGE}
 AWS_REGION=us-west-2
 SUBNET_TAGNAME="MWAAEnvironment Private*"
 STATE_BUCKET_NAME=veda-tf-state-shared

.env_prod

@@ -1,7 +1,11 @@
 STAGE=production
-PREFIX=veda-pipeline-${STAGE}
+APP_NAME=veda-pipeline
+PREFIX=${APP_NAME}-${STAGE}
 SUBNET_TAGNAME="PrivSubnet*"
 STATE_BUCKET_NAME=${PREFIX}-tf-shared-state
 STATE_BUCKET_KEY=veda-mwaa/${PREFIX}-mwaa/terraform.tfstate
 STATE_DYNAMO_TABLE=${PREFIX}-shared-state-mwaa-lock-state
 IAM_ROLE_PERMISSIONS_BOUNDARY=arn:aws:iam::${AWS_ACCOUNT_ID}:policy/mcp-tenantOperator
+TF_VAR_mwaa_environment_class="mw1.2xlarge"
+TF_VAR_ecs_task_cpu=8192
+TF_VAR_ecs_task_memory=16384

.env_staging

@@ -1,5 +1,6 @@
 STAGE=staging
-PREFIX=veda-pipeline-${STAGE}
+APP_NAME=veda-pipeline
+PREFIX=${APP_NAME}-${STAGE}
 SUBNET_TAGNAME="MWAAEnvironment Private*"
 STATE_BUCKET_NAME=veda-tf-state-shared
 STATE_BUCKET_KEY=veda-mwaa/${PREFIX}-mwaa/terraform.tfstate

.flake8

@@ -1,3 +1,3 @@
 [flake8]
 # taken from github actions ignore
-ignore = E1, E2, E3, E5, W1, W2, W3, W5
+ignore = E1, E2, E3, E5, W1, W2, W3, W5

.github/actions/terraform-deploy/action.yml

@@ -0,0 +1,72 @@
+name: Deploy
+
+inputs:
+  env_aws_secret_name:
+    required: true
+    type: string
+  env-file:
+    type: string
+    default: ".env"
+  dir:
+    required: false
+    type: string
+    default: "."
+  script_path:
+    type: string
+  backend_stack_name:
+    type: string
+  auth_stack_name:
+    type: string
+
+runs:
+  using: "composite"
+
+  steps:
+    - name: Set up Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.10"
+        cache: "pip"
+
+    - name: Install python dependencies
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      run: pip install -r deploy_requirements.txt
+
+    - name: Get relevant environment configuration from aws secrets
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      env:
+        SECRET_SSM_NAME: ${{ inputs.env_aws_secret_name }}
+        AWS_DEFAULT_REGION: us-west-2
+      run: |
+        if [[ -z "${{ inputs.script_path }}" ]]; then
+        ./scripts/sync-env.sh ${{ inputs.env_aws_secret_name }}
+        else
+        echo ${{ inputs.auth_stack_name}}
+        echo ${{ inputs.backend_stack_name}}
+        python ${{ inputs.script_path }} --secret-id ${{ inputs.env_aws_secret_name }} --stack-names ${{ inputs.auth_stack_name}},${{ inputs.backend_stack_name}}
+        source .env
+        echo "PREFIX=data-pipeline-$STAGE" >> ${{ inputs.env-file }}
+        cat .env
+        fi
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v3
+      with:
+        terraform_version: 1.6.6
+
+    - name: Deploy
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      run: |
+        ./scripts/deploy.sh ${{ inputs.env-file }} <<< init
+        ./scripts/deploy.sh ${{ inputs.env-file }} <<< deploy
+
+    - name: Output workflows API endpoint
+      id: output_workflows_endpoint
+      shell: bash
+      working-directory: ${{ inputs.dir }}
+      run: |
+        cd ./infrastructure
+        terraform output -json workflows_api > ${HOME}/terraform_outputs.json

.github/workflows/cicd.yml

@@ -1,91 +1,77 @@
-name: CI/CD
+name: CICD 🚀
+
+permissions:
+  id-token: write
+  contents: read
 
 on:
   push:
+    branches:
+      - main
+      - dev
+      - production
   pull_request:
+    branches:
+      - main
+      - dev
+      - production
     types: [ opened, reopened, edited, synchronize ]
 
 jobs:
   gitflow-enforcer:
-      runs-on: ubuntu-latest
-      steps:
-        - name: Check branch
-          run: |
-            if [[ $GITHUB_BASE_REF == "main" ]]; then
-               if [[ $GITHUB_HEAD_REF != "dev" && $GITHUB_HEAD_REF != "revert-"* ]]; then
-                  echo "ERROR: You can only merge to 'main' from 'dev' or a 'revert-*' branch"
-                  exit 1
-               fi
-            elif [[ $GITHUB_BASE_REF == "production" ]]; then
-               if [[ $GITHUB_HEAD_REF != "main" && $GITHUB_HEAD_REF != "revert-"* ]]; then
-                  echo "ERROR: You can only merge to 'production' from 'main' or a 'revert-*' branch"
-                  exit 1
-               fi
-            fi
-
-
-  run-linters:
-    name: Run linters
+    name: GitFlow Enforcer 👮‍
     runs-on: ubuntu-latest
-    needs: gitflow-enforcer
+    steps:
+      - name: Check branch
+        if: github.base_ref == 'main' && github.head_ref != 'dev' || github.base_ref == 'production' && github.head_ref != 'main'
+        run: |
+          echo "ERROR: You can only merge to main from dev and to production from main"
+          exit 1
 
+  define-environment:
+    name: Set ✨ environment ✨
+    needs: gitflow-enforcer
+    runs-on: ubuntu-latest
     steps:
-      - name: Check out Git repository
-        uses: actions/checkout@v2
+      - name: Set the environment based on the branch
+        id: define_environment
+        run: |
+          if [ "${{ github.ref }}" = "refs/heads/main" ]; then
+            echo "env_name=staging" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" = "refs/heads/dev" ]; then
+            echo "env_name=development" >> $GITHUB_OUTPUT
+          elif [ "${{ github.ref }}" = "refs/heads/production" ]; then
+            echo "env_name=production" >> $GITHUB_OUTPUT
+          fi
+      - name: Print the environment
+        run: echo "The environment is ${{ steps.define_environment.outputs.env_name }}"
 
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: "3.10"
+    outputs:
+      env_name: ${{ steps.define_environment.outputs.env_name }}
 
-      - name: Install Python dependencies
-        run: pip install black flake8
+  deploy:
+    name: Deploy to ${{ needs.define-environment.outputs.env_name }} 🚀
+    runs-on: ubuntu-latest
+    if: ${{ needs.define-environment.outputs.env_name }}
+    needs: [gitflow-enforcer, define-environment]
+    environment: ${{ needs.define-environment.outputs.env_name }}
+    concurrency: ${{ needs.define-environment.outputs.env_name }}
 
-      - name: Run linters
-        uses: wearerequired/lint-action@v2
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
         with:
-          continue_on_error: false
-          black: true
-          flake8: true
-          flake8_args: "--ignore E1,E2,E3,E5,W1,W2,W3,W5" # black already handles formatting, this prevents conflicts
-
-  deploy-to-dev:
-    needs: run-linters
-    if: github.ref_name == 'dev'
-    concurrency: development
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: development
-      env-file: ".env_dev"
-      stage: "dev"
-      role-session-name: "veda-data-airflow-github-development-deployment"
-      aws-region: "us-west-2"
-
-    secrets: inherit
-
-  deploy-to-staging:
-    needs: run-linters
-    if: github.ref_name == 'main'
-    concurrency: staging
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: staging
-      env-file: ".env_staging"
-      stage: "staging"
-      role-session-name: "veda-data-airflow-github-staging-deployment"
-      aws-region: "us-west-2"
-    secrets: inherit
-
-  deploy-to-production:
-    needs: run-linters
-    if: github.ref_name == 'production'
-    concurrency: production
-    uses: "./.github/workflows/deploy.yml"
-    with:
-      environment: production
-      env-file: ".env_prod"
-      stage: "production"
-      role-session-name: "veda-data-airflow-github-production-deployment"
-      aws-region: "us-west-2"
+          lfs: "true"
+          submodules: "recursive"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.DEPLOYMENT_ROLE_ARN }}
+          role-session-name: "veda-airflow-github-${{ needs.define-environment.outputs.env_name }}-deployment"
+          aws-region: "us-west-2"
 
-    secrets: inherit
+      - name: Run deployment
+        uses: "./.github/actions/terraform-deploy"
+        with:
+          env_aws_secret_name: ${{ secrets.ENV_AWS_SECRET_NAME }}