Merge pull request #11 from TheJacksonLaboratory/G3-180-add-github-ac…

…tions-and-deployment-manifests-to-geneweaver-aon-api-pt-2

G3 180 add GitHub actions and deployment manifests to geneweaver aon api pt 2

.github/workflows/README.md

@@ -0,0 +1,72 @@
+# GitHub Actions
+
+The files in this directory are used to configure the Github Actions workflows for
+`geneweaver-api`. The workflows are used to automatically build and test the project
+when changes are pushed to the repository.
+
+Any file that starts with an underscore (`_`) is a "reusable workflow". These files
+are not directly used by GitHub Actions, but are instead referenced by the workflows
+files that do not start with an underscore.
+
+There are five reusable workflows:
+
+- Check Coverage (`_check-coverage-action.yml`): This workflow is used to check the code
+  coverage of the project.
+- Format Lint (`_format-lint-action.yml`): This workflow is used to check the formatting
+  and linting of the project.
+- Run Tests (`_run-tests-action.yml`): This workflow is used to run the tests for the
+  project.
+- Skaffold Build (`_skaffold-build-k8s.yml`): This workflow is used to build the
+  Docker images for the project.
+- Skaffold Deploy (`_skaffold-deploy-action.yml`): This workflow is used to deploy the
+  Docker images to kubernetes.
+
+There are two _main_ workflows that are used by GitHub Actions:
+
+- Pull Requests (`pull_requests.yml`): This workflow is used to build and test the
+  project when a pull request is opened.
+- Release (`release.yml`): This workflow that is run whenever the version number changes
+  on the `main` branch.
+
+There are also three quality assurance workflows that are run on any change to the main
+branch:
+
+- Coverage (`coverage.yml`): This workflow is used to check the code coverage of the
+  project.
+- Style (`style.yml`): This workflow is used to check the formatting and linting of the
+  project.
+- Tests (`tests.yml`): This workflow is used to run the tests for the project.
+
+
+## Pull Requests
+
+The pull request workflow is run whenever a pull request is opened. This workflow
+will:
+
+- Check the formatting and linting of the project.
+- Run the tests for the project.
+- Check the code coverage of the project.
+- Build the Docker images (into the `test` registry) for the project.
+- Deploy the Docker images (from the `test` registry) to kubernetes (into the `dev` 
+  environment).
+
+## Release
+
+The release workflow is run whenever the version number changes on the `main` branch.
+This workflow will:
+
+- Check the formatting and linting of the project.
+- Run the tests for the project.
+- Check the code coverage of the project.
+- Build the Docker images (into the `prod` registry) for the project.
+- Deploy the Docker images (from the `prod` registry) to kubernetes (into the `sqa` 
+  environment).
+  - It will wait for approval from SQA before running this step
+- If the version number is not a pre-release version (contains a letter) it will then:
+    - Deploy the Docker images (from the `prod` registry) to kubernetes (into the 
+      `stage` environment).
+      - It will wait for approval from SQA before running this step
+    - Deploy the Docker images (from the `prod` registry) to kubernetes (into the `prod` 
+      environment).
+      - It will wait for approval from SQA before running this step
+    - It will then create a draft GitHub release

.github/workflows/_check-coverage-action.yml

@@ -0,0 +1,134 @@
+name: 'Test Coverage Definition'
+on:
+  workflow_call:
+    inputs:
+      coverage-module:
+        description: "Module to test coverage for"
+        type: string
+        required: true
+      python-version:
+        description: Python version to set up'
+        default: '3.11'
+        type: string
+      runner-os:
+        description: 'Runner OS'
+        default: 'ubuntu-latest'
+        type: string
+      upload-coverage:
+        description: 'Upload coverage results'
+        default: true
+        type: boolean
+      required-coverage:
+        description: 'Required coverage percentage'
+        default: 100
+        type: string
+      show-test-traceback:
+        description: "Show traceback for failed tests"
+        type: string
+        default: "no"
+      report-to-compass:
+        description: "Report coverage to Compass"
+        type: boolean
+        default: false
+jobs:
+  check_coverage:
+    runs-on: ${{ inputs.runner-os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          if [[ "$RUNNER_OS" == "macOS" ]]; then
+            echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH
+          fi
+      - name: Configure Poetry
+        run: poetry config virtualenvs.create false
+      - name: Install dependencies with Poetry
+        run: poetry install
+      - name: Test with pytest
+        run: |
+          poetry run pytest tests \
+          --tb=${{ inputs.show-test-traceback }} \
+          --cov=${{ inputs.coverage-module }} \
+          --cov-report=term \
+          --cov-report=html \
+          --cov-fail-under=${{ inputs.required-coverage }} > coverage_report.txt
+      - name: Upload coverage report
+        if: '!cancelled()'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-report
+          path: coverage_report.txt
+      - name: Upload coverage report
+        if: '!cancelled()'
+        uses: actions/upload-artifact@v3
+        with:
+          name: coverage-report-html
+          path: htmlcov
+      - name: Upload coverage to Compass
+        if: ${{ inputs.report-to-compass }}
+        run: |
+          METRIC_VALUE=$(cat coverage_report.txt | grep 'Total coverage:' | awk '{print $NF}' | sed 's/%//')
+          curl --request POST \
+          --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \
+          --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --data "{
+            \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/1c2a22de-708c-4a73-bebc-c84669a9d32b\",
+            \"value\": $METRIC_VALUE,
+            \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\"
+          }"
+      - name: Upload complexity to Compass
+        if: ${{ inputs.report-to-compass }}
+        run: |
+          METRIC_VALUE=$(poetry run radon cc src --total-average | grep 'Average complexity:' | awk '{print $NF}' | sed 's/[\(\)]//g')
+          curl --request POST \
+          --url https://jacksonlaboratory.atlassian.net/gateway/api/compass/v1/metrics \
+          --user "${{ vars.ATLASSIAN_COMPASS_EMAIL }}:${{ secrets.ATLASSIAN_COMPASS_KEY }}" \
+          --header "Accept: application/json" \
+          --header "Content-Type: application/json" \
+          --data "{
+            \"metricSourceId\": \"ari:cloud:compass:44257862-6c68-4d47-8211-da38d2bb001b:metric-source/90bb0329-f6c5-429a-abbc-8d174535ad21/6cc79ae7-47b1-474d-a1d0-4f78242fc89e\",
+            \"value\": $METRIC_VALUE,
+            \"timestamp\": \"$(date -u +'%Y-%m-%dT%H:%M:%SZ')\"
+          }"
+  comment-coverage-report:
+    needs: [ check_coverage ]
+    runs-on: ubuntu-latest
+    if: ${{always() && github.event_name == 'pull_request'}}
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Download coverage report artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-report
+      - name: Read coverage report
+        id: read-coverage
+        run: |
+          echo "COVERAGE_REPORT<<EOF" >> $GITHUB_ENV
+          cat coverage_report.txt >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: '### Test Coverage Report'
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          body: |
+            ### Test Coverage Report
+            ```
+            ${{ env.COVERAGE_REPORT }}
+            ```

.github/workflows/_format-lint-action.yml

@@ -0,0 +1,32 @@
+name: 'Lint Code Definition'
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: 'Python version to set up'
+        required: true
+        default: '3.9'
+        type: string
+jobs:
+  format-lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          if [[ "$RUNNER_OS" == "macOS" ]]; then
+            echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH
+          fi
+      - name: Configure Poetry
+        run: poetry config virtualenvs.create false
+      - name: Install Black and Ruff
+        run: poetry install --only dev
+      - name: Run Ruff Linter
+        run: ruff src/ tests/
+      - name: Run Black Formatter
+        run: black --check src/ tests/

.github/workflows/_run-tests-action.yml

@@ -0,0 +1,44 @@
+name: 'Python Tests Definition'
+on:
+  workflow_call:
+    inputs:
+      python-version:
+        description: Python version to set up'
+        required: true
+        default: '3.9'
+        type: string
+      runner-os:
+        description: 'Runner OS'
+        required: true
+        default: 'ubuntu-latest'
+        type: string
+      upload-coverage:
+        description: 'Upload coverage results'
+        default: true
+        type: boolean
+      required-coverage:
+        description: 'Required coverage percentage'
+        default: 75
+        type: string
+jobs:
+  run-tests:
+    runs-on: ${{ inputs.runner-os }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ inputs.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ inputs.python-version }}
+      - name: Install Poetry
+        run: |
+          curl -sSL https://install.python-poetry.org | python3 -
+          if [[ "$RUNNER_OS" == "macOS" ]]; then
+            echo "/Users/runner/.local/bin:$PATH" >> $GITHUB_PATH
+          fi
+      - name: Configure Poetry
+        run: poetry config virtualenvs.create false
+      - name: Install dependencies with Poetry
+        run: poetry install
+      - name: Test with pytest
+        run: |
+          poetry run pytest tests

.github/workflows/_skaffold-build-k8s.yml

@@ -0,0 +1,34 @@
+name: 'Skaffold Build'
+on:
+  workflow_call:
+    inputs:
+      default_image_repo:
+        description: 'Default image repo'
+        required: false
+        type: string
+        default: "us-docker.pkg.dev/jax-cs-registry/docker/geneweaver"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCLOUD_REGISTRY_SA_KEY }}'
+      - name: Docker Login
+        run: gcloud auth configure-docker us-docker.pkg.dev,us-east1-docker.pkg.dev
+      - name: Build
+        run: |
+          skaffold build \
+          --default-repo=${{ inputs.default_image_repo }} \
+          --file-output=build.json
+      - name: Upload Build Artifact Information
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-artifact-json
+          path: build.json

.github/workflows/_skaffold-deploy-k8s.yml

@@ -0,0 +1,41 @@
+name: 'Skaffold Deploy'
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: 'Deployment environment/profile'
+        required: true
+        type: string
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download Build Artifact Information
+        uses: actions/download-artifact@v3
+        with:
+          name: build-artifact-json
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCLOUD_CLUSTER_SA_KEY }}'
+      - id: setup-gcloud
+        name: Setup Gcloud
+        uses: 'google-github-actions/setup-gcloud@v1'
+      - id: get-gke-credentials
+        name: Get GKE credentials
+        uses: 'google-github-actions/get-gke-credentials@v1'
+        with:
+          cluster_name: ${{ vars.CLUSTER_NAME }}
+          location: ${{ vars.CLUSTER_REGION }}
+          project_id: ${{ vars.CLUSTER_PROJECT }}
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+      - name: Deploy
+        run: |
+          skaffold deploy \
+          --profile ${{ inputs.environment }} \
+          --build-artifacts=build.json

.github/workflows/coverage.yml

@@ -0,0 +1,15 @@
+name: Coverage
+on:
+  push:
+    branches:
+      - 'main'
+jobs:
+  check-coverage:
+    uses: ./.github/workflows/_check-coverage-action.yml
+    secrets: inherit
+    permissions:
+      pull-requests: write
+    with:
+      required-coverage: ${{ vars.REQUIRED_COVERAGE }}
+      coverage-module: "geneweaver.aon"
+      report-to-compass: true