Merge pull request #4 from TheJacksonLaboratory/G3-5-deploy-geneweave…

…r-api-to-shared-clusters G3 5 deploy geneweaver api to shared clusters
TheJacksonLaboratory · Dec 13, 2023 · cc5b10a · cc5b10a
2 parents 925d4dc + d06cf89
commit cc5b10a
Show file tree

Hide file tree

Showing 27 changed files with 568 additions and 90 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -0,0 +1,13 @@
+deploy/
+htmlcov/
+.github/
+.env/
+.python-version
+skaffold.yaml
+
+
+*.pyc
+*.pyo
+*.pyd
+__pycache__
+.git
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
@@ -0,0 +1,72 @@
+# GitHub Actions
+
+The files in this directory are used to configure the Github Actions workflows for
+`geneweaver-api`. The workflows are used to automatically build and test the project
+when changes are pushed to the repository.
+
+Any file that starts with an underscore (`_`) is a "reusable workflow". These files
+are not directly used by GitHub Actions, but are instead referenced by the workflows
+files that do not start with an underscore.
+
+There are five reusable workflows:
+
+- Check Coverage (`_check-coverage-action.yml`): This workflow is used to check the code
+  coverage of the project.
+- Format Lint (`_format-lint-action.yml`): This workflow is used to check the formatting
+  and linting of the project.
+- Run Tests (`_run-tests-action.yml`): This workflow is used to run the tests for the
+  project.
+- Skaffold Build (`_skaffold-build-k8s.yml`): This workflow is used to build the
+  Docker images for the project.
+- Skaffold Deploy (`_skaffold-deploy-action.yml`): This workflow is used to deploy the
+  Docker images to kubernetes.
+
+There are two _main_ workflows that are used by GitHub Actions:
+
+- Pull Requests (`pull_requests.yml`): This workflow is used to build and test the
+  project when a pull request is opened.
+- Release (`release.yml`): This workflow that is run whenever the version number changes
+  on the `main` branch.
+
+There are also three quality assurance workflows that are run on any change to the main
+branch:
+
+- Coverage (`coverage.yml`): This workflow is used to check the code coverage of the
+  project.
+- Style (`style.yml`): This workflow is used to check the formatting and linting of the
+  project.
+- Tests (`tests.yml`): This workflow is used to run the tests for the project.
+
+
+## Pull Requests
+
+The pull request workflow is run whenever a pull request is opened. This workflow
+will:
+
+- Check the formatting and linting of the project.
+- Run the tests for the project.
+- Check the code coverage of the project.
+- Build the Docker images (into the `test` registry) for the project.
+- Deploy the Docker images (from the `test` registry) to kubernetes (into the `dev` 
+  environment).
+
+## Release
+
+The release workflow is run whenever the version number changes on the `main` branch.
+This workflow will:
+
+- Check the formatting and linting of the project.
+- Run the tests for the project.
+- Check the code coverage of the project.
+- Build the Docker images (into the `prod` registry) for the project.
+- Deploy the Docker images (from the `prod` registry) to kubernetes (into the `sqa` 
+  environment).
+  - It will wait for approval from SQA before running this step
+- If the version number is not a pre-release version (contains a letter) it will then:
+    - Deploy the Docker images (from the `prod` registry) to kubernetes (into the 
+      `stage` environment).
+      - It will wait for approval from SQA before running this step
+    - Deploy the Docker images (from the `prod` registry) to kubernetes (into the `prod` 
+      environment).
+      - It will wait for approval from SQA before running this step
+    - It will then create a draft GitHub release
diff --git a/.github/workflows/_check-coverage-action.yml b/.github/workflows/_check-coverage-action.yml
@@ -27,7 +27,7 @@ on:
         type: string
         default: "no"
 jobs:
-  run-tests:
+  check_coverage:
     runs-on: ${{ inputs.runner-os }}
     steps:
       - uses: actions/checkout@v3
@@ -54,12 +54,49 @@ jobs:
           --cov-report=html \
           --cov-fail-under=${{ inputs.required-coverage }} > coverage_report.txt
       - name: Upload coverage report
+        if: '!cancelled()'
         uses: actions/upload-artifact@v3
         with:
           name: coverage-report
           path: coverage_report.txt
       - name: Upload coverage report
+        if: '!cancelled()'
         uses: actions/upload-artifact@v3
         with:
           name: coverage-report-html
-          path: htmlcov
+          path: htmlcov
+  comment-coverage-report:
+    needs: [ check_coverage ]
+    runs-on: ubuntu-latest
+    if: ${{always() && github.event_name == 'pull_request'}}
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Download coverage report artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: coverage-report
+      - name: Read coverage report
+        id: read-coverage
+        run: |
+          echo "COVERAGE_REPORT<<EOF" >> $GITHUB_ENV
+          cat coverage_report.txt >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Find Comment
+        uses: peter-evans/find-comment@v2
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: '### Test Coverage Report'
+      - name: Create or update comment
+        uses: peter-evans/create-or-update-comment@v3
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          edit-mode: replace
+          body: |
+            ### Test Coverage Report
+            ```
+            ${{ env.COVERAGE_REPORT }}
+            ```
diff --git a/.github/workflows/_skaffold-build-k8s.yml b/.github/workflows/_skaffold-build-k8s.yml
@@ -0,0 +1,34 @@
+name: 'Skaffold Build'
+on:
+  workflow_call:
+    inputs:
+      default_image_repo:
+        description: 'Default image repo'
+        required: false
+        type: string
+        default: "us-docker.pkg.dev/jax-cs-registry/docker/geneweaver"
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCLOUD_REGISTRY_SA_KEY }}'
+      - name: Docker Login
+        run: gcloud auth configure-docker us-docker.pkg.dev,us-east1-docker.pkg.dev
+      - name: Build
+        run: |
+          skaffold build \
+          --default-repo=${{ inputs.default_image_repo }} \
+          --file-output=build.json
+      - name: Upload Build Artifact Information
+        uses: actions/upload-artifact@v3
+        with:
+          name: build-artifact-json
+          path: build.json
diff --git a/.github/workflows/_skaffold-deploy-k8s.yml b/.github/workflows/_skaffold-deploy-k8s.yml
@@ -0,0 +1,41 @@
+name: 'Skaffold Deploy'
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: 'Deployment environment/profile'
+        required: true
+        type: string
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download Build Artifact Information
+        uses: actions/download-artifact@v3
+        with:
+          name: build-artifact-json
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v1'
+        with:
+          credentials_json: '${{ secrets.GCLOUD_CLUSTER_SA_KEY }}'
+      - id: setup-gcloud
+        name: Setup Gcloud
+        uses: 'google-github-actions/setup-gcloud@v1'
+      - id: get-gke-credentials
+        name: Get GKE credentials
+        uses: 'google-github-actions/get-gke-credentials@v1'
+        with:
+          cluster_name: ${{ vars.CLUSTER_NAME }}
+          location: ${{ vars.CLUSTER_REGION }}
+          project_id: ${{ vars.CLUSTER_PROJECT }}
+      - name: Install Skaffold
+        run: |
+          curl -Lo skaffold https://storage.googleapis.com/skaffold/releases/latest/skaffold-linux-amd64 && \
+          sudo install skaffold /usr/local/bin/
+      - name: Deploy
+        run: |
+          skaffold deploy \
+          --profile ${{ inputs.environment }} \
+          --build-artifacts=build.json
diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml
@@ -1,49 +1,13 @@
 name: Coverage
 on:
-  pull_request:
-    branches:
-      - 'main'
   push:
     branches:
       - 'main'
 jobs:
   check-coverage:
     uses: ./.github/workflows/_check-coverage-action.yml
+    permissions:
+      pull-requests: write
     with:
       required-coverage: ${{ vars.REQUIRED_COVERAGE }}
       coverage-module: "geneweaver.api"
-  comment-coverage-report:
-    needs: [ check-coverage ]
-    runs-on: ubuntu-latest
-    if: github.event_name == 'pull_request'
-    permissions:
-      pull-requests: write
-    steps:
-      - name: Download coverage report artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: coverage-report
-      - name: Read coverage report
-        id: read-coverage
-        run: |
-          echo "COVERAGE_REPORT<<EOF" >> $GITHUB_ENV
-          cat coverage_report.txt >> $GITHUB_ENV
-          echo "EOF" >> $GITHUB_ENV
-      - name: Find Comment
-        uses: peter-evans/find-comment@v2
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: 'github-actions[bot]'
-          body-includes: '### Test Coverage Report'
-      - name: Create or update comment
-        uses: peter-evans/create-or-update-comment@v3
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          edit-mode: replace
-          body: |
-            ### Test Coverage Report
-            ```
-            ${{ env.COVERAGE_REPORT }}
-            ```
diff --git a/.github/workflows/pull_requests.yml b/.github/workflows/pull_requests.yml
@@ -0,0 +1,46 @@
+name: Pull Request Test, Build and Deploy
+on:
+  pull_request:
+    branches:
+      - 'main'
+jobs:
+  format-lint:
+    name: "Format and Lint"
+    uses: ./.github/workflows/_format-lint-action.yml
+    with:
+      python-version: '3.9'
+  check-coverage:
+    name: "Check Coverage"
+    needs: [format-lint]
+    uses: ./.github/workflows/_check-coverage-action.yml
+    permissions:
+      pull-requests: write
+    with:
+      required-coverage: ${{ vars.REQUIRED_COVERAGE }}
+      coverage-module: "geneweaver.api"
+  test:
+    name: "Run Tests"
+    needs: [format-lint]
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        python-version: ['3.10', '3.11']
+    uses: ./.github/workflows/_run-tests-action.yml
+    with:
+      runner-os: ${{ matrix.os }}
+      python-version: ${{ matrix.python-version }}
+      required-coverage: ${{ vars.REQUIRED_COVERAGE }}
+  build:
+    name: "Build: Dev"
+    needs: [test, check-coverage]
+    uses: ./.github/workflows/_skaffold-build-k8s.yml
+    secrets: inherit
+    with:
+      default_image_repo: "us-east1-docker.pkg.dev/jax-cs-registry/docker-test/geneweaver"
+  deploy:
+    name: "Deploy: Dev"
+    needs: [build]
+    uses: ./.github/workflows/_skaffold-deploy-k8s.yml
+    secrets: inherit
+    with:
+      environment: "jax-cluster-dev-10--dev"