Skip to content

Commit

Permalink
Merge pull request #121 from alliance-genome/webUI-deployment-KANBAN-583
Browse files Browse the repository at this point in the history 
AWS infra components and automations for web UI image deployment KANBAN-583
  • Loading branch information
@mluypaert
mluypaert authored Jul 5, 2024
2 parents 7183482 + 875c1aa commit e586a0d
Show file tree
Hide file tree
Showing 50 changed files with 2,337 additions and 591 deletions.
315 changes: 225 additions & 90 deletions .github/workflows/PR-validation.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,114 @@ jobs:
name: webui_deps_lock
path: webui/package-lock.json
if-no-files-found: error
shared-aws-infra-update-dependency-lock-files:
name: shared_aws_infra update and upload dependency lock file
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./shared_aws_infra/
steps:
- uses: actions/setup-python@v5
with:
python-version: '3.12'
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
run: |
make pip-tools update-deps-lock update-test-deps-lock
- name: Upload updated (main) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra/requirements.txt
if-no-files-found: error
- name: Upload updated (test) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests/requirements.txt
if-no-files-found: error
shared-aws-infra-code-checks:
name: shared_aws_infra code checks
needs:
- shared-aws-infra-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
- name: Download updated shared_aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra
- name: Download updated shared_aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
shared-aws-infra-package-build:
name: shared_aws_infra python package build
needs:
- shared-aws-infra-code-checks
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
- name: Download updated shared_aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra
- name: Download updated shared_aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build package
run: |
make clean build
- name: Upload package as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_package
path: shared_aws_infra/dist/pavi_shared_aws_infra-0.0.0-py3-none-any.whl
pipeline-aws-infra-update-dependency-lock-files:
name: pipeline/aws_infra update and upload dependency lock file
needs:
Expand Down Expand Up @@ -197,13 +305,15 @@ jobs:
name: api_aws_infra_tests_deps_lock
path: api/aws_infra/tests/requirements.txt
if-no-files-found: error
shared-aws-infra-update-dependency-lock-files:
name: shared_aws_infra update and upload dependency lock file
webui-aws-infra-update-dependency-lock-files:
name: webui/aws_infra update and upload dependency lock file
needs:
- shared-aws-infra-package-build
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
working-directory: ./shared_aws_infra/
working-directory: ./webui/aws_infra/
steps:
- uses: actions/setup-python@v5
with:
Expand All @@ -213,7 +323,12 @@ jobs:
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
webui/aws_infra/
- name: Download shared AWS infra package
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_package
path: /tmp/
- name: Update lock files
if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
shell: bash
Expand All @@ -222,89 +337,15 @@ jobs:
- name: Upload updated (main) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra/requirements.txt
name: webui_aws_infra_deps_lock
path: webui/aws_infra/requirements.txt
if-no-files-found: error
- name: Upload updated (test) lock file as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests/requirements.txt
name: webui_aws_infra_tests_deps_lock
path: webui/aws_infra/tests/requirements.txt
if-no-files-found: error
shared-aws-infra-code-checks:
name: shared_aws_infra code checks
needs:
- shared-aws-infra-update-dependency-lock-files
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
- name: Download updated shared_aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra
- name: Download updated shared_aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Unit tests
run: make run-unit-tests
shared-aws-infra-package-build:
name: shared_aws_infra python package build
needs:
- shared-aws-infra-code-checks
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./shared_aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
shared_aws_infra/
- name: Download updated shared_aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_deps_lock
path: shared_aws_infra
- name: Download updated shared_aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_test_deps_lock
path: shared_aws_infra/tests
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build package
run: |
make build
- name: Upload package as artifact
uses: actions/upload-artifact@v4
with:
name: shared_aws_infra_package
path: shared_aws_infra/dist/pavi_shared_aws_infra-0.0.0-py3-none-any.whl
pipeline-seq-retrieval-container-image-build:
name: pipeline/seq_retrieval container-image build
needs:
Expand Down Expand Up @@ -806,6 +847,65 @@ jobs:
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-all
webui-aws-infra-code-checks:
name: webui/aws_infra code checks
needs:
- webui-aws-infra-update-dependency-lock-files
- shared-aws-infra-package-build
permissions:
id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
runs-on: ubuntu-22.04
defaults:
run:
working-directory: ./webui/aws_infra
steps:
- name: Check out repository code
uses: actions/checkout@v4
with:
fetch-depth: 0
sparse-checkout: |
webui/aws_infra/
- name: Download updated webui/aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_deps_lock
path: webui/aws_infra
- name: Download updated webui/aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_tests_deps_lock
path: webui/aws_infra/tests
- name: Download shared AWS infra package
uses: actions/download-artifact@v4
with:
name: shared_aws_infra_package
path: /tmp/
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Code typing test
run: |
make run-type-checks
- name: Code style test
run: |
make run-style-checks
- name: Setup node.js (CDK requirement)
uses: actions/setup-node@v4
with:
node-version: "18"
- name: Install CDK CLI
run: npm install -g aws-cdk
- name: Install CDK stack dependencies
run: pip install -r requirements.txt
- name: AWS credentials configuration
uses: aws-actions/configure-aws-credentials@v4
with:
role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
aws-region: us-east-1
- name: Validate production CDK stack code
run: make validate-all
webui-code-checks:
name: webUI code checks
needs:
Expand Down Expand Up @@ -870,19 +970,21 @@ jobs:
runs-on: ubuntu-22.04
#Only commit updated lock files on successfull validation (to prevent PR cluttering)
needs:
- pipeline-seq-retrieval-update-dependency-lock-files
- pipeline-seq-retrieval-code-checks
- shared-aws-infra-update-dependency-lock-files
- shared-aws-infra-package-build
- pipeline-aws-infra-update-dependency-lock-files
- pipeline-aws-infra-code-checks
- pipeline-workflow-integration-testing
- api-aws-infra-update-dependency-lock-files
- api-aws-infra-code-checks
- pipeline-aws-infra-update-dependency-lock-files
- pipeline-aws-infra-code-checks
- webui-aws-infra-update-dependency-lock-files
- webui-aws-infra-code-checks
- api-update-dependency-lock-files
- api-container-integration-testing
- api-unit-integration-testing
- api-code-checks
- pipeline-seq-retrieval-update-dependency-lock-files
- pipeline-seq-retrieval-code-checks
- pipeline-workflow-integration-testing
- webui-update-dependency-lock-files
- webui-code-checks
- webui-container-image-build
Expand All @@ -892,6 +994,7 @@ jobs:
- uses: actions/checkout@v4
with:
ref: ${{ github.head_ref }}
fetch-depth: 0
- name: Download updated seq_retrieval (main) dependencies lock file
uses: actions/download-artifact@v4
with:
Expand Down Expand Up @@ -942,14 +1045,46 @@ jobs:
with:
name: api_aws_infra_tests_deps_lock
path: api/aws_infra/tests
- name: Download updated webui/aws_infra (main) dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_deps_lock
path: webui/aws_infra
- name: Download updated webui/aws_infra (test) dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_aws_infra_tests_deps_lock
path: webui/aws_infra/tests
- name: Download updated webui dependencies lock file
uses: actions/download-artifact@v4
with:
name: webui_deps_lock
path: webui
# Commit all changed dependency lock files back to the open PR
- uses: stefanzweifel/git-auto-commit-action@v5
# Independent shared_aws_infra lock files commit required to pin hash representing the package
# to be included in depending aws_infra components' lock files.
- name: commit shared_aws_infra dependency lock file changes
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: Auto-updated pavi_shared_aws_infra deps lock files
file_pattern: 'shared_aws_infra/requirements.txt shared_aws_infra/tests/requirements.txt'
disable_globbing: true
# Build pavi_shared_aws_infra package (to ensure hash includes latest commit date)
- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Build and install the pavi_shared_aws_infra package
working-directory: ./shared_aws_infra
run: |
make clean build install
- name: Update pavi_shared_aws_infra dependencies
run: |
make -C api/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only
make -C pipeline/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only
make -C webui/aws_infra/ pip-tools update-deps-lock-shared-aws-infra-only update-test-deps-lock-shared-aws-infra-only
- name: commit remaining dependency lock file changes
uses: stefanzweifel/git-auto-commit-action@v5
with:
commit_message: Auto-updated deps lock files
file_pattern: '*requirements.txt'
file_pattern: '*requirements.txt *package-lock.json'
disable_globbing: true
Loading

0 comments on commit e586a0d

Please sign in to comment.