KANBAN-621 KANBAN-622 central logging - debugging #1981

Workflow file for this run

.github/workflows/PR-validation.yml at 2b69fb0

	name: PR validation
	on:
	pull_request:
	types: [synchronize, opened, reopened, edited, labeled, unlabeled]
	branches:
	- main
	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
	cancel-in-progress: true
	jobs:
	container-build-context:
	name: define container image build context variables
	runs-on: ubuntu-22.04
	outputs:
	tagname: ${{ steps.store_tagname.outputs.tagname }}
	steps:
	- name: Store image tag in env
	id: store_tagname
	shell: bash
	run: \|
	echo "tagname=PR-${{ github.event.pull_request.number }}-validation" >> $GITHUB_OUTPUT
	pipeline-seq-retrieval-update-dependency-lock-files:
	name: pipeline/seq_retrieval update and upload dependency lock file
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./pipeline/seq_retrieval/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/seq_retrieval/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Upload updated (main) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: pipeline_seq_retrieval_deps_lock
	path: pipeline/seq_retrieval/requirements.txt
	if-no-files-found: error
	- name: Upload updated (test) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: pipeline_seq_retrieval_tests_deps_lock
	path: pipeline/seq_retrieval/tests/requirements.txt
	if-no-files-found: error
	api-update-dependency-lock-files:
	name: api update and upload dependency lock file
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./api/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	api/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Upload updated (main) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: api_deps_lock
	path: api/requirements.txt
	if-no-files-found: error
	- name: Upload updated (test) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests/requirements.txt
	if-no-files-found: error
	webui-update-dependency-lock-files:
	name: webui update and upload dependency lock file
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./webui/
	steps:
	- uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	webui/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-lock
	- name: Upload updated lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: webui_deps_lock
	path: webui/package-lock.json
	if-no-files-found: error
	shared-aws-py-package-update-dependency-lock-files:
	name: shared_aws py-package update and upload dependency lock file
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./shared_aws/py_package/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	shared_aws/py_package/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Upload updated (main) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: shared_aws_py_package_deps_lock
	path: shared_aws/py_package/requirements.txt
	if-no-files-found: error
	- name: Upload updated (test) lock file as artifact
	uses: actions/upload-artifact@v4
	with:
	name: shared_aws_py_package_test_deps_lock
	path: shared_aws/py_package/tests/requirements.txt
	if-no-files-found: error
	shared-aws-py-package-code-checks:
	name: shared_aws py-package code checks
	needs:
	- shared-aws-py-package-update-dependency-lock-files
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./shared_aws/py_package/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	shared_aws/py_package/
	- name: Download updated shared_aws py-package prod dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_deps_lock
	path: shared_aws/py_package
	- name: Download updated shared_aws py-package test dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_test_deps_lock
	path: shared_aws/py_package/tests
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Unit tests
	run: make run-unit-tests
	shared-aws-py-package-build:
	name: shared_aws python package build
	needs:
	- shared-aws-py-package-code-checks
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./shared_aws/py_package/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	shared_aws/py_package/
	- name: Download updated shared_aws py-package prod dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_deps_lock
	path: shared_aws/py_package
	- name: Download updated shared_aws py-package test dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_test_deps_lock
	path: shared_aws/py_package/tests
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Build package
	run: \|
	make clean build
	- name: Upload package as artifact
	uses: actions/upload-artifact@v4
	with:
	name: shared_aws_py_package
	path: shared_aws/py_package/dist/pavi_shared_aws-0.0.0-py3-none-any.whl
	shared-aws-infra-update-dependency-lock-files:
	name: shared_aws aws_infra update and upload dependency lock file
	needs:
	- shared-aws-py-package-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./shared_aws/aws_infra/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	shared_aws/aws_infra/
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Bundle all deps lock files
	run: \|
	tar -czv -f shared-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
	- name: Upload deps-lock-files bundle as artifact
	uses: actions/upload-artifact@v4
	with:
	name: shared_aws_infra_deps_locks_bundle
	path: shared_aws/aws_infra/shared-aws-infra-deps-locks.tar.gz
	if-no-files-found: error
	shared-aws-infra-code-checks:
	name: shared_aws infra code checks
	needs:
	- shared-aws-infra-update-dependency-lock-files
	- shared-aws-py-package-build
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./shared_aws/aws_infra/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	shared_aws/aws_infra/
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Download updated shared_aws/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_infra_deps_locks_bundle
	path: shared_aws/aws_infra
	- name: Unpack the bundle
	run: \|
	tar -xzv -f shared-aws-infra-deps-locks.tar.gz
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Unit tests
	run: make run-unit-tests
	- name: Setup node.js (CDK CLI requirement)
	uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
	aws-region: us-east-1
	- name: Validate production CDK stack code
	run: make validate-stack
	pipeline-aws-infra-update-dependency-lock-files:
	name: pipeline/aws_infra update and upload dependency lock file
	needs:
	- shared-aws-py-package-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./pipeline/aws_infra/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/aws_infra/
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Bundle all deps lock files
	run: \|
	tar -czv -f pipeline-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
	- name: Upload deps-lock-files bundle as artifact
	uses: actions/upload-artifact@v4
	with:
	name: pipeline_aws_infra_deps_locks_bundle
	path: pipeline/aws_infra/pipeline-aws-infra-deps-locks.tar.gz
	if-no-files-found: error
	api-aws-infra-update-dependency-lock-files:
	name: api/aws_infra update and upload dependency lock file
	needs:
	- shared-aws-py-package-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./api/aws_infra/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	api/aws_infra/
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Bundle all deps lock files
	run: \|
	tar -czv -f api-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
	- name: Upload deps-lock-files bundle as artifact
	uses: actions/upload-artifact@v4
	with:
	name: api_aws_infra_deps_locks_bundle
	path: api/aws_infra/api-aws-infra-deps-locks.tar.gz
	if-no-files-found: error
	webui-aws-infra-update-dependency-lock-files:
	name: webui/aws_infra update and upload dependency lock file
	needs:
	- shared-aws-py-package-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./webui/aws_infra/
	steps:
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	webui/aws_infra/
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Update lock files
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	shell: bash
	run: \|
	make update-deps-locks-all
	- name: Bundle all deps lock files
	run: \|
	tar -czv -f webui-aws-infra-deps-locks.tar.gz requirements.txt tests/requirements.txt package-lock.json
	- name: Upload deps-lock-files bundle as artifact
	uses: actions/upload-artifact@v4
	with:
	name: webui_aws_infra_deps_locks_bundle
	path: webui/aws_infra/webui-aws-infra-deps-locks.tar.gz
	if-no-files-found: error
	pipeline-seq-retrieval-container-image-build:
	name: pipeline/seq_retrieval container-image build
	needs:
	- container-build-context
	- pipeline-seq-retrieval-update-dependency-lock-files
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/seq_retrieval/
	- name: Download updated seq_retrieval (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: pipeline_seq_retrieval_deps_lock
	path: pipeline/seq_retrieval
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	# This step will configure environment variables to be used by all steps
	# involving AWS interaction further down
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-seq-retrieval-image-upload
	aws-region: us-east-1
	- name: Amazon ECR login
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@v2
	- name: Build container image
	uses: docker/build-push-action@v6
	env:
	DOCKER_BUILD_SUMMARY: false
	with:
	context: ./pipeline/seq_retrieval/
	push: false
	tags: agr_pavi/pipeline_seq_retrieval:latest
	outputs: type=docker,dest=/tmp/pavi_seq_retrieval_docker_image.tar
	- name: Upload image as artifact (share between jobs)
	uses: actions/upload-artifact@v4
	with:
	name: seq_retrieval_image
	path: /tmp/pavi_seq_retrieval_docker_image.tar
	- name: Load, tag and push image to registry
	run: \|
	docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
	docker tag agr_pavi/pipeline_seq_retrieval:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
	docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_seq_retrieval:${{needs.container-build-context.outputs.tagname}}
	pipeline-seq-retrieval-code-checks:
	name: pipeline/seq_retrieval code checks
	needs:
	- pipeline-seq-retrieval-update-dependency-lock-files
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./pipeline/seq_retrieval/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/seq_retrieval/
	- name: Download updated seq_retrieval (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: pipeline_seq_retrieval_deps_lock
	path: pipeline/seq_retrieval
	- name: Download updated seq_retrieval (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: pipeline_seq_retrieval_tests_deps_lock
	path: pipeline/seq_retrieval/tests
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Run unit tests
	run: \|
	make run-unit-tests
	pipeline-alignment-container-image-build:
	name: pipeline/alignment container-image build
	needs:
	- container-build-context
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/alignment/
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	# This step will configure environment variables to be used by all steps
	# involving AWS interaction further down
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-alignment-image-upload
	aws-region: us-east-1
	- name: Amazon ECR login
	id: login-ecr
	uses: aws-actions/amazon-ecr-login@v2
	- name: Build container image
	uses: docker/build-push-action@v6
	env:
	DOCKER_BUILD_SUMMARY: false
	with:
	context: ./pipeline/alignment/
	push: false
	tags: agr_pavi/pipeline_alignment:latest
	outputs: type=docker,dest=/tmp/pavi_alignment_docker_image.tar
	- name: Upload image as artifact (share between jobs)
	uses: actions/upload-artifact@v4
	with:
	name: alignment_image
	path: /tmp/pavi_alignment_docker_image.tar
	- name: Load, tag and push image to registry
	run: \|
	docker load --input /tmp/pavi_alignment_docker_image.tar
	docker tag agr_pavi/pipeline_alignment:latest ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
	docker push ${{ steps.login-ecr.outputs.registry }}/agr_pavi/pipeline_alignment:${{needs.container-build-context.outputs.tagname}}
	pipeline-alignment-unit-testing:
	name: pipeline/alignment unit testing
	needs:
	- pipeline-alignment-container-image-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./pipeline/alignment/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/alignment/
	- name: Download alignment image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: alignment_image
	path: /tmp
	- name: Load alignment Docker image
	run: \|
	docker load --input /tmp/pavi_alignment_docker_image.tar
	- name: Run unit test
	run: \|
	make run-unit-tests
	pipeline-workflow-integration-testing:
	name: pipeline/workflow integration testing
	needs:
	- container-build-context
	- pipeline-seq-retrieval-container-image-build
	- pipeline-alignment-container-image-build
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./pipeline/workflow/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/workflow/
	tests/resources/
	- name: Download seq_retrieval image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: seq_retrieval_image
	path: /tmp
	- name: Download alignment image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: alignment_image
	path: /tmp
	- name: Load seq_retrieval Docker image
	run: \|
	docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
	- name: Load alignment Docker image
	run: \|
	docker load --input /tmp/pavi_alignment_docker_image.tar
	- name: Run local integration test
	run: \|
	make run-integration-test-local
	# This step will configure environment variables to be used by all steps
	# involving AWS interaction further down
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-workflow-aws-test
	aws-region: us-east-1
	- name: Run AWS integration test
	run: \|
	make run-integration-test-aws TAG_NAME=${{needs.container-build-context.outputs.tagname}}
	pipeline-aws-infra-code-checks:
	name: pipeline/aws_infra code checks
	needs:
	- pipeline-aws-infra-update-dependency-lock-files
	- shared-aws-py-package-build
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./pipeline/aws_infra
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	pipeline/aws_infra/
	- name: Download updated pipeline/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: pipeline_aws_infra_deps_locks_bundle
	path: pipeline/aws_infra
	- name: Unpack the bundle
	working-directory: pipeline/aws_infra/
	run: \|
	tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Setup node.js (CDK CLI requirement)
	uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
	aws-region: us-east-1
	- name: Validate production CDK stack code
	run: make validate
	api-code-checks:
	name: API code checks
	needs:
	- api-update-dependency-lock-files
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./api/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	api/
	- name: Download updated api (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_deps_lock
	path: api
	- name: Download updated api (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	api-container-image-build:
	name: API container-image build
	needs:
	- api-update-dependency-lock-files
	runs-on: ubuntu-22.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: Download updated api (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_deps_lock
	path: api
	- name: Download updated api (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	- name: Build container image
	uses: docker/build-push-action@v6
	env:
	DOCKER_BUILD_SUMMARY: false
	with:
	context: ./
	file: api/Dockerfile
	push: false
	tags: agr_pavi/api:latest
	outputs: type=docker,dest=/tmp/pavi_api_docker_image.tar
	- name: Upload image as artifact (share between jobs)
	uses: actions/upload-artifact@v4
	with:
	name: api_image
	path: /tmp/pavi_api_docker_image.tar
	api-unit-integration-testing:
	name: API unit and integration testing
	needs:
	- api-update-dependency-lock-files
	- pipeline-seq-retrieval-container-image-build
	- pipeline-alignment-container-image-build
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./api/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: Download updated api (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_deps_lock
	path: api
	- name: Download updated api (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests
	- name: Download seq_retrieval image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: seq_retrieval_image
	path: /tmp
	- name: Download alignment image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: alignment_image
	path: /tmp
	- name: Load seq_retrieval Docker image
	run: \|
	docker load --input /tmp/pavi_seq_retrieval_docker_image.tar
	- name: Load alignment Docker image
	run: \|
	docker load --input /tmp/pavi_alignment_docker_image.tar
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	- name: Run unit and integration tests
	run: \|
	make run-tests
	api-container-integration-testing:
	name: API container integration testing (AWS batch execution)
	needs:
	- container-build-context
	- pipeline-seq-retrieval-container-image-build
	- pipeline-alignment-container-image-build
	- api-container-image-build
	- api-update-dependency-lock-files
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./api/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	api/
	tests/resources/
	- name: Download updated api (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_deps_lock
	path: api
	- name: Download updated api (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests
	- name: Download API image artifact (from previous job)
	uses: actions/download-artifact@v4
	with:
	name: api_image
	path: /tmp
	- name: Load API Docker image
	run: \|
	docker load --input /tmp/pavi_api_docker_image.tar
	- uses: actions/setup-python@v5
	with:
	python-version: '3.12'
	# This step will configure environment variables to be used by all steps
	# involving AWS interaction further down
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-api-aws-test
	aws-region: us-east-1
	- name: Run container integration test (AWS execution)
	run: \|
	make run-integration-test-container TAG_NAME=${{needs.container-build-context.outputs.tagname}}
	api-aws-infra-code-checks:
	name: api/aws_infra code checks
	needs:
	- api-aws-infra-update-dependency-lock-files
	- shared-aws-py-package-build
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./api/aws_infra
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	api/aws_infra/
	- name: Download updated api/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: api_aws_infra_deps_locks_bundle
	path: api/aws_infra
	- name: Unpack the bundle
	working-directory: api/aws_infra/
	run: \|
	tar -xzv -f api-aws-infra-deps-locks.tar.gz
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Setup node.js (CDK CLI requirement)
	uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
	aws-region: us-east-1
	- name: Validate production CDK stack code
	run: make validate-all
	webui-aws-infra-code-checks:
	name: webui/aws_infra code checks
	needs:
	- webui-aws-infra-update-dependency-lock-files
	- shared-aws-py-package-build
	permissions:
	id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions
	runs-on: ubuntu-22.04
	defaults:
	run:
	working-directory: ./webui/aws_infra
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	webui/aws_infra/
	- name: Download updated webui/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: webui_aws_infra_deps_locks_bundle
	path: webui/aws_infra
	- name: Unpack the bundle
	working-directory: webui/aws_infra/
	run: \|
	tar -xzv -f webui-aws-infra-deps-locks.tar.gz
	- name: Download shared AWS package
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package
	path: /tmp/
	- name: Setup Python
	uses: actions/setup-python@v5
	with:
	python-version: "3.12"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Setup node.js (CDK CLI requirement)
	uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test
	aws-region: us-east-1
	- name: Validate production CDK stack code
	run: make validate-all
	webui-code-checks:
	name: webUI code checks
	needs:
	- webui-update-dependency-lock-files
	runs-on: ubuntu-22.04
	defaults:
	run:
	shell: bash
	working-directory: ./webui/
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	sparse-checkout: \|
	webui/
	- name: Download updated webui dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: webui_deps_lock
	path: webui
	- uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: Code typing test
	run: \|
	make run-type-checks
	- name: Code style test
	run: \|
	make run-style-checks
	- name: Unit tests
	run: make run-unit-tests
	webui-container-image-build:
	name: webUI container-image build
	needs:
	- webui-update-dependency-lock-files
	runs-on: ubuntu-22.04
	steps:
	- name: Check out repository code
	uses: actions/checkout@v4
	with:
	fetch-depth: 0
	- name: Download updated webui dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: webui_deps_lock
	path: webui
	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@v3
	- name: Build container image
	uses: docker/build-push-action@v6
	env:
	DOCKER_BUILD_SUMMARY: false
	with:
	context: ./webui/
	file: webui/Dockerfile
	push: false
	tags: agr_pavi/webui:latest
	outputs: type=docker,dest=/tmp/pavi_webui_docker_image.tar
	- name: Upload image as artifact (share between jobs)
	uses: actions/upload-artifact@v4
	with:
	name: webui_image
	path: /tmp/pavi_webui_docker_image.tar
	e2e-testing:
	name: end-to-end testing
	needs:
	- webui-update-dependency-lock-files
	- webui-container-image-build
	- api-container-image-build
	permissions:
	id-token: write # Required for OIDC authentication to AWS (to perform AWS actions / access AGR AWS resources)
	runs-on: ubuntu-22.04
	steps:
	# Configure environment variables used for AWS interaction,
	# required for API container interactions with pipeline resources
	- name: AWS credentials configuration
	uses: aws-actions/configure-aws-credentials@v4
	with:
	role-to-assume: ${{ secrets.GH_ACTIONS_AWS_ROLE }}
	role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-e2e-testing
	aws-region: us-east-1
	- name: Check out repository code
	uses: actions/checkout@v4
	- name: Download API image artifact
	uses: actions/download-artifact@v4
	with:
	name: api_image
	path: /tmp
	- name: Load API Docker image
	run: \|
	docker load --input /tmp/pavi_api_docker_image.tar
	- name: Run local API container to run E2E tests on
	working-directory: api/
	run: \|
	make run-container-dev
	- name: Download webUI image artifact
	uses: actions/download-artifact@v4
	with:
	name: webui_image
	path: /tmp
	- name: Load webUI Docker image
	run: \|
	docker load --input /tmp/pavi_webui_docker_image.tar
	- name: Run local webUI container to run E2E tests on
	working-directory: webui/
	run: \|
	PAVI_API_PORT=8080 make run-container-dev
	- name: Download updated webui dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: webui_deps_lock
	path: webui
	- name: setup webui-compatible node.js version
	uses: actions/setup-node@v4
	with:
	node-version: "20"
	- name: Run cypress E2E tests
	uses: cypress-io/github-action@v6
	with:
	working-directory: webui/
	wait-on: 'http://localhost:8080/api/health, http://localhost:3000/health'
	env: 'API_BASE_URL=http://localhost:8080'
	- name: Report API container logs on E2E test failure
	if: failure()
	run: \|
	docker logs agr.pavi.dev-local.api.server
	- name: Report webUI container logs on E2E test failure
	if: failure()
	run: \|
	docker logs agr.pavi.dev-local.webui.server
	- name: Cleanup webUI server (running container)
	if: success() \|\| failure()
	working-directory: webui/
	run: \|
	make stop-container-dev
	- name: Cleanup API server (running container)
	if: success() \|\| failure()
	working-directory: api/
	run: \|
	make stop-container-dev
	stage-deps-lock-updates:
	if: ${{ !contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	runs-on: ubuntu-22.04
	permissions:
	pull-requests: write
	# Always stage updated lock files, even on failure,
	# to highlight dependency changes which could be the cause of the failure.
	needs:
	- shared-aws-py-package-update-dependency-lock-files
	- shared-aws-infra-update-dependency-lock-files
	- api-aws-infra-update-dependency-lock-files
	- pipeline-aws-infra-update-dependency-lock-files
	- webui-aws-infra-update-dependency-lock-files
	- api-update-dependency-lock-files
	- pipeline-seq-retrieval-update-dependency-lock-files
	- webui-update-dependency-lock-files
	steps:
	- uses: actions/checkout@v4
	with:
	ref: ${{ github.head_ref }}
	fetch-depth: 0
	- name: Download updated seq_retrieval (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: pipeline_seq_retrieval_deps_lock
	path: pipeline/seq_retrieval
	- name: Download updated seq_retrieval (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: pipeline_seq_retrieval_tests_deps_lock
	path: pipeline/seq_retrieval/tests
	- name: Download updated api (main) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_deps_lock
	path: api
	- name: Download updated api (test) dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: api_tests_deps_lock
	path: api/tests
	- name: Download updated pipeline/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: pipeline_aws_infra_deps_locks_bundle
	path: pipeline/aws_infra
	- name: Unpack the bundle
	working-directory: pipeline/aws_infra/
	run: \|
	tar -xzv -f pipeline-aws-infra-deps-locks.tar.gz
	- name: Download updated shared_aws py-package prod dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_deps_lock
	path: shared_aws/py_package
	- name: Download updated shared_aws py-package test dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_py_package_test_deps_lock
	path: shared_aws/py_package/tests
	- name: Download updated shared_aws/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: shared_aws_infra_deps_locks_bundle
	path: shared_aws/aws_infra
	- name: Unpack the bundle
	working-directory: shared_aws/aws_infra/
	run: \|
	tar -xzv -f shared-aws-infra-deps-locks.tar.gz
	- name: Download updated api/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: api_aws_infra_deps_locks_bundle
	path: api/aws_infra
	- name: Unpack the bundle
	working-directory: api/aws_infra/
	run: \|
	tar -xzv -f api-aws-infra-deps-locks.tar.gz
	- name: Download updated webui/aws_infra dependencies lock files bundle
	uses: actions/download-artifact@v4
	with:
	name: webui_aws_infra_deps_locks_bundle
	path: webui/aws_infra
	- name: Unpack the bundle
	working-directory: webui/aws_infra/
	run: \|
	tar -xzv -f webui-aws-infra-deps-locks.tar.gz
	- name: Download updated webui dependencies lock file
	uses: actions/download-artifact@v4
	with:
	name: webui_deps_lock
	path: webui
	- name: Bundle all deps lock files
	run: \|
	find . -regex '.*/$requirements\.txt\\|package-lock\.json$' -print \| \
	tar -czv -f deps-lock-files.tar.gz --files-from -
	# Upload the deps-lock-files bundle as workflow artifact (to be downloaded on PR merge)
	- name: Upload deps-lock-files bundle as artifact
	uses: actions/upload-artifact@v4
	with:
	name: deps_lock_files_bundle
	path: deps-lock-files.tar.gz
	# Share diff of changes in PR
	- name: Generate list of files that were updated
	id: diff-files-list
	run: \|
	git diff --name-only -- > diff-files.txt
	{
	echo 'content<<EOF'
	cat diff-files.txt
	echo EOF
	} >> "$GITHUB_OUTPUT"
	- name: Calculate diff for each individual file
	id: diff-files-content
	if: ${{ steps.diff-files-list.outputs.content }}
	run: \|
	mkdir diff-files/
	cat diff-files.txt \| xargs -I {} bash ci-scripts/create-git-diff-markdown.sh {} diff-files
	{
	echo 'total_char_count<<EOF'
	find ./diff-files/ -type f -exec cat {} \; \| wc -m
	echo EOF
	} >> "$GITHUB_OUTPUT"
	- name: Generate and store change message summary
	if: ${{ steps.diff-files-list.outputs.content }}
	run: \|
	{
	echo 'The validation workflow automatically applied the below dependency lock file updates.'
	echo 'A bundle with all dependency lock files is available [here](${{ github.event.repository.html_url }}/actions/runs/${{ github.run_id }}) as artifact called `deps_lock_files_bundle`.'
	} > git-diff-message.md
	# gh pr addComment API has a max 65536 character limit
	- name: Report all diffs in a single comment
	if: ${{ steps.diff-files-content.outputs.total_char_count < 60000 }}
	run: \|
	{
	echo ''
	cat diff-files.txt \| xargs -I {} cat diff-files/{}.md
	} >> git-diff-message.md
	- name: Generate and store no-change message
	if: ${{ !steps.diff-files-list.outputs.content }}
	run: \|
	echo 'Automatic dependency lock file updates enabled, but no updates found. No (additional) dependency updates will be made when merging this PR.' \
	> git-diff-message.md
	- name: Post PR comment with change message
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file git-diff-message.md
	- name: Report each file diff in a separate comment
	if: ${{ steps.diff-files-content.outputs.total_char_count >= 60000 }}
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	cat diff-files.txt \| xargs -I {} gh pr comment ${{ github.event.pull_request._links.html.href }} --body-file diff-files/{}.md
	report-no-deps-lock-updates-label:
	if: ${{ contains(github.event.pull_request.labels.*.name, 'no-deps-lock-updates') }}
	runs-on: ubuntu-22.04
	permissions:
	pull-requests: write
	steps:
	- name: Post PR comment with notification no updates will be made
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	gh pr comment ${{ github.event.pull_request._links.html.href }} --body '`no-deps-lock-updates` label detected, so automatic dependency lock file updates are disabled. No (additional) dependency updates will be made when merging this PR.'

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

KANBAN-621 KANBAN-622 central logging - debugging #1981

Workflow file

KANBAN-621 KANBAN-622 central logging - debugging #1981

Jobs

Run details

Workflow file for this run