diff --git a/.github/workflows/PR-validation.yml b/.github/workflows/PR-validation.yml index 0ad99721..d965b13b 100644 --- a/.github/workflows/PR-validation.yml +++ b/.github/workflows/PR-validation.yml @@ -151,3 +151,34 @@ jobs: - name: Run integration test run: | make run-integration-test + pipeline-aws-infra-cdk-test: + name: pipeline/aws_infra CDK test + permissions: + id-token: write # This is required for requesting the JWT for gaining permissions to assume the IAM role to perform AWS actions + runs-on: ubuntu-22.04 + defaults: + run: + working-directory: pipeline/aws_infra + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: Setup node.js (CDK requirement) + uses: actions/setup-node@v4 + with: + node-version: "18" + - name: Install CDK + run: npm install -g aws-cdk + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install CDK stack dependencies + run: pip install -r requirements.txt + - name: AWS credentials configuration + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{secrets.GH_ACTIONS_AWS_ROLE}} + role-session-name: gh-actions-${{github.run_id}}.${{github.run_number}}.${{github.run_attempt}}-cdk-test + aws-region: us-east-1 + - name: cdk diff + run: cdk diff diff --git a/pipeline/aws_infra/.gitignore b/pipeline/aws_infra/.gitignore new file mode 100644 index 00000000..64699e47 --- /dev/null +++ b/pipeline/aws_infra/.gitignore @@ -0,0 +1,22 @@ +*.swp +package-lock.json +__pycache__ +.pytest_cache +*.egg-info + +## Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# CDK asset staging directory +.cdk.staging +cdk.out + +# CDK CLI installation artifacts +node_modules/ +package.json diff --git a/pipeline/aws_infra/README.md b/pipeline/aws_infra/README.md new file mode 100644 index 00000000..9cef0785 --- /dev/null +++ b/pipeline/aws_infra/README.md @@ -0,0 +1,96 @@ + +# AGR PAVI infra using CDK (Python) + +This is the AWS infrastructure component of the AGR PAVI application. +This includes all AWS resources and infrastructure required to make the main +application executable in AWS, and is defined and deployed using +AWS CDK and written as Python code in this subdirectory. + +AWS CDK is an open-source framework that enables writing +the entire cloud application as code, including all event sources and other AWS resources +which are require to make the application executable in AWS in addition to the application code. +This allows for an easy and reproducible deployment, that can be fully documented and versioned as code. + +For instructions on how to install the AWS CDK CLI, +see the [AWS docs](https://docs.aws.amazon.com/cdk/v2/guide/getting_started.html#getting_started_install). + +CDK CLI tool used for this repository is tested and deployed using node.js v18. + + +As this project is set up as a Python project, it is advised to use [virtualenv](https://docs.python.org/3/library/venv.html) +to allow isolated dependency installation. Furthermore, it is advised to store +the virtual env in a `venv` subdirectory in this directory. +As this path was added to the .gitignore file, it will automatically be excluded +from any git operations. + +To create a new virtual environment (for first first time use of this directory): +```bash +$ python3 -m venv venv +``` + +Then for every subsequent use of this directory (for coding, deployment, ...) + +First activate the virtualenv: +```bash +$ source venv/bin/activate +``` + +Once the virtualenv is activated, install the required dependencies: +```bash +$ pip install -r requirements.txt +``` + +After that, you can execute all required CDK commands as described in below chapters. +Once done working with the code in this directory, deactivate the virtualenv: +```bash +$ deactivate +``` + +## Important files +Two standard CDK configuration files can be found at the root level of this directory: + * [cdk.json](./cdk.json) + Contains the main CDK execution configuration parameters + * [cdk.context.json](./cdk.context.json) + Contains the VPC context in which to deploy the CDK Stack. + +Then the AWS Stack to be deployed using CDK is define in the following files and directories: + * [cdk_app.py](./cdk_app.py) + The root level CDK application, defining the entire AWS Stack to be deployed. + * [cdk_classes/](./cdk_classes/) + Python sub-classes defining the CDK stack (representing a single CloudFormation stack) + and all individual CDK constructs, representing individual cloud components. + +## Validating +When making changes to any of the CDK files, validate them before requesting a PR +or attempting a deployment, by running the following command: +```bash +> cdk diff +``` +This will attempt to synthesize the (Cloudformation) stack and produce errors when (syntax) errors +would be present in any of the CDK code. +When no (more) errors are present, `cdk diff` will compare the stack to the deployed stack, +and display the changes that would get deployed. Inspect these changes to ensure +the code changes made will have the expected effect on the deployed AWS resources. + +This allows the developer to fix any errors before deployment, reducing the amount of +troubleshooting that would otherwise be required on failing or incorrect deployments. + +## Deployment +After making all necessary changes to the CDK code and [validating](#validating) +the resulting stack changes are as expected, you can deploy +the new stack to AWS by running the following command: +```bash +# This command will interactively ask for confirmation before deploying security-sensitive changes. +# To disable this approval (e.g. for use in CI/CD pipelines), add "--require-approval never" +> cdk deploy +``` +Any code pushed to the main branch of this repository (both main application and CDK code) +automatically gets built and deployed, through [github actions](./.github/workflows/main-build-and-deploy.yml). + +## Other useful CDK commands +Here's a list of the most useful CDK commands. For a full list, call `cdk help`. + * `cdk ls` list all stacks in the app + * `cdk synth` emits the synthesized CloudFormation template + * `cdk deploy` deploy this stack to AWS + * `cdk diff` compare deployed stack with current state + * `cdk docs` open CDK documentation diff --git a/pipeline/aws_infra/cdk.context.json b/pipeline/aws_infra/cdk.context.json new file mode 100644 index 00000000..a31b3c72 --- /dev/null +++ b/pipeline/aws_infra/cdk.context.json @@ -0,0 +1,81 @@ +{ + "vpc-provider:account=100225593120:filter.vpc-id=vpc-55522232:region=us-east-1:returnAsymmetricSubnets=true": { + "vpcId": "vpc-55522232", + "vpcCidrBlock": "172.31.0.0/16", + "availabilityZones": [], + "subnetGroups": [ + { + "name": "Public", + "type": "Public", + "subnets": [ + { + "subnetId": "subnet-3ebf4477", + "cidr": "172.31.0.0/20", + "availabilityZone": "us-east-1a", + "routeTableId": "rtb-c63af6a0" + }, + { + "subnetId": "subnet-df7c7487", + "cidr": "172.31.16.0/20", + "availabilityZone": "us-east-1b", + "routeTableId": "rtb-c63af6a0" + }, + { + "subnetId": "subnet-81c95ee4", + "cidr": "172.31.64.0/20", + "availabilityZone": "us-east-1c", + "routeTableId": "rtb-c63af6a0" + }, + { + "subnetId": "subnet-ff838bd5", + "cidr": "172.31.48.0/20", + "availabilityZone": "us-east-1d", + "routeTableId": "rtb-c63af6a0" + }, + { + "subnetId": "subnet-af62dca3", + "cidr": "172.31.80.0/20", + "availabilityZone": "us-east-1f", + "routeTableId": "rtb-c63af6a0" + } + ] + }, + { + "name": "Private", + "type": "Private", + "subnets": [ + { + "subnetId": "subnet-0d4703177afb1797d", + "cidr": "172.31.96.0/24", + "availabilityZone": "us-east-1a", + "routeTableId": "rtb-05e12b551f60b37ed" + }, + { + "subnetId": "subnet-04262fc338f638054", + "cidr": "172.31.97.0/24", + "availabilityZone": "us-east-1b", + "routeTableId": "rtb-05e12b551f60b37ed" + }, + { + "subnetId": "subnet-044457c061edf85f2", + "cidr": "172.31.98.0/24", + "availabilityZone": "us-east-1c", + "routeTableId": "rtb-05e12b551f60b37ed" + }, + { + "subnetId": "subnet-04019d42d5c9e6fb9", + "cidr": "172.31.99.0/24", + "availabilityZone": "us-east-1d", + "routeTableId": "rtb-05e12b551f60b37ed" + }, + { + "subnetId": "subnet-049778993fb504a7c", + "cidr": "172.31.101.0/24", + "availabilityZone": "us-east-1f", + "routeTableId": "rtb-05e12b551f60b37ed" + } + ] + } + ] + } +} diff --git a/pipeline/aws_infra/cdk.json b/pipeline/aws_infra/cdk.json new file mode 100644 index 00000000..4dee0039 --- /dev/null +++ b/pipeline/aws_infra/cdk.json @@ -0,0 +1,39 @@ +{ + "app": "python3 cdk_app.py", + "watch": { + "include": [ + "**" + ], + "exclude": [ + "README.md", + "cdk*.json", + "requirements*.txt", + "**/__init__.py", + "python/__pycache__", + "tests" + ] + }, + "context": { + "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, + "@aws-cdk/core:stackRelativeExports": true, + "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, + "@aws-cdk/aws-lambda:recognizeVersionProps": true, + "@aws-cdk/aws-lambda:recognizeLayerVersion": true, + "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, + "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, + "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, + "@aws-cdk/core:checkSecretUsage": true, + "@aws-cdk/aws-iam:minimizePolicies": true, + "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, + "@aws-cdk/core:validateSnapshotRemovalPolicy": true, + "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, + "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, + "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, + "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, + "@aws-cdk/core:enablePartitionLiterals": true, + "@aws-cdk/core:target-partitions": [ + "aws", + "aws-cn" + ] + } +} diff --git a/pipeline/aws_infra/cdk_app.py b/pipeline/aws_infra/cdk_app.py new file mode 100644 index 00000000..cead0240 --- /dev/null +++ b/pipeline/aws_infra/cdk_app.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +import os + +import aws_cdk as cdk + +from cdk_classes.cdk_infra_stack import CdkInfraStack + + +app = cdk.App() +CdkInfraStack(app, "PaviPipelineCdkStack", + env=cdk.Environment(account='100225593120', region='us-east-1')) + +app.synth() diff --git a/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py b/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py new file mode 100644 index 00000000..2603229d --- /dev/null +++ b/pipeline/aws_infra/cdk_classes/cdk_infra_stack.py @@ -0,0 +1,14 @@ +from aws_cdk import ( + Stack +) + +from constructs import Construct + +from cdk_classes.pipeline_stack import PipelineSeqRetrievalEcrRepository + +class CdkInfraStack(Stack): + + def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: + super().__init__(scope, construct_id, **kwargs) + + PipelineSeqRetrievalEcrRepository(self) diff --git a/pipeline/aws_infra/cdk_classes/pipeline_stack.py b/pipeline/aws_infra/cdk_classes/pipeline_stack.py new file mode 100644 index 00000000..da439ba6 --- /dev/null +++ b/pipeline/aws_infra/cdk_classes/pipeline_stack.py @@ -0,0 +1,19 @@ +from aws_cdk import ( + Stack, + aws_ecr as ecr, + RemovalPolicy +) + +class PipelineSeqRetrievalEcrRepository: + + repository: ecr.Repository + + def __init__(self, scope: Stack) -> None: + + # Create the ECR repository + repo = ecr.Repository(scope, "PAVI-pipeline-seq-retrieval-repo", repository_name='agr_pavi/seq_retrieval', + empty_on_delete=False,removal_policy=RemovalPolicy.RETAIN) + self.repository = repo + + def get_repo_arn(self) -> str: + return self.repository.repository_arn diff --git a/pipeline/aws_infra/requirements-dev.txt b/pipeline/aws_infra/requirements-dev.txt new file mode 100644 index 00000000..269f9b06 --- /dev/null +++ b/pipeline/aws_infra/requirements-dev.txt @@ -0,0 +1 @@ +pytest==8.0.* diff --git a/pipeline/aws_infra/requirements.txt b/pipeline/aws_infra/requirements.txt new file mode 100644 index 00000000..d32f0af1 --- /dev/null +++ b/pipeline/aws_infra/requirements.txt @@ -0,0 +1,2 @@ +aws-cdk-lib==2.136.* +constructs==10.0.0,<11.0.0