From 813f4c1b8c33afef28cfec2fb22e67d3ac7436bc Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 11 Oct 2024 10:17:07 +0100 Subject: [PATCH 1/2] feat: Add scripts to trigger ASG scale in/out events This makes it easier to test certain deployment situations, as scaling events become somewhat deterministic. --- README.md | 10 ++-- .../__snapshots__/cdk-playground.test.ts.snap | 53 ++++++++++++++++++- cdk/lib/cdk-playground.ts | 34 +++++++++++- script/scale-in | 46 ++++++++++++++++ script/scale-out | 46 ++++++++++++++++ 5 files changed, 181 insertions(+), 8 deletions(-) create mode 100755 script/scale-in create mode 100755 script/scale-out diff --git a/README.md b/README.md index 948b3f8c..1ae88abf 100644 --- a/README.md +++ b/README.md @@ -10,10 +10,12 @@ That is, CDK Playground offers a low risk environment to experiment with. The CDK stack is defined in the [cdk](./cdk) directory. There are a couple of helpful scripts in the [script](./script) directory: - 1. `./script/start-play` to run the Play app - 1. `./script/start-cdk` to start Jest in watch mode, to test the CDK stack - 1. `./script/build-cdk` to synthesise the CDK stack into a template - 1. `./script/switch-cdk` to install GuCDK from a GitHub branch. This is useful to test changes _without_ publishing to NPM first. +1. `./script/start-play` to run the Play app +2. `./script/start-cdk` to start Jest in watch mode, to test the CDK stack +3. `./script/build-cdk` to synthesise the CDK stack into a template +4. `./script/switch-cdk` to install GuCDK from a GitHub branch. This is useful to test changes _without_ publishing to NPM first. +5. `./script/scale-out` to simulate a scale out event, increasing the capacity of the autoscaling group +6. `./script/scale-in` to simulate a scale in event, decreasing the capacity of the autoscaling group ## Deploying The app is set up in the usual way, with CI on each branch (via GitHub Actions) and [CD](https://riffraff.gutools.co.uk/deployment/history?projectName=devx%3A%3Acdk-playground&stage=PROD&pageSize=20&page=1) on `main`. diff --git a/cdk/lib/__snapshots__/cdk-playground.test.ts.snap b/cdk/lib/__snapshots__/cdk-playground.test.ts.snap index 42dabdb0..b89e71f5 100644 --- a/cdk/lib/__snapshots__/cdk-playground.test.ts.snap +++ b/cdk/lib/__snapshots__/cdk-playground.test.ts.snap @@ -32,6 +32,11 @@ Object { "gu:cdk:version": "TEST", }, "Outputs": Object { + "AutoscalingGroupName": Object { + "Value": Object { + "Ref": "AutoScalingGroupCdkplaygroundASGD6E49F0F", + }, + }, "LoadBalancerCdkplaygroundDnsName": Object { "Description": "DNS entry for LoadBalancerCdkplayground", "Value": Object { @@ -41,6 +46,22 @@ Object { ], }, }, + "ScaleInArn": Object { + "Value": Object { + "Fn::GetAtt": Array [ + "AutoScalingGroupCdkplaygroundScaleIn24F0C6B8", + "Arn", + ], + }, + }, + "ScaleOutArn": Object { + "Value": Object { + "Fn::GetAtt": Array [ + "AutoScalingGroupCdkplaygroundScaleOutA06BF2EE", + "Arn", + ], + }, + }, "lambdacdkplaygroundlambdaapiEndpoint0E1DFC5F": Object { "Value": Object { "Fn::Join": Array [ @@ -79,6 +100,11 @@ Object { "Description": "SSM parameter containing the Name (not ARN) on the kinesis stream", "Type": "AWS::SSM::Parameter::Value", }, + "MinInstancesInServiceForcdkplayground": Object { + "Default": 1, + "MaxValue": 1, + "Type": "Number", + }, "VpcId": Object { "Default": "/account/vpc/primary/id", "Description": "Virtual Private Cloud to run EC2 instances within. Should NOT be the account default VPC.", @@ -138,7 +164,6 @@ Object { "AsgRollingUpdatePolicy2A1DDC6F", ], "Properties": Object { - "DesiredCapacity": "1", "HealthCheckGracePeriod": 120, "HealthCheckType": "ELB", "LaunchTemplate": Object { @@ -211,7 +236,9 @@ Object { "UpdatePolicy": Object { "AutoScalingRollingUpdate": Object { "MaxBatchSize": 2, - "MinInstancesInService": 1, + "MinInstancesInService": Object { + "Ref": "MinInstancesInServiceForcdkplayground", + }, "MinSuccessfulInstancesPercent": 100, "PauseTime": "PT3M", "SuspendProcesses": Array [ @@ -221,6 +248,28 @@ Object { }, }, }, + "AutoScalingGroupCdkplaygroundScaleIn24F0C6B8": Object { + "Properties": Object { + "AdjustmentType": "ChangeInCapacity", + "AutoScalingGroupName": Object { + "Ref": "AutoScalingGroupCdkplaygroundASGD6E49F0F", + }, + "PolicyType": "SimpleScaling", + "ScalingAdjustment": -1, + }, + "Type": "AWS::AutoScaling::ScalingPolicy", + }, + "AutoScalingGroupCdkplaygroundScaleOutA06BF2EE": Object { + "Properties": Object { + "AdjustmentType": "ChangeInCapacity", + "AutoScalingGroupName": Object { + "Ref": "AutoScalingGroupCdkplaygroundASGD6E49F0F", + }, + "PolicyType": "SimpleScaling", + "ScalingAdjustment": 1, + }, + "Type": "AWS::AutoScaling::ScalingPolicy", + }, "CertificateCdkplayground47FCF7D9": Object { "DeletionPolicy": "Retain", "Properties": Object { diff --git a/cdk/lib/cdk-playground.ts b/cdk/lib/cdk-playground.ts index 25223f04..7d1c4820 100644 --- a/cdk/lib/cdk-playground.ts +++ b/cdk/lib/cdk-playground.ts @@ -6,7 +6,8 @@ import { GuStack } from '@guardian/cdk/lib/constructs/core'; import { GuCname } from '@guardian/cdk/lib/constructs/dns'; import { GuEc2AppExperimental } from '@guardian/cdk/lib/experimental/patterns/ec2-app'; import type { App } from 'aws-cdk-lib'; -import { Duration } from 'aws-cdk-lib'; +import { CfnOutput, Duration } from 'aws-cdk-lib'; +import { CfnScalingPolicy } from 'aws-cdk-lib/aws-autoscaling'; import { InstanceClass, InstanceSize, InstanceType } from 'aws-cdk-lib/aws-ec2'; import { Runtime } from 'aws-cdk-lib/aws-lambda'; @@ -35,7 +36,7 @@ export class CdkPlayground extends GuStack { const ec2App = 'cdk-playground'; const ec2AppDomainName = 'cdk-playground.gutools.co.uk'; - const { loadBalancer } = new GuEc2AppExperimental(this, { + const { loadBalancer, autoScalingGroup } = new GuEc2AppExperimental(this, { buildIdentifier, applicationPort: 9000, app: ec2App, @@ -62,6 +63,35 @@ export class CdkPlayground extends GuStack { imageRecipe: 'developerPlayground-arm64-java11', }); + const scaleOutPolicy = new CfnScalingPolicy(autoScalingGroup, 'ScaleOut', { + autoScalingGroupName: autoScalingGroup.autoScalingGroupName, + policyType: 'SimpleScaling', + adjustmentType: 'ChangeInCapacity', + scalingAdjustment: 1, + }); + + const scaleInPolicy = new CfnScalingPolicy(autoScalingGroup, 'ScaleIn', { + autoScalingGroupName: autoScalingGroup.autoScalingGroupName, + policyType: 'SimpleScaling', + adjustmentType: 'ChangeInCapacity', + scalingAdjustment: -1, + }); + + new CfnOutput(this, 'ScaleOutArn', { + key: 'ScaleOutArn', + value: scaleOutPolicy.attrArn, + }); + + new CfnOutput(this, 'ScaleInArn', { + key: 'ScaleInArn', + value: scaleInPolicy.attrArn, + }); + + new CfnOutput(this, 'AutoscalingGroupName', { + key: 'AutoscalingGroupName', + value: autoScalingGroup.autoScalingGroupName, + }); + new GuCname(this, 'EC2AppDNS', { app: ec2App, ttl: Duration.hours(1), diff --git a/script/scale-in b/script/scale-in new file mode 100755 index 00000000..fc8bc31a --- /dev/null +++ b/script/scale-in @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +CLOUDFORMATION_STACK_NAME=playground-PROD-cdk-playground + +POLICY_ARN=$( + aws cloudformation describe-stacks \ + --stack-name "$CLOUDFORMATION_STACK_NAME" \ + --profile developerPlayground \ + --region eu-west-1 \ + --no-cli-pager | \ + jq -r '.Stacks[].Outputs[] | select( [.OutputKey | contains("ScaleInArn") ] | any) | .OutputValue' +) + +ASG_NAME=$( + aws cloudformation describe-stacks \ + --stack-name "$CLOUDFORMATION_STACK_NAME" \ + --profile developerPlayground \ + --region eu-west-1 \ + --no-cli-pager | \ + jq -r '.Stacks[].Outputs[] | select( [.OutputKey | contains("AutoscalingGroupName") ] | any) | .OutputValue' +) + +CURRENT_DESIRED_CAPACITY=$( + aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --profile developerPlayground \ + --region eu-west-1 | \ + jq -r '.AutoScalingGroups[].DesiredCapacity' +) + +aws autoscaling execute-policy \ + --policy-name "$POLICY_ARN" \ + --profile developerPlayground \ + --region eu-west-1 + +NEW_DESIRED_CAPACITY=$( + aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --profile developerPlayground \ + --region eu-west-1 | \ + jq -r '.AutoScalingGroups[].DesiredCapacity' +) + +echo "Desired capacity has been updated from $CURRENT_DESIRED_CAPACITY to $NEW_DESIRED_CAPACITY" diff --git a/script/scale-out b/script/scale-out new file mode 100755 index 00000000..c348137b --- /dev/null +++ b/script/scale-out @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -e + +CLOUDFORMATION_STACK_NAME=playground-CODE-scaling-asg-rolling-update + +POLICY_ARN=$( + aws cloudformation describe-stacks \ + --stack-name "$CLOUDFORMATION_STACK_NAME" \ + --profile developerPlayground \ + --region eu-west-1 \ + --no-cli-pager | \ + jq -r '.Stacks[].Outputs[] | select( [.OutputKey | contains("ScaleOutArn") ] | any) | .OutputValue' +) + +ASG_NAME=$( + aws cloudformation describe-stacks \ + --stack-name "$CLOUDFORMATION_STACK_NAME" \ + --profile developerPlayground \ + --region eu-west-1 \ + --no-cli-pager | \ + jq -r '.Stacks[].Outputs[] | select( [.OutputKey | contains("AutoscalingGroupName") ] | any) | .OutputValue' +) + +CURRENT_DESIRED_CAPACITY=$( + aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --profile developerPlayground \ + --region eu-west-1 | \ + jq -r '.AutoScalingGroups[].DesiredCapacity' +) + +aws autoscaling execute-policy \ + --policy-name "$POLICY_ARN" \ + --profile developerPlayground \ + --region eu-west-1 + +NEW_DESIRED_CAPACITY=$( + aws autoscaling describe-auto-scaling-groups \ + --auto-scaling-group-names "$ASG_NAME" \ + --profile developerPlayground \ + --region eu-west-1 | \ + jq -r '.AutoScalingGroups[].DesiredCapacity' +) + +echo "Desired capacity has been updated from $CURRENT_DESIRED_CAPACITY to $NEW_DESIRED_CAPACITY" From ea1ec552c36c539647452357dc257a2edbcf5393 Mon Sep 17 00:00:00 2001 From: akash1810 Date: Fri, 11 Oct 2024 10:20:41 +0100 Subject: [PATCH 2/2] fix: Provide room to scale out --- cdk/lib/__snapshots__/cdk-playground.test.ts.snap | 6 +++--- cdk/lib/cdk-playground.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdk/lib/__snapshots__/cdk-playground.test.ts.snap b/cdk/lib/__snapshots__/cdk-playground.test.ts.snap index b89e71f5..0a3b7e94 100644 --- a/cdk/lib/__snapshots__/cdk-playground.test.ts.snap +++ b/cdk/lib/__snapshots__/cdk-playground.test.ts.snap @@ -102,7 +102,7 @@ Object { }, "MinInstancesInServiceForcdkplayground": Object { "Default": 1, - "MaxValue": 1, + "MaxValue": 9, "Type": "Number", }, "VpcId": Object { @@ -177,7 +177,7 @@ Object { ], }, }, - "MaxSize": "2", + "MaxSize": "10", "MetricsCollection": Array [ Object { "Granularity": "1Minute", @@ -235,7 +235,7 @@ Object { "Type": "AWS::AutoScaling::AutoScalingGroup", "UpdatePolicy": Object { "AutoScalingRollingUpdate": Object { - "MaxBatchSize": 2, + "MaxBatchSize": 10, "MinInstancesInService": Object { "Ref": "MinInstancesInServiceForcdkplayground", }, diff --git a/cdk/lib/cdk-playground.ts b/cdk/lib/cdk-playground.ts index 7d1c4820..55ebcf6a 100644 --- a/cdk/lib/cdk-playground.ts +++ b/cdk/lib/cdk-playground.ts @@ -54,7 +54,7 @@ export class CdkPlayground extends GuStack { monitoringConfiguration: { noMonitoring: true }, scaling: { minimumInstances: 1, - maximumInstances: 2, + maximumInstances: 10, }, applicationLogging: { enabled: true,