Merge pull request #719 from guardian/aa/cfn-deploy-proper

feat: Perform EC2 deployments via CloudFormation
guardian · Sep 17, 2024 · 210fd38 · 210fd38
2 parents fcd8bac + de05bfc
commit 210fd38
Show file tree

Hide file tree

Showing 7 changed files with 198 additions and 25 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -41,6 +41,10 @@ jobs:
           distribution: 'corretto'
           cache: 'sbt'
 
+      - run: |
+          LAST_TEAMCITY_BUILD=1007
+          echo "BUILD_NUMBER=$(( $GITHUB_RUN_NUMBER + $LAST_TEAMCITY_BUILD ))" >> $GITHUB_ENV
+
       # See https://github.com/github/scripts-to-rule-them-all
       - name: Run script/ci
         run: ./script/ci
@@ -50,11 +54,11 @@ jobs:
         with:
           githubToken: ${{ secrets.GITHUB_TOKEN }}
           roleArn: ${{ secrets.GU_RIFF_RAFF_ROLE_ARN }}
-          buildNumberOffset: 1007
+          buildNumber: ${{ env.BUILD_NUMBER }}
           projectName: tools::amiable
           configPath: cdk/cdk.out/riff-raff.yaml
           contentDirectories: |
             cdk.out:
               - cdk/cdk.out
             amiable:
-              - amiable.deb
+              - dist
diff --git a/cdk/jest.setup.js b/cdk/jest.setup.js
@@ -1 +1,3 @@
 jest.mock("@guardian/cdk/lib/constants/tracking-tag");
+
+process.env.BUILD_NUMBER = "TEST";
diff --git a/cdk/lib/amiable/__snapshots__/amiable.test.ts.snap b/cdk/lib/amiable/__snapshots__/amiable.test.ts.snap
@@ -10,7 +10,7 @@ exports[`The Amiable stack matches the snapshot 1`] = `
       "GuVpcParameter",
       "GuSubnetListParameter",
       "GuSubnetListParameter",
-      "GuPlayApp",
+      "GuEc2AppExperimental",
       "GuCertificate",
       "GuInstanceRole",
       "GuSsmSshPolicy",
@@ -104,8 +104,46 @@ exports[`The Amiable stack matches the snapshot 1`] = `
       },
       "Type": "AWS::SSM::Parameter",
     },
+    "AsgRollingUpdatePolicy2A1DDC6F": {
+      "Properties": {
+        "PolicyDocument": {
+          "Statement": [
+            {
+              "Action": "cloudformation:SignalResource",
+              "Effect": "Allow",
+              "Resource": {
+                "Ref": "AWS::StackId",
+              },
+            },
+            {
+              "Action": "elasticloadbalancing:DescribeTargetHealth",
+              "Effect": "Allow",
+              "Resource": "*",
+            },
+          ],
+          "Version": "2012-10-17",
+        },
+        "PolicyName": "AsgRollingUpdatePolicy2A1DDC6F",
+        "Roles": [
+          {
+            "Ref": "InstanceRoleAmiable56110022",
+          },
+        ],
+      },
+      "Type": "AWS::IAM::Policy",
+    },
     "AutoScalingGroupAmiableASGFCD99427": {
+      "CreationPolicy": {
+        "AutoScalingCreationPolicy": {
+          "MinSuccessfulInstancesPercent": 100,
+        },
+        "ResourceSignal": {
+          "Count": 1,
+          "Timeout": "PT2M",
+        },
+      },
       "Properties": {
+        "DesiredCapacity": "1",
         "HealthCheckGracePeriod": 120,
         "HealthCheckType": "ELB",
         "LaunchTemplate": {
@@ -123,10 +161,6 @@ exports[`The Amiable stack matches the snapshot 1`] = `
         "MetricsCollection": [
           {
             "Granularity": "1Minute",
-            "Metrics": [
-              "GroupTotalInstances",
-              "GroupInServiceInstances",
-            ],
           },
         ],
         "MinSize": "1",
@@ -179,6 +213,18 @@ exports[`The Amiable stack matches the snapshot 1`] = `
         },
       },
       "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "UpdatePolicy": {
+        "AutoScalingRollingUpdate": {
+          "MaxBatchSize": 2,
+          "MinInstancesInService": 1,
+          "MinSuccessfulInstancesPercent": 100,
+          "PauseTime": "PT2M",
+          "SuspendProcesses": [
+            "AlarmNotification",
+          ],
+          "WaitOnResourceSignals": true,
+        },
+      },
     },
     "CertificateAmiable3AC0F81D": {
       "DeletionPolicy": "Retain",
@@ -1037,6 +1083,17 @@ exports[`The Amiable stack matches the snapshot 1`] = `
                 "",
                 [
                   "#!/bin/bash
+function exitTrap(){
+exitCode=$?
+
+        cfn-signal --stack ",
+                  {
+                    "Ref": "AWS::StackId",
+                  },
+                  "           --resource AutoScalingGroupAmiableASGFCD99427           --region eu-west-1           --exit-code $exitCode || echo 'Failed to send Cloudformation Signal'
+        
+}
+trap exitTrap EXIT
 
           mkdir /amiable
           aws --region eu-west-1 s3 cp s3://",
@@ -1053,9 +1110,33 @@ exports[`The Amiable stack matches the snapshot 1`] = `
                   {
                     "Ref": "DistributionBucketName",
                   },
-                  "/deploy/CODE/amiable/amiable.deb /amiable/
+                  "/deploy/CODE/amiable/amiable-TEST.deb /amiable/amiable.deb
+
+          dpkg -i /amiable/amiable.deb
+# GuEc2AppExperimental UserData Start
+
+      TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+      INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/instance-id")
+
+      STATE=$(aws elbv2 describe-target-health         --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupAmiable6987B5AC",
+                  },
+                  "         --region eu-west-1         --targets Id=$INSTANCE_ID,Port=9000         --query "TargetHealthDescriptions[0].TargetHealth.State")
+
+      until [ "$STATE" == "\\"healthy\\"" ]; do
+        echo "Instance not yet healthy within target group. Current state $STATE. Sleeping..."
+        sleep 5
+        STATE=$(aws elbv2 describe-target-health           --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupAmiable6987B5AC",
+                  },
+                  "           --region eu-west-1           --targets Id=$INSTANCE_ID,Port=9000           --query "TargetHealthDescriptions[0].TargetHealth.State")
+      done
 
-          dpkg -i /amiable/amiable.deb",
+      echo "Instance is healthy in target group."
+      
+# GuEc2AppExperimental UserData End",
                 ],
               ],
             },
@@ -1201,7 +1282,7 @@ exports[`The Amiable stack matches the snapshot 2`] = `
       "GuVpcParameter",
       "GuSubnetListParameter",
       "GuSubnetListParameter",
-      "GuPlayApp",
+      "GuEc2AppExperimental",
       "GuCertificate",
       "GuInstanceRole",
       "GuSsmSshPolicy",
@@ -1297,8 +1378,46 @@ exports[`The Amiable stack matches the snapshot 2`] = `
       },
       "Type": "AWS::SSM::Parameter",
     },
+    "AsgRollingUpdatePolicy2A1DDC6F": {
+      "Properties": {
+        "PolicyDocument": {
+          "Statement": [
+            {
+              "Action": "cloudformation:SignalResource",
+              "Effect": "Allow",
+              "Resource": {
+                "Ref": "AWS::StackId",
+              },
+            },
+            {
+              "Action": "elasticloadbalancing:DescribeTargetHealth",
+              "Effect": "Allow",
+              "Resource": "*",
+            },
+          ],
+          "Version": "2012-10-17",
+        },
+        "PolicyName": "AsgRollingUpdatePolicy2A1DDC6F",
+        "Roles": [
+          {
+            "Ref": "InstanceRoleAmiable56110022",
+          },
+        ],
+      },
+      "Type": "AWS::IAM::Policy",
+    },
     "AutoScalingGroupAmiableASGFCD99427": {
+      "CreationPolicy": {
+        "AutoScalingCreationPolicy": {
+          "MinSuccessfulInstancesPercent": 100,
+        },
+        "ResourceSignal": {
+          "Count": 1,
+          "Timeout": "PT2M",
+        },
+      },
       "Properties": {
+        "DesiredCapacity": "1",
         "HealthCheckGracePeriod": 120,
         "HealthCheckType": "ELB",
         "LaunchTemplate": {
@@ -1316,10 +1435,6 @@ exports[`The Amiable stack matches the snapshot 2`] = `
         "MetricsCollection": [
           {
             "Granularity": "1Minute",
-            "Metrics": [
-              "GroupTotalInstances",
-              "GroupInServiceInstances",
-            ],
           },
         ],
         "MinSize": "1",
@@ -1372,6 +1487,18 @@ exports[`The Amiable stack matches the snapshot 2`] = `
         },
       },
       "Type": "AWS::AutoScaling::AutoScalingGroup",
+      "UpdatePolicy": {
+        "AutoScalingRollingUpdate": {
+          "MaxBatchSize": 2,
+          "MinInstancesInService": 1,
+          "MinSuccessfulInstancesPercent": 100,
+          "PauseTime": "PT2M",
+          "SuspendProcesses": [
+            "AlarmNotification",
+          ],
+          "WaitOnResourceSignals": true,
+        },
+      },
     },
     "CertificateAmiable3AC0F81D": {
       "DeletionPolicy": "Retain",
@@ -2465,6 +2592,17 @@ exports[`The Amiable stack matches the snapshot 2`] = `
                 "",
                 [
                   "#!/bin/bash
+function exitTrap(){
+exitCode=$?
+
+        cfn-signal --stack ",
+                  {
+                    "Ref": "AWS::StackId",
+                  },
+                  "           --resource AutoScalingGroupAmiableASGFCD99427           --region eu-west-1           --exit-code $exitCode || echo 'Failed to send Cloudformation Signal'
+        
+}
+trap exitTrap EXIT
 
           mkdir /amiable
           aws --region eu-west-1 s3 cp s3://",
@@ -2481,9 +2619,33 @@ exports[`The Amiable stack matches the snapshot 2`] = `
                   {
                     "Ref": "DistributionBucketName",
                   },
-                  "/deploy/PROD/amiable/amiable.deb /amiable/
+                  "/deploy/PROD/amiable/amiable-TEST.deb /amiable/amiable.deb
+
+          dpkg -i /amiable/amiable.deb
+# GuEc2AppExperimental UserData Start
+
+      TOKEN=$(curl -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
+      INSTANCE_ID=$(curl -H "X-aws-ec2-metadata-token: $TOKEN" "http://169.254.169.254/latest/meta-data/instance-id")
+
+      STATE=$(aws elbv2 describe-target-health         --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupAmiable6987B5AC",
+                  },
+                  "         --region eu-west-1         --targets Id=$INSTANCE_ID,Port=9000         --query "TargetHealthDescriptions[0].TargetHealth.State")
+
+      until [ "$STATE" == "\\"healthy\\"" ]; do
+        echo "Instance not yet healthy within target group. Current state $STATE. Sleeping..."
+        sleep 5
+        STATE=$(aws elbv2 describe-target-health           --target-group-arn ",
+                  {
+                    "Ref": "TargetGroupAmiable6987B5AC",
+                  },
+                  "           --region eu-west-1           --targets Id=$INSTANCE_ID,Port=9000           --query "TargetHealthDescriptions[0].TargetHealth.State")
+      done
 
-          dpkg -i /amiable/amiable.deb",
+      echo "Instance is healthy in target group."
+      
+# GuEc2AppExperimental UserData End",
                 ],
               ],
             },

diff --git a/cdk/lib/amiable/amiable.ts b/cdk/lib/amiable/amiable.ts
@@ -4,7 +4,7 @@ import { GuDistributionBucketParameter, GuStack, GuStringParameter } from "@guar
 import { GuCname } from "@guardian/cdk/lib/constructs/dns";
 import { GuHttpsEgressSecurityGroup } from "@guardian/cdk/lib/constructs/ec2";
 import { GuAllowPolicy, GuSESSenderPolicy } from "@guardian/cdk/lib/constructs/iam";
-import { GuPlayApp } from "@guardian/cdk/lib/patterns/ec2-app";
+import { GuEc2AppExperimental } from "@guardian/cdk/lib/experimental/patterns/ec2-app";
 import type { App } from "aws-cdk-lib";
 import { Duration, SecretValue } from "aws-cdk-lib";
 import { InstanceClass, InstanceSize, InstanceType, UserData } from "aws-cdk-lib/aws-ec2";
@@ -27,16 +27,19 @@ export class Amiable extends GuStack {
 
     const distBucket = GuDistributionBucketParameter.getInstance(this).valueAsString;
 
+    const buildNumber = process.env.BUILD_NUMBER ?? "DEV";
+
     const userData = UserData.forLinux();
     userData.addCommands(`
           mkdir /amiable
           aws --region eu-west-1 s3 cp s3://${distBucket}/${stack}/${stage}/${app}/conf/amiable-service-account-cert.json /amiable/
           aws --region eu-west-1 s3 cp s3://${distBucket}/${stack}/${stage}/${app}/conf/amiable.conf /etc/
-          aws --region eu-west-1 s3 cp s3://${distBucket}/${stack}/${stage}/${app}/amiable.deb /amiable/
+          aws --region eu-west-1 s3 cp s3://${distBucket}/${stack}/${stage}/${app}/amiable-${buildNumber}.deb /amiable/amiable.deb
 
           dpkg -i /amiable/amiable.deb`);
 
-    const ec2App = new GuPlayApp(this, {
+    const ec2App = new GuEc2AppExperimental(this, {
+      applicationPort: 9000,
       app,
       instanceType: InstanceType.of(InstanceClass.T4G, InstanceSize.SMALL),
       userData,

diff --git a/cdk/package-lock.json b/cdk/package-lock.json
diff --git a/cdk/package.json b/cdk/package.json
@@ -17,7 +17,7 @@
     "synth": "cdk synth --path-metadata false --version-reporting false"
   },
   "devDependencies": {
-    "@guardian/cdk": "59.3.5",
+    "@guardian/cdk": "59.5.0",
     "@guardian/eslint-config-typescript": "^12.0.0",
     "@types/jest": "^29.5.13",
     "@types/node": "22.5.5",

diff --git a/script/ci b/script/ci
@@ -9,4 +9,6 @@ set -e
 
 
 sbt clean scalafmtCheckAll compile test debian:packageBin
-mv target/*.deb ./amiable.deb
+
+mkdir -p dist
+mv target/*.deb "dist/amiable-${BUILD_NUMBER}.deb"