V0.9.13 (#111)

* Have copy-to-s3 type support ZipBeforePut #107
* Use org-formation functions within a cloudformation template #108
* Detect unchanged register-type task and optimize to no-op #113
* New command: print all stacks for a task file #112

CHANGELOG.md

@@ -1,6 +1,17 @@
 # Changelog
 All notable changes to aws organization formation will be documented in this file.
 
+**version 0.9.13**
+- Added a new command: `print-tasks`, which will generate all cloudformation templates and write to disk.
+- Added `zip-before-put` support to `copy-to-s3` task.
+- Added support for `!ReadFile` and `!JsonString` inside CloudFormation templates.
+- Added functions `!MD5Dir` and `!MD5File`, which can be used in both task files and cloudformation.
+- Added psuedo parameter `ORG::StateBucketName`.
+- Optimized build time by locally skipping resource providers if task did not change.
+- Updated codebuild image used to create new pipelines with to standard:4.0.
+Note: If you are running a pipeline generated by org-formation, you might want to update the build image for faster provisioning time!
+
+
 **version 0.9.12**
 - Allow failure tolerance to be set to 0 on validate-tasks command (allows CI/CD processes to fail on validation)
 - Added support for `Mappings` section / `!FindInMap` / `!Select` for task files.

cli-program.ts

@@ -14,6 +14,7 @@ import {
     ValidateTasksCommand,
     RemoveCommand,
 } from '~commands/index';
+import { PrintTasksCommand } from '~commands/print-tasks';
 
 export class CliProgram {
 
@@ -48,6 +49,7 @@ export class CliProgram {
         new InitPipelineCommand(this.program);
         new InitOrganizationCommand(this.program);
         new PerformTasksCommand(this.program);
+        new PrintTasksCommand(this.program);
         new PrintStacksCommand(this.program);
         new UpdateOrganizationCommand(this.program);
         new UpdateStacksCommand(this.program);

examples/lambda-using-read-file/lambda-using-read-file.yml

@@ -0,0 +1,29 @@
+
+AWSTemplateFormatVersion: 2010-09-09-OC
+Description: Org formation example
+
+Resources:
+  MyRole:
+    Type: AWS::IAM::Role
+    Properties:
+      ManagedPolicyArns:
+      - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - lambda.amazonaws.com
+            Action:
+              - 'sts:AssumeRole'
+
+  MyLambda:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: 'org-formation-example-lambda-using-read-file'
+      Code:
+        ZipFile: !ReadFile './src/index.js'
+      Handler: index.handler
+      Role: !GetAtt MyRole.Arn
+      Runtime: nodejs12.x

examples/lambda-using-read-file/organization-tasks.yml

@@ -0,0 +1,15 @@
+# this example uses features that are part of the 0.9.13 release
+
+OrganizationUpdate:
+  Type: update-organization
+  Skip: true
+  Template: ./organization.yml
+
+DeployCodeAndLambda:
+  Type: update-stacks
+  Template: ./lambda-using-read-file.yml
+  StackName: org-formation-example-lambda-using-read-file
+  DefaultOrganizationBinding:
+    Account: !Ref AccountA
+    Region: eu-central-1
+

examples/lambda-using-read-file/organization.yml

@@ -0,0 +1,15 @@
+AWSTemplateFormatVersion: '2010-09-09-OC'
+
+Organization:
+  MasterAccount:
+    Type: OC::ORG::MasterAccount
+    Properties:
+      AccountId: '102625093955'
+      RootEmail: org-master@olafconijn.awsapps.com
+      AccountName: Organization Master Account
+
+  AccountA:
+    Type: OC::ORG::Account
+    Properties:
+      RootEmail: account+a@olafconijn.awsapps.com
+      AccountName: Account A

examples/lambda-using-read-file/src/index.js

@@ -0,0 +1,5 @@
+
+exports.handler = function (event, context) {
+	console.log(event);
+	context.succeed('hello ' + event.name);
+};

examples/lambda-using-uploaded-zip/lambda-template-using-zip.yml

@@ -0,0 +1,40 @@
+
+AWSTemplateFormatVersion: 2010-09-09-OC
+Description: Org formation example
+
+Parameters:
+
+  deploymentBucketName:
+    Type: String
+    Description: Name of the bucket that contains the lambda source code
+
+  lambdaS3Key:
+    Type: String
+    Description: S3 Key that contains the location of lambda source code
+
+Resources:
+  MyRole:
+    Type: AWS::IAM::Role
+    Properties:
+      ManagedPolicyArns:
+      - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole
+      AssumeRolePolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - lambda.amazonaws.com
+            Action:
+              - 'sts:AssumeRole'
+
+  MyLambda:
+    Type: AWS::Lambda::Function
+    Properties:
+      FunctionName: 'org-formation-example-lambda-using-uploaded-zip'
+      Code:
+        S3Bucket: !Ref deploymentBucketName
+        S3Key: !Ref lambdaS3Key
+      Handler: index.handler
+      Role: !GetAtt MyRole.Arn
+      Runtime: nodejs12.x

examples/lambda-using-uploaded-zip/org-formation-deployment-bucket.yml

@@ -0,0 +1,49 @@
+
+AWSTemplateFormatVersion: 2010-09-09-OC
+
+Parameters:
+  deploymentBucketName:
+    Type: String
+
+  organizationPrincipalId:
+    Type: String
+
+Resources:
+
+  OrgFormationDeploymentBucket:
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Ref deploymentBucketName
+      BucketEncryption:
+        ServerSideEncryptionConfiguration:
+          - ServerSideEncryptionByDefault:
+              SSEAlgorithm: AES256
+
+
+  OrgFormationDeploymentBucketPolicy:
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      Bucket: !Ref OrgFormationDeploymentBucket
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Sid: 'OwnerAllowEverything'
+            Effect: 'Allow'
+            Principal:
+              AWS: !Ref AWS::AccountId
+            Action: 's3:*'
+            Resource:
+              - !Sub '${OrgFormationDeploymentBucket.Arn}'
+              - !Sub '${OrgFormationDeploymentBucket.Arn}/*'
+          - Sid: 'OrgAllowGetObject'
+            Effect: 'Allow'
+            Principal: '*'
+            Action:
+              - 's3:GetObject'
+              - 's3:GetObjectVersion'
+            Resource:
+              - !Sub '${OrgFormationDeploymentBucket.Arn}/*'
+            Condition:
+              StringEquals:
+                  'aws:PrincipalOrgID':
+                    - !Ref organizationPrincipalId

examples/lambda-using-uploaded-zip/organization-tasks.yml

@@ -0,0 +1,49 @@
+# this example uses features that are part of the 0.9.13 release
+
+
+Parameters:
+  deploymentBucketName:
+    Type: String
+    Default: !Sub '${ORG::StateBucketName}-deployments'
+
+OrganizationUpdate:
+  Type: update-organization
+  Skip: true
+  Template: ./organization.yml
+
+OrgFormationUploadBucket:
+  Type: update-stacks
+  Template: ./org-formation-deployment-bucket.yml
+  StackName: org-formation-deployment-bucket
+  StackDescription: Creates a bucket that can be used by org-formation to upload artifacts and use this bucket to deploy resources across the organization
+  DefaultOrganizationBinding:
+    IncludeMasterAccount: true
+    Region: eu-central-1
+  Parameters:
+    deploymentBucketName: !Ref deploymentBucketName
+    organizationPrincipalId: !Ref ORG::PrincipalOrgID
+
+DeployLambdaSourceCode:
+  Type: copy-to-s3
+  RemotePath: !Sub
+  - s3://${bucket}/lambdas/my-lambda-source-${hashOfDir}.zip
+  - { bucket: !Ref deploymentBucketName, hashOfDir: !MD5Dir ./src }
+  LocalPath: ./src
+  ZipBeforePut: true
+  OrganizationBinding:
+    IncludeMasterAccount: true
+    Region: eu-central-1
+
+DeployLambda:
+  Type: update-stacks
+  DependsOn: DeployLambdaSourceCode
+  Template: ./lambda-template-using-zip.yml
+  StackName: org-formation-example-lambda-using-uploaded-zip
+  DefaultOrganizationBinding:
+    Account: !Ref AccountA
+    Region: eu-central-1
+  Parameters:
+    deploymentBucketName: !Ref deploymentBucketName
+    lambdaS3Key: !Sub
+    - lambdas/my-lambda-source-${hashOfDir}.zip
+    - { bucket: !Ref ORG::StateBucketName, hashOfDir: !MD5Dir ./src }

examples/lambda-using-uploaded-zip/organization.yml

@@ -0,0 +1,15 @@
+AWSTemplateFormatVersion: '2010-09-09-OC'
+
+Organization:
+  MasterAccount:
+    Type: OC::ORG::MasterAccount
+    Properties:
+      AccountId: '102625093955'
+      RootEmail: org-master@olafconijn.awsapps.com
+      AccountName: Organization Master Account
+
+  AccountA:
+    Type: OC::ORG::Account
+    Properties:
+      RootEmail: account+a@olafconijn.awsapps.com
+      AccountName: Account A

examples/lambda-using-uploaded-zip/src/index.js

@@ -0,0 +1,6 @@
+const logger = require('./logger')
+
+exports.handler = function (event, context) {
+	logger.log(event);
+	context.succeed('hello ' + event.name);
+};

examples/lambda-using-uploaded-zip/src/logger.js

@@ -0,0 +1,4 @@
+
+exports.log = function (x) {
+	console.log(x);
+};

examples/lambda-using-uploaded-zip/src/package.json

@@ -0,0 +1,11 @@
+{
+  "name": "org-formation-example-lambda",
+  "version": "1.0.0",
+  "description": "",
+  "main": "run-local.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "",
+  "license": "ISC"
+}

examples/lambda-using-uploaded-zip/src/run-local.js

@@ -0,0 +1,7 @@
+const lambda = require('./index');
+
+const event = {name: 'me'};
+const context = {
+    succeed: (x) => console.log(`succeeded: ${x}`)
+}
+lambda.handler(event, context)

package.json

@@ -1,6 +1,6 @@
 {
   "name": "aws-organization-formation",
-  "version": "0.9.13-beta.2",
+  "version": "0.9.13",
   "description": "Infrastructure as code solution for AWS Organizations",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

resources/orgformation-codepipeline.yml

@@ -264,7 +264,7 @@ Resources:
       Artifacts: { Type: NO_ARTIFACTS }
       Environment:
         Type: LINUX_CONTAINER
-        Image: aws/codebuild/standard:1.0
+        Image: aws/codebuild/standard:4.0
         ComputeType: BUILD_GENERAL1_SMALL
         ImagePullCredentialsType: CODEBUILD
       QueuedTimeoutInMinutes: 480
@@ -278,4 +278,4 @@ Resources:
           GroupName: !Ref OrgBuildLogGroup
           Status: ENABLED
       SourceVersion: refs/heads/master
-      TimeoutInMinutes: 60
+      TimeoutInMinutes: 180

src/build-tasks/build-configuration.ts

@@ -36,6 +36,19 @@ export class BuildConfiguration {
         return result;
     }
 
+    public enumPrintTasks(command: IPerformTasksCommandArgs): IBuildTask[] {
+        this.fixateOrganizationFile(command);
+        const result: IBuildTask[] = [];
+        for (const taskConfig of this.tasks) {
+            const task = BuildTaskProvider.createPrintTask(taskConfig, command);
+            if (task !== undefined) {
+                result.push(task);
+            }
+        }
+
+        return result;
+    }
+
     public enumBuildTasks(command: IPerformTasksCommandArgs): IBuildTask[] {
         this.fixateOrganizationFile(command);
         const result: IBuildTask[] = [];

src/build-tasks/build-runner.ts

@@ -12,6 +12,16 @@ export class BuildRunner {
         };
         await GenericTaskRunner.RunTasks<IBuildTask>(tasks, delegate);
     }
+    public static async RunPrintTasks(tasks: IBuildTask[], logVerbose: boolean, maxConcurrentTasks = 1, failedTasksTolerance = 0): Promise<void> {
+        const delegate: ITaskRunnerDelegates<IBuildTask> = {
+            getName: task => `Task ${task.name}`,
+            getVerb: () => 'print',
+            maxConcurrentTasks,
+            failedTasksTolerance,
+            logVerbose,
+        };
+        await GenericTaskRunner.RunTasks<IBuildTask>(tasks, delegate);
+    }
     public static async RunValidationTasks(tasks: IBuildTask[], logVerbose: boolean, maxConcurrentTasks = 1, failedTasksTolerance = 0): Promise<void> {
         const delegate: ITaskRunnerDelegates<IBuildTask> = {
             getName: task => `Task ${task.name}`,

src/build-tasks/build-task-provider.ts

@@ -43,6 +43,14 @@ export class BuildTaskProvider {
         return validationTask;
     }
 
+    static createPrintTask(configuration: IBuildTaskConfiguration, command: IPerformTasksCommandArgs): IBuildTask {
+        const taskProvider = this.GetBuildTaskProvider();
+        const provider = taskProvider.providers[configuration.Type];
+        if (provider === undefined) {throw new OrgFormationError(`unable to load file ${configuration.FilePath}, unknown configuration type ${configuration.Type}`);}
+        const validationTask = provider.createTaskForPrint(configuration, command);
+        return validationTask;
+    }
+
     public static createBuildTask(configuration: IBuildTaskConfiguration, command: IPerformTasksCommandArgs): IBuildTask {
         const taskProvider = this.GetBuildTaskProvider();
         const provider = taskProvider.providers[configuration.Type];
@@ -104,5 +112,6 @@ export interface IBuildTaskProvider<TConfig extends IBuildTaskConfiguration> {
     type: string;
     createTask(config: TConfig, command: IPerformTasksCommandArgs): IBuildTask;
     createTaskForValidation(config: TConfig, command: IPerformTasksCommandArgs): IBuildTask | undefined;
+    createTaskForPrint(config: TConfig, command: IPerformTasksCommandArgs): IBuildTask | undefined;
     createTaskForCleanup(logicalId: string, physicalId: string, command: IPerformTasksCommandArgs): IBuildTask | undefined;
 }