Allow using S3 as lambda source (#13)

* Add option to allow S3 bucket as source option - will be a global setting * Fix * Added changelog entry * Rename local name
covidgreen · Aug 25, 2020 · 7dd0e1f · 7dd0e1f
1 parent a358477
commit 7dd0e1f
Show file tree

Hide file tree

Showing 15 changed files with 198 additions and 103 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file.
 
 
 ## [Unreleased]
+- Added: Option to use an S3 bucket as the source for lambdas, will be a global setting and we do not manage this bucket as this is a non default option
 - Added: Added option to send callback notifications using email via an SNS topic - subscription will not be automated
 - Fixed: Altered the ECS image so the custom vars are just for the image and do not include the tag, we append the tag if using a custom image using the tag var
 - Fixed: Fixed the "bastion_amazon_ssm_managed_instance_core" aws_iam_role_policy_attachment (Incorrect casing in name), this will result in a Terraform apply failure when applied, can run a second time to fix

diff --git a/lambda-authorizer.tf b/lambda-authorizer.tf
@@ -58,15 +58,19 @@ resource "aws_iam_role_policy_attachment" "authorizer_logs" {
 }
 
 resource "aws_lambda_function" "authorizer" {
-  filename         = "${path.module}/.zip/${module.labels.id}_authorizer.zip"
-  function_name    = "${module.labels.id}-authorizer"
-  source_code_hash = data.archive_file.authorizer.output_base64sha256
-  role             = aws_iam_role.authorizer.arn
-  runtime          = "nodejs12.x"
-  handler          = "authorizer.handler"
-  memory_size      = var.lambda_authorizer_memory_size
-  timeout          = var.lambda_authorizer_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_authorizer.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_authorizer_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.authorizer.output_base64sha256
+
+  function_name = "${module.labels.id}-authorizer"
+  handler       = "authorizer.handler"
+  memory_size   = var.lambda_authorizer_memory_size
+  role          = aws_iam_role.authorizer.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_authorizer_timeout
 
   depends_on = [aws_cloudwatch_log_group.authorizer]
 

diff --git a/lambda-callback.tf b/lambda-callback.tf
@@ -103,23 +103,22 @@ resource "aws_iam_role_policy_attachment" "callback_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "callback" {
-  filename         = "${path.module}/.zip/${module.labels.id}_callback.zip"
-  function_name    = "${module.labels.id}-callback"
-  source_code_hash = data.archive_file.callback.output_base64sha256
-  role             = aws_iam_role.callback.arn
-  runtime          = "nodejs12.x"
-  handler          = "callback.handler"
-  memory_size      = var.lambda_callback_memory_size
-  timeout          = var.lambda_callback_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_callback.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_callback_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.callback.output_base64sha256
+
+  function_name = "${module.labels.id}-callback"
+  handler       = "callback.handler"
+  memory_size   = var.lambda_callback_memory_size
+  role          = aws_iam_role.callback.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_callback_timeout
 
   depends_on = [aws_cloudwatch_log_group.callback]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -132,6 +131,11 @@ resource "aws_lambda_function" "callback" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
 }
 
 resource "aws_lambda_event_source_mapping" "callback" {

diff --git a/lambda-cso.tf b/lambda-cso.tf
@@ -77,24 +77,24 @@ resource "aws_iam_role_policy_attachment" "cso_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "cso" {
-  count            = local.lambda_cso_count
-  filename         = "${path.module}/.zip/${module.labels.id}_cso.zip"
-  function_name    = "${module.labels.id}-cso"
-  source_code_hash = data.archive_file.cso.output_base64sha256
-  role             = aws_iam_role.cso[0].arn
-  runtime          = "nodejs12.x"
-  handler          = "cso.handler"
-  memory_size      = var.lambda_cso_memory_size
-  timeout          = var.lambda_cso_timeout
-  tags             = module.labels.tags
+  count = local.lambda_cso_count
+
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_cso.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_cso_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.cso.output_base64sha256
+
+  function_name = "${module.labels.id}-cso"
+  handler       = "cso.handler"
+  memory_size   = var.lambda_cso_memory_size
+  role          = aws_iam_role.cso[0].arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_cso_timeout
 
   depends_on = [aws_cloudwatch_log_group.cso]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -107,6 +107,11 @@ resource "aws_lambda_function" "cso" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
 }
 
 resource "aws_cloudwatch_event_rule" "cso_schedule" {

diff --git a/lambda-daily-registrations-reporter.tf b/lambda-daily-registrations-reporter.tf
@@ -29,6 +29,8 @@ module "daily_registrations_reporter" {
   kms_writer_arns                = [aws_kms_key.sns.arn]
   log_retention_days             = var.logs_retention_days
   memory_size                    = var.lambda_daily_registrations_reporter_memory_size
+  s3_bucket                      = var.lambdas_custom_s3_bucket
+  s3_key                         = var.lambda_daily_registrations_reporter_s3_key
   security_group_ids             = [module.lambda_sg.id]
   sns_topic_arns_to_publish_to   = aws_sns_topic.daily_registrations_reporter.*.arn
   subnet_ids                     = module.vpc.private_subnets

diff --git a/lambda-download.tf b/lambda-download.tf
@@ -23,6 +23,8 @@ module "download" {
   handler                        = "download.handler"
   log_retention_days             = var.logs_retention_days
   memory_size                    = var.lambda_download_memory_size
+  s3_bucket                      = var.lambdas_custom_s3_bucket
+  s3_key                         = var.lambda_download_s3_key
   security_group_ids             = [module.lambda_sg.id]
   subnet_ids                     = module.vpc.private_subnets
   tags                           = module.labels.tags

diff --git a/lambda-exposures.tf b/lambda-exposures.tf
@@ -78,23 +78,22 @@ resource "aws_iam_role_policy_attachment" "exposures_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "exposures" {
-  filename         = "${path.module}/.zip/${module.labels.id}_exposures.zip"
-  function_name    = "${module.labels.id}-exposures"
-  source_code_hash = data.archive_file.exposures.output_base64sha256
-  role             = aws_iam_role.exposures.arn
-  runtime          = "nodejs12.x"
-  handler          = "exposures.handler"
-  memory_size      = var.lambda_exposures_memory_size
-  timeout          = var.lambda_exposures_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_exposures.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_exposures_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.exposures.output_base64sha256
+
+  function_name = "${module.labels.id}-exposures"
+  handler       = "exposures.handler"
+  memory_size   = var.lambda_exposures_memory_size
+  role          = aws_iam_role.exposures.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_exposures_timeout
 
   depends_on = [aws_cloudwatch_log_group.exposures]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -107,6 +106,11 @@ resource "aws_lambda_function" "exposures" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
 }
 
 resource "aws_cloudwatch_event_rule" "exposures_schedule" {

diff --git a/lambda-settings.tf b/lambda-settings.tf
@@ -74,23 +74,22 @@ resource "aws_iam_role_policy_attachment" "settings_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "settings" {
-  filename         = "${path.module}/.zip/${module.labels.id}_settings.zip"
-  function_name    = "${module.labels.id}-settings"
-  source_code_hash = data.archive_file.settings.output_base64sha256
-  role             = aws_iam_role.settings.arn
-  runtime          = "nodejs12.x"
-  handler          = "settings.handler"
-  memory_size      = var.lambda_settings_memory_size
-  timeout          = var.lambda_settings_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_settings.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_settings_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.settings.output_base64sha256
+
+  function_name = "${module.labels.id}-settings"
+  handler       = "settings.handler"
+  memory_size   = var.lambda_settings_memory_size
+  role          = aws_iam_role.settings.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_settings_timeout
 
   depends_on = [aws_cloudwatch_log_group.settings]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -103,6 +102,11 @@ resource "aws_lambda_function" "settings" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
 }
 
 resource "aws_cloudwatch_event_rule" "settings_schedule" {

diff --git a/lambda-sms.tf b/lambda-sms.tf
@@ -30,6 +30,8 @@ module "sms" {
   kms_reader_arns                            = [aws_kms_key.sqs.arn]
   log_retention_days                         = var.logs_retention_days
   memory_size                                = var.lambda_sms_memory_size
+  s3_bucket                                  = var.lambdas_custom_s3_bucket
+  s3_key                                     = var.lambda_sms_s3_key
   security_group_ids                         = [module.lambda_sg.id]
   sqs_queue_arns_to_consume_from             = [aws_sqs_queue.sms.arn]
   subnet_ids                                 = module.vpc.private_subnets

diff --git a/lambda-stats.tf b/lambda-stats.tf
@@ -75,23 +75,22 @@ resource "aws_iam_role_policy_attachment" "stats_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "stats" {
-  filename         = "${path.module}/.zip/${module.labels.id}_stats.zip"
-  function_name    = "${module.labels.id}-stats"
-  source_code_hash = data.archive_file.stats.output_base64sha256
-  role             = aws_iam_role.stats.arn
-  runtime          = "nodejs12.x"
-  handler          = "stats.handler"
-  memory_size      = var.lambda_stats_memory_size
-  timeout          = var.lambda_stats_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_stats.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_stats_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.stats.output_base64sha256
+
+  function_name = "${module.labels.id}-stats"
+  handler       = "stats.handler"
+  memory_size   = var.lambda_stats_memory_size
+  role          = aws_iam_role.stats.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_stats_timeout
 
   depends_on = [aws_cloudwatch_log_group.stats]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -104,6 +103,11 @@ resource "aws_lambda_function" "stats" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
 }
 
 module "lambda_sg" {

diff --git a/lambda-token.tf b/lambda-token.tf
@@ -72,23 +72,22 @@ resource "aws_iam_role_policy_attachment" "token_aws_managed_policy" {
 }
 
 resource "aws_lambda_function" "token" {
-  filename         = "${path.module}/.zip/${module.labels.id}_token.zip"
-  function_name    = "${module.labels.id}-token"
-  source_code_hash = data.archive_file.token.output_base64sha256
-  role             = aws_iam_role.token.arn
-  runtime          = "nodejs12.x"
-  handler          = "token.handler"
-  memory_size      = var.lambda_token_memory_size
-  timeout          = var.lambda_token_timeout
-  tags             = module.labels.tags
+  # Default is to use the stub file, but we need to cater for S3 bucket file being the source
+  filename         = local.lambdas_use_s3_as_source ? null : "${path.module}/.zip/${module.labels.id}_token.zip"
+  s3_bucket        = local.lambdas_use_s3_as_source ? var.lambdas_custom_s3_bucket : null
+  s3_key           = local.lambdas_use_s3_as_source ? var.lambda_token_s3_key : null
+  source_code_hash = local.lambdas_use_s3_as_source ? "" : data.archive_file.token.output_base64sha256
+
+  function_name = "${module.labels.id}-token"
+  handler       = "token.handler"
+  memory_size   = var.lambda_token_memory_size
+  role          = aws_iam_role.token.arn
+  runtime       = "nodejs12.x"
+  tags          = module.labels.tags
+  timeout       = var.lambda_token_timeout
 
   depends_on = [aws_cloudwatch_log_group.token]
 
-  vpc_config {
-    security_group_ids = [module.lambda_sg.id]
-    subnet_ids         = module.vpc.private_subnets
-  }
-
   environment {
     variables = {
       CONFIG_VAR_PREFIX = local.config_var_prefix,
@@ -101,4 +100,11 @@ resource "aws_lambda_function" "token" {
       source_code_hash,
     ]
   }
+
+  vpc_config {
+    security_group_ids = [module.lambda_sg.id]
+    subnet_ids         = module.vpc.private_subnets
+  }
+
+
 }
diff --git a/lambda-upload.tf b/lambda-upload.tf
@@ -23,6 +23,8 @@ module "upload" {
   handler                        = "upload.handler"
   log_retention_days             = var.logs_retention_days
   memory_size                    = var.lambda_upload_memory_size
+  s3_bucket                      = var.lambdas_custom_s3_bucket
+  s3_key                         = var.lambda_upload_s3_key
   security_group_ids             = [module.lambda_sg.id]
   subnet_ids                     = module.vpc.private_subnets
   tags                           = module.labels.tags

diff --git a/locals.tf b/locals.tf
@@ -48,6 +48,11 @@ locals {
   lambda_download_count                     = contains(var.optional_lambdas_to_include, "download") ? 1 : 0
   lambda_upload_count                       = contains(var.optional_lambdas_to_include, "upload") ? 1 : 0
 
+  # Lambdas using S3 bucket as source - is a global value, so will apply to all of them
+  # If set will assume the S3 key is provided and that a file exists in the bucket
+  # Since this is an override, we do not manage this bucket or access to the same
+  lambdas_use_s3_as_source = var.lambdas_custom_s3_bucket != ""
+
   # RDS enhanced monitoring count
   rds_enhanced_monitoring_enabled_count = var.rds_enhanced_monitoring_interval > 0 ? 1 : 0