Merge pull request #65 from dasmeta/DMVP-3029

fix(DMVP-3029): Cloudwatch and s3 to trigger lambda

modules/cloudwatch-to-lambda/README.md

@@ -0,0 +1,43 @@
+# cloudfront-to-s3-to-cloudwatch
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_lambda"></a> [lambda](#module\_lambda) | terraform-aws-modules/lambda/aws | 4.7.1 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_cloudwatch_log_subscription_filter.subscription_filter](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_subscription_filter) | resource |
+| [aws_lambda_permission.cloudwatch_permission_to_trigger_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_cloudwatch_log_group_names"></a> [cloudwatch\_log\_group\_names](#input\_cloudwatch\_log\_group\_names) | Cloudwatch Log group name | `list(string)` | n/a | yes |
+| <a name="input_lambda_configs"></a> [lambda\_configs](#input\_lambda\_configs) | Lambda Configs | <pre>object({<br>    environment_variables = optional(object({<br>      url = optional(string, "")<br>    }), {})<br>  })</pre> | `{}` | no |
+| <a name="input_name"></a> [name](#input\_name) | Lambda name | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_name"></a> [name](#output\_name) | n/a |
+| <a name="output_role_arn"></a> [role\_arn](#output\_role\_arn) | n/a |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/cloudwatch-to-lambda/data.tf

@@ -0,0 +1,3 @@
+data "aws_region" "current" {}
+
+data "aws_caller_identity" "current" {}

modules/cloudwatch-to-lambda/main.tf

@@ -0,0 +1,14 @@
+module "lambda" {
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "4.7.1"
+
+  function_name = var.name
+  handler       = "lambda.handler"
+  runtime       = "python3.7"
+  publish       = true
+  source_path   = "${path.module}/src/"
+
+  role_name = var.name
+
+  environment_variables = try(var.lambda_configs.environment_variables, {})
+}

modules/cloudwatch-to-lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "name" {
+  value = module.lambda.lambda_function_name
+}
+
+output "role_arn" {
+  value = module.lambda.lambda_role_arn
+}

modules/cloudwatch-to-lambda/src/lambda.py

@@ -0,0 +1,60 @@
+import logging
+import json
+import os
+import base64
+import gzip
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+LOGLEVEL = os.environ.get('LOG_LEVEL', 'INFO').upper()
+FALLBACK_SUBJECT = 'AWS Alerts'
+
+logger = logging.getLogger()
+logger.setLevel(getattr(logging, LOGLEVEL))
+
+logger.info("log level: {}".format(LOGLEVEL))
+
+def request_sender(uncompressed_data):
+    url = os.environ['url']
+    headers = {'Content-Type': 'application/json'}
+    method = 'POST'
+
+    req = Request(
+        url,
+        uncompressed_data,
+        headers, method=method
+    )
+
+    try:
+        response = urlopen(req)
+        response.read()
+        logger.info("Message posted")
+        return {"status": "200 OK"}
+    except HTTPError as e:
+        logger.debug(e)
+        logger.error("Request failed: %d %s", e.code, e.reason)
+        return e
+    except URLError as e:
+        logger.error("Server connection failed: %s", e.reason)
+        return e
+
+def handler(event, context):
+    data = event["awslogs"]["data"]
+    print("event")
+    print(event)
+    print("data")
+    print(data)
+    try:
+        # Decode the base64 data
+        decoded_data = base64.b64decode(data)
+        print("decoded_data")
+        print(decoded_data)
+        uncompressed_data = gzip.decompress(decoded_data)
+        print("Uncompressed data: ")
+        print(uncompressed_data)
+        result = json.loads(uncompressed_data)
+        print("Json Loads: ")
+        print(result)
+        request_sender(result)
+    except base64.binascii.Error as e:
+        print("Error decoding base64 data:", e)

modules/cloudwatch-to-lambda/subscription.tf

@@ -0,0 +1,22 @@
+resource "aws_cloudwatch_log_subscription_filter" "subscription_filter" {
+  for_each = { for log_name in var.cloudwatch_log_group_names : log_name => log_name }
+
+  name            = "${var.name}_${each.value}"
+  log_group_name  = each.value
+  filter_pattern  = ""
+  destination_arn = module.lambda.lambda_function_arn
+
+  depends_on = [
+    aws_lambda_permission.cloudwatch_permission_to_trigger_lambda
+  ]
+}
+
+resource "aws_lambda_permission" "cloudwatch_permission_to_trigger_lambda" {
+  for_each = { for log_name in var.cloudwatch_log_group_names : log_name => log_name }
+
+  statement_id  = "AllowExecutionFromCloudWatch"
+  action        = "lambda:InvokeFunction"
+  function_name = module.lambda.lambda_function_arn
+  principal     = "logs.${data.aws_region.current.name}.amazonaws.com"
+  source_arn    = "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:${each.value}:*"
+}

modules/cloudwatch-to-lambda/tests/basic/README.md

@@ -0,0 +1,29 @@
+# basic
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_cloudwatch-to-lambda"></a> [cloudwatch-to-lambda](#module\_cloudwatch-to-lambda) | ../../ | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/cloudwatch-to-lambda/tests/basic/customised_provider.tftest.hcl

@@ -0,0 +1,15 @@
+run "valid_string_concat" {
+  module {
+    source = "../../"
+  }
+  variables {
+    name                       = "CloudwatchtoLambda"
+    cloudwatch_log_group_names = ["aws-cloudtrail-logs-753273213401-85b50618"]
+
+    lambda_configs = {
+      environment_variables = {
+        url = "https://4eb8-195-250-6etwyeywtey-4w32.ngrok-free.app/aws"
+      }
+    }
+  }
+}

modules/cloudwatch-to-lambda/tests/basic/main.tf

@@ -0,0 +1,12 @@
+module "cloudwatch-to-lambda" {
+  source = "../../"
+
+  name                       = "CloudwatchtoLambda"
+  cloudwatch_log_group_names = ["aws-cloudtrail-logs-753273213401-85b50618"]
+
+  lambda_configs = {
+    environment_variables = {
+      url = "https://4eb8-195-250-69-234.ngrok-free.app/aws"
+    }
+  }
+}

modules/cloudwatch-to-lambda/variables.tf

@@ -0,0 +1,19 @@
+variable "name" {
+  type        = string
+  description = "Lambda name"
+}
+
+variable "cloudwatch_log_group_names" {
+  type        = list(string)
+  description = "Cloudwatch Log group name"
+}
+
+variable "lambda_configs" {
+  type = object({
+    environment_variables = optional(object({
+      url = optional(string, "")
+    }), {})
+  })
+  default     = {}
+  description = "Lambda Configs"
+}

modules/s3-to-lambda/README.md

@@ -0,0 +1,41 @@
+# cloudfront-to-s3-to-cloudwatch
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_lambda"></a> [lambda](#module\_lambda) | terraform-aws-modules/lambda/aws | 4.7.1 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_lambda_permission.s3_permission_to_trigger_lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lambda_permission) | resource |
+| [aws_s3_bucket_notification.incoming](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_notification) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_bucket_name"></a> [bucket\_name](#input\_bucket\_name) | S3 bucket for s3 subscription | `list(string)` | n/a | yes |
+| <a name="input_lambda_configs"></a> [lambda\_configs](#input\_lambda\_configs) | Lambda Configs | <pre>object({<br>    environment_variables = optional(object({<br>      url = optional(string, "")<br>    }), {})<br>  })</pre> | `{}` | no |
+| <a name="input_name"></a> [name](#input\_name) | Lambda name | `string` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_name"></a> [name](#output\_name) | n/a |
+| <a name="output_role_arn"></a> [role\_arn](#output\_role\_arn) | n/a |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/s3-to-lambda/main.tf

@@ -0,0 +1,14 @@
+module "lambda" {
+  source  = "terraform-aws-modules/lambda/aws"
+  version = "4.7.1"
+
+  function_name = var.name
+  handler       = "lambda.handler"
+  runtime       = "python3.7"
+  publish       = true
+  source_path   = "${path.module}/src/"
+
+  role_name = var.name
+
+  environment_variables = try(var.lambda_configs.environment_variables, {})
+}

modules/s3-to-lambda/outputs.tf

@@ -0,0 +1,7 @@
+output "name" {
+  value = module.lambda.lambda_function_name
+}
+
+output "role_arn" {
+  value = module.lambda.lambda_role_arn
+}

modules/s3-to-lambda/src/lambda.py

@@ -0,0 +1,43 @@
+import logging
+import json
+import os
+import requests
+import importlib
+import sys
+import types
+from urllib.request import Request, urlopen
+from urllib.error import URLError, HTTPError
+
+LOGLEVEL = os.environ.get('LOG_LEVEL', 'INFO').upper()
+FALLBACK_SUBJECT = 'AWS Alerts'
+
+logger = logging.getLogger()
+logger.setLevel(getattr(logging, LOGLEVEL))
+
+logger.info("log level: {}".format(LOGLEVEL))
+
+def handler(event, context):
+
+    url = "https://4eb8-195-250-69-234.ngrok-free.app/aws"
+    headers = {'Content-Type': 'application/json'}
+    method = 'POST'
+
+    print("lambda Request: ")
+    req = Request(
+        url,
+        json.dumps(event).encode('utf-8'),
+        headers, method=method
+    )
+
+    try:
+        response = urlopen(req)
+        response.read()
+        logger.info("Message posted")
+        return {"status": "200 OK"}
+    except HTTPError as e:
+        logger.debug(e)
+        logger.error("Request failed: %d %s", e.code, e.reason)
+        return e
+    except URLError as e:
+        logger.error("Server connection failed: %s", e.reason)
+        return e

modules/s3-to-lambda/subscription.tf

@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_notification" "incoming" {
+  for_each = { for bucket in var.bucket_name : bucket => bucket }
+  bucket   = each.value
+
+  lambda_function {
+    lambda_function_arn = module.lambda.lambda_function_arn
+    events              = ["s3:ObjectCreated:Put"]
+  }
+
+  depends_on = [aws_lambda_permission.s3_permission_to_trigger_lambda]
+}
+
+resource "aws_lambda_permission" "s3_permission_to_trigger_lambda" {
+  for_each = { for bucket in var.bucket_name : bucket => bucket }
+
+  statement_id  = "AllowExecutionFromS3Bucket"
+  action        = "lambda:InvokeFunction"
+  function_name = module.lambda.lambda_function_arn
+  principal     = "s3.amazonaws.com"
+  source_arn    = "arn:aws:s3:::${each.value}"
+}

modules/s3-to-lambda/tests/basic/README.md

@@ -0,0 +1,29 @@
+# basic
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+No requirements.
+
+## Providers
+
+No providers.
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_s3-to-lambda"></a> [s3-to-lambda](#module\_s3-to-lambda) | ../../ | n/a |
+
+## Resources
+
+No resources.
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->