Merge pull request #9 from nearform/current

Several changes version v0.1.1

.editorconfig

@@ -0,0 +1,8 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+charset = utf-8
+trim_trailing_whitespace = false
+insert_final_newline = false

.gitignore

@@ -1,4 +1,6 @@
 .terraform
 *.tfplan
 *.pem
-*.zip
+*.zip
+
+.idea

.pre-commit-config.yaml

@@ -0,0 +1,16 @@
+# See https://github.com/gruntwork-io/pre-commit
+# Will need to install some pre-requisites
+#       pre-commit      https://pre-commit.com/#install
+#       tflint          https://github.com/terraform-linters/tflint/
+# Can then install with
+#       pre-commit install
+# Can do a manual run with
+#       pre-commit run
+
+repos:
+  - repo: https://github.com/gruntwork-io/pre-commit
+    rev: v0.1.9                                                 # Get the latest from: https://github.com/gruntwork-io/pre-commit/releases
+    hooks:
+      - id: terraform-fmt
+      - id: tflint 
+        args: ["--deep", "--module"]

CHANGELOG.md

@@ -0,0 +1,34 @@
+# Change Log
+
+All notable changes to this project will be documented in this file.
+
+## [Unreleased]
+- Added: ALB logging - both ALBs log to the same bucket, using distinct prefixes - api and push
+- Updated: Upgraded AWS provider from = 2.68.0 to ~> 2.70.0
+- Updated: Switched to using templatefile function rather than deprecated template provider
+
+
+## [v0.1.1] 2020-08-13
+- Added: Added ability to set ECS image url and image tag overrides, this is based on @segfault's PR at https://github.com/covidgreen/covid-green-infra/pull/4
+- Updated: Switched lambdas from using the AWSLambdaBasicExecutionRole to AWSLambdaVPCAccessExecutionRole managed policy
+- Updated: Renamed the "root_profile" var and "root" AWS provider to "dns" as this is confusing, removed a redundant aws provider "root_us"
+- Updated: Added changes @segfault added re explicit usage of the AWS CLI `--output json` usage in some of the scripts
+- Removed: Removed Terraform validate from the pre-commit hook config as this is being used as a module, have left the s3 backend config for now
+- Updated: Switched all the lambdas from nodejs10.x to nodejs12.x
+- Added: Use variables for the lambda memory size and timeout attributes with defaults, so we can configure via env-vars files
+- Removed: Extracted cti, gct and ni content into specific repos - Will not be managed by this repo
+- Added: Added new AWS parameters certificate_audience and jwt_issuer and removed security_exposure_limit AWS parameter
+- Added: Docs on the 2 approaches to managing a project - external to this repo and internal to this repo
+- Added: Added RDS reader/writer endpoint outputs
+- Added: Surfaced bastion ASG desired count as a variable, will need when we use this repo as a module
+- Added: Switched to using path.module prefixes for the CloudWatch dashboard template and the ECS container defintion templates
+- Fixed: Changed the create TF store backend script to cater for us-east-1 being a special case - Location constraints
+- Fixed: Replace all refs to cti, gct and ni with xyz in docs and shell script comments - this is just prep for the open source branch
+- Added: Added the following optional lambdas to the operators group execute list: daily-registrations-reporter, download and upload
+- Added: Pre-commit hook to include TF fmt, validation and linting
+- Fixed: Linting issues - no logic
+- Added: Split RDS user usage so we no longer need to use the master credentials
+
+
+## [v0.1.0] 2020-08-13
+- Initial content

Makefile

@@ -3,8 +3,8 @@
 # 	make xyz-dev-init
 # 	make xyz-dev-plan
 # 	make xyz-dev-apply
-# We assume you have an AWS profile with the the PROJECT-ENVIRONMENT so in above case xyz-dev
-# 	This is the same profile that should exist in the env-vars file
+# We assume you have an AWS profile named PROJECT-ENVIRONMENT i.e. xyz-dev
+# 	This is the same profile that should exist in the env-vars files
 
 .ONESHELL:
 .SHELL := /bin/bash
@@ -48,6 +48,7 @@ AWS_PROFILE = $(PROJECT_ENVIRONMENT_KEY)
 # Determine from tf vars file(s) - will use the first match
 AWS_REGION = $(shell grep '^aws_region' $(PROJECT_ENVIRONMENT_TFVAR_FILE) $(PROJECT_TFVAR_FILE) | head -n 1 | cut -d '"' -f 2)
 
+# NOTE: Uses a convention
 TERRAFORM_BACKEND_BUCKET ?= $(PROJECT_ENVIRONMENT_KEY)-terraform-store
 TERRAFORM_BACKEND_KEY ?= $(PROJECT_KEY)
 TERRAFORM_BACKEND_TABLE ?= $(PROJECT_ENVIRONMENT_KEY)-terraform-lock
@@ -65,24 +66,6 @@ xyz-dev-plan:
 xyz-dev-apply:
 	@$(MAKE_TF_APPLY) ENVIRONMENT=dev PROJECT_KEY=xyz
 
-# QA
-.PHONY: xyz-qa-init xyz-qa-plan xyz-qa-apply
-xyz-qa-init:
-	@$(MAKE_TF_INIT) ENVIRONMENT=qa PROJECT_KEY=xyz
-xyz-qa-plan:
-	@$(MAKE_TF_PLAN) ENVIRONMENT=qa PROJECT_KEY=xyz
-xyz-qa-apply:
-	@$(MAKE_TF_APPLY) ENVIRONMENT=qa PROJECT_KEY=xyz
-
-# PROD
-.PHONY: xyz-prod-init xyz-prod-plan xyz-prod-apply
-xyz-prod-init:
-	@$(MAKE_TF_INIT) ENVIRONMENT=prod PROJECT_KEY=xyz
-xyz-prod-plan:
-	@$(MAKE_TF_PLAN) ENVIRONMENT=prod PROJECT_KEY=xyz
-xyz-prod-apply:
-	@$(MAKE_TF_APPLY) ENVIRONMENT=prod PROJECT_KEY=xyz
-
 
 # #########################################
 # Generic targets - not dependant on a project

README.md

@@ -26,20 +26,32 @@ The RDS Aurora cluster is avilable to both `private` and `intra` subnets.
 
 ## Development
 
+We use a git pre-commit hook to do some checks and linting, see the top of [.pre-commit-config.yaml](./.pre-commit-config.yaml) for installation instructions
+
 You need first to set up AWS profiles locally for every AWS account/project/environment you're going to work on. Once done change it in the project/environment variables override files in `env-vars`. The project uses 2 different AWS profiles, one to manage the infrastructure and one to manage DNS entries. This is because the AWS account used to spin up an environments could be different from the account from where the DNS zone is registered.
 
 See [Creating a new project](./docs/creating-a-new-project.md) for setting up the Terraform backend setup.
 
 Make file usage
 ```
-# Using the cti project (HSE) and dev environment
-make cti-dev-init
-make cti-dev-plan
-make cti-dev-apply
+# Using the xyz project and dev environment
+make xyz-dev-init
+make xyz-dev-plan
+make xyz-dev-apply
 ```
 
 Every `terraform` command can be launched via the `Makefile`, it will take care of initializing the folder to use different backends, planning, applying, and destroying changes.
 
+### Change log
+
+We maintain a change log [here](./CHANGELOG.md)
+
+### Note
+Sometimes on the plan/apply change(s) will appear for the bastions
+- The bastion AMI is based on using the latest **Amazon Linux 2** AMI - this is fine to apply and will not terminate any running instances
+- The bastion ASG count will show a change from 1 -> 0, this is due to someone having a bastion instance running, in this case you do not want to terminate their instance
+	- Easiest thing to do is alter your local bastion.tf **desired_capacity = 1** and re-run the plan, this way the instance will remain running and this change will no longer appear in the plan
+
 ## Lambdas
 ### authorizer
 Checks a JWT is valid when the Gateway tried to access items in the S3 bucket.
@@ -48,25 +60,34 @@ Checks a JWT is valid when the Gateway tried to access items in the S3 bucket.
 Used to collect symptom info from the app from an SQS queue. Currently unused.
 
 ### cso - Optional
-This lambda is specific to the Irish app and compiles symptom info into a CSV file. The file is then encrypted using GPG (symmetric key in Secrets) and then uploaded via SFTP to the Central Statistics Office in Ireland.
+This lambda is specific to the Irish app and compiles symptom info into a CSV file. The file is then encrypted using GPG (symmetric key in Secrets) and then uploaded via SFTP to the Central Statistics Office in Ireland. This is obviously not used in Gibraltar.
+
+### daily-registrations-reporter - Optional
+This lambda is currently specific to the Gibraltar app and generates a report of cumulative API registrations by day for the app, which it sends to an SNS topic.
+
+### download - Optional
+This lambda is for environments involved in testing the interoperability service. Downloads any new batches of exposure keys from the interop service since the lambda last ran, and stores them ready to be included in future export files.
 
 ### exposures
 Generates exposure files in zip format for the S3 bucket. Those files contain the encrypted contact tracing information the phone API uses to determine if you have had a close contact with someone. This lambda runs on a schedule, selecting the exposure info from the database and making the archive available once complete.
 
-### notify_slack
-For calling Slack web hooks with info.
+### settings
+This lambda is used to generate a settings.json file which contains values that can override the app defaults.
+This saves us having to go through a full App Store release cycle to change minor details like phone numbers etc.
+
+### sms
+This lambda is triggered by the SMS SQS queue, and allows project specific handling of sending SMS via different providers.
 
 ### stats
 This lambda is used to generate a daily stats.json file from a web service run by the Central Statistics Office in Ireland.
 This info is used in the Irish app to power various graphs and info screens.
 
-### settings
-This lambda is used to generate a settings.json file which contains values that can override the app defaults.
-This saves us having to go through a full App Store release cycle to change minor details like phone numbers etc.
-
 ### token
 This lambda is used to generate tokens for testing. It is not used by clients or end users. The phone app and backend APIs make use of a service called Device check which validates that we are talking to an actual device. To get around this for testing, we have a lambda that can generate two different kinds of token, one for register and one for push. The register token allows you to bypass the checks in the backend API and the push token works for the push API service.
 
+### upload - Optional
+This lambda is for environments involved in testing the interoperability service. Uploads any new exposure keys to the interop service, ready for other back-ends to download.
+
 ## AWS secrets and parameters
 Secrets are stored in AWS Secrets Manager, these are populated outside of this Terraform content.
 - Some are optional as they are not used by all instances

alb.tf

@@ -29,8 +29,13 @@ resource "aws_lb" "api" {
   enable_http2                     = true
   ip_address_type                  = "dualstack"
   enable_deletion_protection       = true
+  tags                             = module.labels.tags
 
-  tags = module.labels.tags
+  access_logs {
+    bucket  = module.alb_logs.aws_logs_bucket
+    prefix  = "api"
+    enabled = true
+  }
 }
 
 resource "aws_lb_target_group" "api" {
@@ -120,8 +125,13 @@ resource "aws_lb" "push" {
   enable_http2                     = true
   ip_address_type                  = "dualstack"
   enable_deletion_protection       = true
+  tags                             = module.labels.tags
 
-  tags = module.labels.tags
+  access_logs {
+    bucket  = module.alb_logs.aws_logs_bucket
+    prefix  = "push"
+    enabled = true
+  }
 }
 
 resource "aws_lb_target_group" "push" {
@@ -160,3 +170,20 @@ resource "aws_lb_listener" "push_https" {
     type             = "forward"
   }
 }
+
+# #########################################
+# ALB logs - single bucket for both ALBs each with their own prefix
+# #########################################
+module "alb_logs" {
+  source  = "trussworks/logs/aws"
+  version = "8.1.0"
+
+  alb_logs_prefixes       = ["api", "push"]
+  allow_alb               = true
+  default_allow           = false
+  force_destroy           = true
+  region                  = var.aws_region
+  s3_bucket_name          = format("%s-alb-logs", module.labels.id)
+  s3_log_bucket_retention = var.logs_retention_days
+  tags                    = module.labels.tags
+}

bastion.tf

@@ -14,7 +14,7 @@ data "aws_ami" "amazon_linux_2" {
 
 resource "aws_autoscaling_group" "bastion" {
   count               = local.bastion_enabled_count
-  desired_capacity    = 1
+  desired_capacity    = var.bastion_asg_desired_count
   max_size            = 1
   min_size            = 0
   name                = format("%s-bastion", module.labels.id)

dashboard.tf

@@ -1,25 +1,21 @@
-data "template_file" "dashboard" {
-  template = file("templates/dashboard.json")
-
-  vars = {
-    account_id            = data.aws_caller_identity.current.account_id
-    region                = var.aws_region
-    environment           = var.environment
-    gateway_name          = "${module.labels.id}-gw"
-    ecs_cluster_name      = module.labels.id
-    ecs_push_service_name = aws_ecs_service.push.name
-    ecs_api_service_name  = aws_ecs_service.api.name
-    rds_db_cluster_name   = module.rds_cluster_aurora_postgres.cluster_identifier
-    lambda_token_fn_name  = "${module.labels.id}-token"
-    lambda_cso_fn_name    = "${module.labels.id}-cso"
-    lambda_stats_fn_name  = "${module.labels.id}-stats"
-    api_lb_arn_suffix     = aws_lb.api.arn_suffix
-    push_lb_arn_suffix    = aws_lb.push.arn_suffix
-    api_log_group         = "${module.labels.id}-api"
-  }
-}
-
 resource "aws_cloudwatch_dashboard" "monitoring_alarms_dashboard" {
   dashboard_name = module.labels.id
-  dashboard_body = data.template_file.dashboard.rendered
-}
+
+  dashboard_body = templatefile(format("%s/templates/dashboard.json", path.module),
+    {
+      account_id            = data.aws_caller_identity.current.account_id
+      region                = var.aws_region
+      environment           = var.environment
+      gateway_name          = "${module.labels.id}-gw"
+      ecs_cluster_name      = module.labels.id
+      ecs_push_service_name = aws_ecs_service.push.name
+      ecs_api_service_name  = aws_ecs_service.api.name
+      rds_db_cluster_name   = module.rds_cluster_aurora_postgres.cluster_identifier
+      lambda_token_fn_name  = "${module.labels.id}-token"
+      lambda_cso_fn_name    = "${module.labels.id}-cso"
+      lambda_stats_fn_name  = "${module.labels.id}-stats"
+      api_lb_arn_suffix     = aws_lb.api.arn_suffix
+      push_lb_arn_suffix    = aws_lb.push.arn_suffix
+      api_log_group         = "${module.labels.id}-api"
+  })
+}

dns.tf

@@ -3,7 +3,7 @@
 # #########################################
 data "aws_route53_zone" "primary" {
   count        = local.enable_dns_count
-  provider     = aws.root
+  provider     = aws.dns
   name         = var.route53_zone
   private_zone = false
 }
@@ -24,7 +24,7 @@ resource "aws_acm_certificate" "wildcard_cert" {
 
 resource "aws_route53_record" "wildcard_cert_validation" {
   count           = local.enable_certificates_count
-  provider        = aws.root
+  provider        = aws.dns
   name            = aws_acm_certificate.wildcard_cert[0].domain_validation_options.0.resource_record_name
   type            = aws_acm_certificate.wildcard_cert[0].domain_validation_options.0.resource_record_type
   zone_id         = data.aws_route53_zone.primary[0].id
@@ -49,7 +49,7 @@ resource "aws_acm_certificate_validation" "wildcard_cert" {
 
 resource "aws_acm_certificate" "wildcard_cert_us" {
   count             = local.enable_certificates_count
-  provider          = aws.us
+  provider          = aws.us_east_1
   domain_name       = var.wildcard_domain
   validation_method = "DNS"
 
@@ -60,7 +60,7 @@ resource "aws_acm_certificate" "wildcard_cert_us" {
 
 resource "aws_route53_record" "wildcard_cert_validation_us" {
   count           = local.enable_certificates_count
-  provider        = aws.root
+  provider        = aws.dns
   name            = aws_acm_certificate.wildcard_cert_us[0].domain_validation_options.0.resource_record_name
   type            = aws_acm_certificate.wildcard_cert_us[0].domain_validation_options.0.resource_record_type
   zone_id         = data.aws_route53_zone.primary[0].id
@@ -75,7 +75,7 @@ resource "aws_route53_record" "wildcard_cert_validation_us" {
 
 resource "aws_acm_certificate_validation" "wildcard_cert_us" {
   count                   = local.enable_certificates_count
-  provider                = aws.us
+  provider                = aws.us_east_1
   certificate_arn         = aws_acm_certificate.wildcard_cert_us[0].arn
   validation_record_fqdns = [aws_route53_record.wildcard_cert_validation_us[0].fqdn]
 
@@ -89,7 +89,7 @@ resource "aws_acm_certificate_validation" "wildcard_cert_us" {
 # #########################################
 resource "aws_route53_record" "api" {
   count    = local.enable_dns_count
-  provider = aws.root
+  provider = aws.dns
   zone_id  = data.aws_route53_zone.primary[0].id
   name     = var.api_dns
   type     = "A"
@@ -110,7 +110,7 @@ resource "aws_route53_record" "api" {
 
 resource "aws_route53_record" "push" {
   count    = local.enable_dns_count
-  provider = aws.root
+  provider = aws.dns
   zone_id  = data.aws_route53_zone.primary[0].id
   name     = var.push_dns
   type     = "A"

docs/bastion.md

@@ -10,7 +10,7 @@ There is a schedule to scale down the bastions at 21:01 UTC each day - so your s
 - You should then be able to see the instance on the AWS console and can select and hit Connect -> Session Manager
 
 
-# Postgres client installation
+# PostgreSQL client installation
 The postgresql11 package will be installed using cloud-init, but if you need to install can use
 ```sudo amazon-linux-extras install postgresql11```
 
@@ -19,5 +19,5 @@ The postgresql11 package will be installed using cloud-init, but if you need to
 - Anyone wishing to use the bastion will need to have an AWS account
 - Will need to connect via the AWS console
 - If we do an apply if a bastion instance is running it will be terminated as we will reset the desired count to 0
-	- Can temp edit the bastion.tf file and set the desired_count to 1 to avoid this
+	- Can temp edit the bastion_asg_desired_count in the variables.tf file and set the desired_count to 1 to avoid this
 - Since we use a data source to get the latest AWS Linux 2 AMI this may sometimes appear as change in the plan