From e790e640b39a412a2909fe659cfa2b972de76091 Mon Sep 17 00:00:00 2001 From: Martin Burke <66889764+burmek@users.noreply.github.com> Date: Sun, 14 Apr 2024 20:44:20 -0400 Subject: [PATCH 1/2] Expand subnets, add new MPM compute, add new MPM buckets/perms --- main.tf | 13 +++- modules/comet_eks/main.tf | 115 +++++++++++++++++++++++++++------ modules/comet_eks/variables.tf | 35 ++++++++++ modules/comet_s3/main.tf | 54 +++++++++++++--- modules/comet_s3/outputs.tf | 2 +- modules/comet_s3/variables.tf | 5 ++ modules/comet_vpc/main.tf | 2 +- terraform.tfvars | 3 + variables.tf | 41 ++++++++++++ 9 files changed, 237 insertions(+), 33 deletions(-) diff --git a/main.tf b/main.tf index c1fbebb..a88c221 100644 --- a/main.tf +++ b/main.tf @@ -75,6 +75,15 @@ module "comet_eks" { s3_enabled = var.enable_s3 comet_ec2_s3_iam_policy = var.enable_s3 ? module.comet_s3[0].comet_s3_iam_policy_arn : null + + enable_mpm_infra = var.enable_mpm_infra + + eks_druid_instance_type = var.eks_druid_instance_type + eks_druid_node_count = var.eks_druid_node_count + eks_zookeeper_instance_type = var.eks_zookeeper_instance_type + eks_zookeeper_node_count = var.eks_zookeeper_node_count + eks_airflow_instance_type = var.eks_airflow_instance_type + eks_airflow_node_count = var.eks_airflow_node_count } module "comet_elasticache" { @@ -124,6 +133,8 @@ module "comet_s3" { count = var.enable_s3 ? 1 : 0 environment = var.environment - comet_s3_bucket = var.s3_bucket_name + comet_s3_bucket = var.s3_bucket_name s3_force_destroy = var.s3_force_destroy + + enable_mpm_infra = var.enable_mpm_infra } \ No newline at end of file diff --git a/modules/comet_eks/main.tf b/modules/comet_eks/main.tf index 2ad74bf..c913eb3 100644 --- a/modules/comet_eks/main.tf +++ b/modules/comet_eks/main.tf @@ -3,6 +3,9 @@ locals { Terraform = "true" Environment = var.environment } + volume_type = "gp3" + volume_encrypted = false + volume_delete_on_termination = true } data "aws_iam_policy" "ebs_csi_policy" { @@ -22,32 +25,104 @@ module "eks" { eks_managed_node_group_defaults = { ami_type = var.eks_mng_ami_type } - eks_managed_node_groups = { - one = { - name = var.eks_mng_name - instance_types = var.eks_node_types - min_size = var.eks_mng_desired_size - max_size = var.eks_mng_max_size - desired_size = var.eks_mng_desired_size - block_device_mappings = { - xvda = { - device_name = "/dev/xvda" - ebs = { - volume_size = var.eks_mng_disk_size - volume_type = "gp3" - encrypted = false - delete_on_termination = true + eks_managed_node_groups = merge( + { + comet = { + name = var.eks_mng_name + instance_types = var.eks_node_types + min_size = var.eks_mng_desired_size + max_size = var.eks_mng_max_size + desired_size = var.eks_mng_desired_size + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = var.eks_mng_disk_size + volume_type = local.volume_type + encrypted = local.volume_encrypted + delete_on_termination = local.volume_delete_on_termination + } } } + labels = { + nodegroup_name = "comet" + } + iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {} } - - iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {} - } - } - + }, + var.enable_mpm_infra ? { + druid = { + name = "druid" + instance_types = [var.eks_druid_instance_type] + min_size = var.eks_druid_node_count + max_size = var.eks_druid_node_count + desired_size = var.eks_druid_node_count + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = var.eks_mng_disk_size + volume_type = local.volume_type + encrypted = local.volume_encrypted + delete_on_termination = local.volume_delete_on_termination + } + } + } + labels = { + nodegroup_name = "druid" + } + iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {} + }, + zookeeper = { + name = "zookeeper" + instance_types = [var.eks_zookeeper_instance_type] + min_size = var.eks_zookeeper_node_count + max_size = var.eks_zookeeper_node_count + desired_size = var.eks_zookeeper_node_count + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = var.eks_mng_disk_size + volume_type = local.volume_type + encrypted = local.volume_encrypted + delete_on_termination = local.volume_delete_on_termination + } + } + } + labels = { + nodegroup_name = "zookeeper" + } + iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {} + }, + airflow = { + name = "airflow" + instance_types = [var.eks_airflow_instance_type] + min_size = var.eks_airflow_node_count + max_size = var.eks_airflow_node_count + desired_size = var.eks_airflow_node_count + block_device_mappings = { + xvda = { + device_name = "/dev/xvda" + ebs = { + volume_size = var.eks_mng_disk_size + volume_type = local.volume_type + encrypted = local.volume_encrypted + delete_on_termination = local.volume_delete_on_termination + } + } + } + labels = { + nodegroup_name = "airflow" + } + iam_role_additional_policies = var.s3_enabled ? { comet_s3_access = var.comet_ec2_s3_iam_policy } : {} + } + } : {} + ) tags = local.tags } + module "irsa-ebs-csi" { source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" version = "4.7.0" diff --git a/modules/comet_eks/variables.tf b/modules/comet_eks/variables.tf index 8bbfe92..06458ed 100644 --- a/modules/comet_eks/variables.tf +++ b/modules/comet_eks/variables.tf @@ -88,4 +88,39 @@ variable "comet_ec2_s3_iam_policy" { description = "Policy with access to S3 to associate with EKS worker nodes" type = string default = null +} + +variable "enable_mpm_infra" { + description = "Sets MNGs to be created for MPM compute" + type = bool +} + +variable "eks_druid_instance_type" { + description = "Instance type for EKS Druid nodes" + type = string +} + +variable "eks_zookeeper_instance_type" { + description = "Instance type for EKS Zookeeper nodes" + type = string +} + +variable "eks_airflow_instance_type" { + description = "Instance type for EKS Airflow nodes" + type = string +} + +variable "eks_druid_node_count" { + description = "Instance count for EKS Druid nodes" + type = number +} + +variable "eks_zookeeper_node_count" { + description = "Instance count for EKS Zookeeper nodes" + type = number +} + +variable "eks_airflow_node_count" { + description = "Instance count for EKS Airflow nodes" + type = number } \ No newline at end of file diff --git a/modules/comet_s3/main.tf b/modules/comet_s3/main.tf index 37fdfcd..f05ee95 100644 --- a/modules/comet_s3/main.tf +++ b/modules/comet_s3/main.tf @@ -3,6 +3,7 @@ locals { Terraform = "true" Environment = var.environment } + suffix = substr(sha1("${var.environment}"), 0, 8) } resource "aws_s3_bucket" "comet_s3_bucket" { @@ -15,19 +16,52 @@ resource "aws_s3_bucket" "comet_s3_bucket" { }) } +resource "aws_s3_bucket" "comet_druid_bucket" { + count = var.enable_mpm_infra ? 1 : 0 + + bucket = "comet-druid-${local.suffix}" + + force_destroy = var.s3_force_destroy + + tags = merge(local.tags, { + Name = "comet-druid-${local.suffix}" + }) +} + +resource "aws_s3_bucket" "comet_airflow_bucket" { + count = var.enable_mpm_infra ? 1 : 0 + + bucket = "comet-airflow-${local.suffix}" + + force_destroy = var.s3_force_destroy + + tags = merge(local.tags, { + Name = "comet-airflow-${local.suffix}" + }) +} + resource "aws_iam_policy" "comet_s3_iam_policy" { - name = "comet-s3-access-policy" - description = "comet-s3-access-policy" + name = "comet-s3-access-policy-${local.suffix}" + description = "Policy for access to comet S3 buckets" + policy = jsonencode({ - "Version" : "2012-10-17", - "Statement" : [ + Version = "2012-10-17", + Statement = [ { - "Effect" : "Allow", - "Action" : "s3:*", - "Resource" : [ - "arn:aws:s3:::${var.comet_s3_bucket}", - "arn:aws:s3:::${var.comet_s3_bucket}/*" - ] + Effect = "Allow", + Action = "s3:*", + Resource = concat( + [ + aws_s3_bucket.comet_s3_bucket.arn, + "${aws_s3_bucket.comet_s3_bucket.arn}/*" + ], + var.enable_mpm_infra ? [ + aws_s3_bucket.comet_druid_bucket[0].arn, + "${aws_s3_bucket.comet_druid_bucket[0].arn}/*", + aws_s3_bucket.comet_airflow_bucket[0].arn, + "${aws_s3_bucket.comet_airflow_bucket[0].arn}/*" + ] : [] + ) } ] }) diff --git a/modules/comet_s3/outputs.tf b/modules/comet_s3/outputs.tf index 4d89da5..7b338ae 100644 --- a/modules/comet_s3/outputs.tf +++ b/modules/comet_s3/outputs.tf @@ -1,4 +1,4 @@ output "comet_s3_iam_policy_arn" { - description = "ARN of the IAM policy granting access to the provisioned bucket" + description = "ARN of the IAM policy granting access to the provisioned bucket(s)" value = aws_iam_policy.comet_s3_iam_policy.arn } \ No newline at end of file diff --git a/modules/comet_s3/variables.tf b/modules/comet_s3/variables.tf index 7cc2254..f2b5676 100644 --- a/modules/comet_s3/variables.tf +++ b/modules/comet_s3/variables.tf @@ -11,4 +11,9 @@ variable "comet_s3_bucket" { variable "s3_force_destroy" { description = "Option to enable force delete of S3 bucket" type = bool +} + +variable "enable_mpm_infra" { + description = "Sets buckets to be created for MPM Druid/Airflow" + type = bool } \ No newline at end of file diff --git a/modules/comet_vpc/main.tf b/modules/comet_vpc/main.tf index a21ea0b..379b2a7 100644 --- a/modules/comet_vpc/main.tf +++ b/modules/comet_vpc/main.tf @@ -20,7 +20,7 @@ module "vpc" { azs = local.azs public_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k)] - private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 10)] + private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 5, 3 * k + 1)] enable_nat_gateway = true enable_dns_hostnames = true diff --git a/terraform.tfvars b/terraform.tfvars index 8e2dede..e142213 100644 --- a/terraform.tfvars +++ b/terraform.tfvars @@ -22,6 +22,9 @@ enable_rds = false # Create S3 resources for storing Comet objects enable_s3 = false +# Create EKS nodegroups for MPM compute +enable_mpm_infra = false + ################ #### Global #### ################ diff --git a/variables.tf b/variables.tf index 8fd040d..9e6be0b 100644 --- a/variables.tf +++ b/variables.tf @@ -36,6 +36,11 @@ variable "enable_s3" { type = bool } +variable "enable_mpm_infra" { + description = "Sets MNGs to be created for MPM compute" + type = bool +} + ################ #### Global #### ################ @@ -213,6 +218,42 @@ variable "eks_external_dns_r53_zones" { ] } +variable "eks_druid_instance_type" { + description = "Instance type for EKS Druid nodes" + type = string + default = "m6i.4xlarge" +} + +variable "eks_zookeeper_instance_type" { + description = "Instance type for EKS Zookeeper nodes" + type = string + default = "m6i.4xlarge" +} + +variable "eks_airflow_instance_type" { + description = "Instance type for EKS Airflow nodes" + type = string + default = "m6i.4xlarge" +} + +variable "eks_druid_node_count" { + description = "Instance count for EKS Druid nodes" + type = number + default = 6 +} + +variable "eks_zookeeper_node_count" { + description = "Instance count for EKS Zookeeper nodes" + type = number + default = 3 +} + +variable "eks_airflow_node_count" { + description = "Instance count for EKS Airflow nodes" + type = number + default = 3 +} + #### comet_elasticache #### variable "elasticache_allow_from_sg" { description = "Security group from which to allow connections to ElastiCache, to use when provisioning with existing compute" From d59140c5a235348517e7330d7bd5693336430303 Mon Sep 17 00:00:00 2001 From: Martin Burke <66889764+burmek@users.noreply.github.com> Date: Sun, 14 Apr 2024 20:45:27 -0400 Subject: [PATCH 2/2] Fix main formatting --- main.tf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/main.tf b/main.tf index a88c221..1aa94c0 100644 --- a/main.tf +++ b/main.tf @@ -78,12 +78,12 @@ module "comet_eks" { enable_mpm_infra = var.enable_mpm_infra - eks_druid_instance_type = var.eks_druid_instance_type - eks_druid_node_count = var.eks_druid_node_count + eks_druid_instance_type = var.eks_druid_instance_type + eks_druid_node_count = var.eks_druid_node_count eks_zookeeper_instance_type = var.eks_zookeeper_instance_type - eks_zookeeper_node_count = var.eks_zookeeper_node_count - eks_airflow_instance_type = var.eks_airflow_instance_type - eks_airflow_node_count = var.eks_airflow_node_count + eks_zookeeper_node_count = var.eks_zookeeper_node_count + eks_airflow_instance_type = var.eks_airflow_instance_type + eks_airflow_node_count = var.eks_airflow_node_count } module "comet_elasticache" {