diff --git a/README.md b/README.md index 5783a513..648bdb88 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,6 @@ aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force |------|--------|---------| | [bastion](#module\_bastion) | ./submodules/bastion | n/a | | [eks](#module\_eks) | ./submodules/eks | n/a | -| [k8s\_setup](#module\_k8s\_setup) | ./submodules/k8s | n/a | | [network](#module\_network) | ./submodules/network | n/a | | [storage](#module\_storage) | ./submodules/storage | n/a | @@ -99,8 +98,8 @@ aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force | [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source | | [aws_ec2_instance_type_offerings.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | | [aws_iam_policy_document.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_role.eks_master_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source | +| [aws_subnet.pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | | [aws_subnet.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | | [aws_subnet.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | | [tls_public_key.domino](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/public_key) | data source | @@ -120,6 +119,9 @@ aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force | [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | `"1.24"` | no | | [kubeconfig\_path](#input\_kubeconfig\_path) | fully qualified path name to write the kubeconfig file | `string` | `""` | no | | [number\_of\_azs](#input\_number\_of\_azs) | Number of AZ to distribute the deployment, EKS needs at least 2. | `number` | `3` | no | +| [pod\_cidr](#input\_pod\_cidr) | The IPv4 CIDR block for the VPC. | `string` | `"100.64.0.0/16"` | no | +| [pod\_cidr\_network\_bits](#input\_pod\_cidr\_network\_bits) | Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. | `number` | `19` | no | +| [pod\_subnets](#input\_pod\_subnets) | Optional list of pod subnet ids | `list(string)` | `null` | no | | [private\_cidr\_network\_bits](#input\_private\_cidr\_network\_bits) | Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. | `number` | `19` | no | | [private\_subnets](#input\_private\_subnets) | Optional list of private subnet ids | `list(string)` | `null` | no | | [public\_cidr\_network\_bits](#input\_public\_cidr\_network\_bits) | Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs. | `number` | `27` | no | @@ -129,6 +131,8 @@ aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force | [s3\_force\_destroy\_on\_deletion](#input\_s3\_force\_destroy\_on\_deletion) | Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets | `bool` | `false` | no | | [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | | [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | +| [update\_kubeconfig\_extra\_args](#input\_update\_kubeconfig\_extra\_args) | Optional extra args when generating kubeconfig | `string` | `""` | no | +| [use\_pod\_cidr](#input\_use\_pod\_cidr) | Use additional pod CIDR range (ie 100.64.0.0/16) for pod/service networking | `bool` | `true` | no | | [vpc\_id](#input\_vpc\_id) | Optional VPC ID, it will bypass creation of such. public\_subnets and private\_subnets are also required. | `string` | `null` | no | ## Outputs diff --git a/main.tf b/main.tf index eadaa158..ab0ae549 100644 --- a/main.tf +++ b/main.tf @@ -39,6 +39,11 @@ data "aws_subnet" "private" { id = var.private_subnets[count.index] } +data "aws_subnet" "pod" { + count = var.vpc_id != null ? length(var.pod_subnets) : 0 + id = var.pod_subnets[count.index] +} + locals { # Get zones where ALL instance types are offered(intersection). zone_intersection_instance_offerings = setintersection([for k, v in data.aws_ec2_instance_type_offerings.nodes : toset(v.locations)]...) @@ -89,6 +94,12 @@ locals { public_cidr_blocks = slice(local.subnet_cidr_blocks, 0, local.num_of_azs) ## Match the private subnet var to the list of cidr blocks private_cidr_blocks = slice(local.subnet_cidr_blocks, local.num_of_azs, length(local.subnet_cidr_blocks)) + ## Determine cidr blocks for pod network + base_pod_cidr_network_bits = tonumber(regex("[^/]*$", var.pod_cidr)) + pod_cidr_blocks = !var.use_pod_cidr ? [] : cidrsubnets( + var.pod_cidr, + [for n in range(0, local.num_of_azs) : var.pod_cidr_network_bits - local.base_pod_cidr_network_bits]... + ) } module "network" { @@ -98,9 +109,12 @@ module "network" { deploy_id = var.deploy_id region = var.region cidr = var.cidr + pod_cidr = var.pod_cidr + use_pod_cidr = var.use_pod_cidr availability_zones = local.azs_to_use public_cidrs = local.public_cidr_blocks private_cidrs = local.private_cidr_blocks + pod_cidrs = local.pod_cidr_blocks flow_log_bucket_arn = { arn = module.storage.s3_buckets["monitoring"].arn } } @@ -108,6 +122,7 @@ locals { vpc_id = var.vpc_id != null ? var.vpc_id : module.network[0].vpc_id public_subnets = var.vpc_id != null ? [for s in data.aws_subnet.public : { subnet_id = s.id, az = s.availability_zone }] : module.network[0].public_subnets private_subnets = var.vpc_id != null ? [for s in data.aws_subnet.private : { subnet_id = s.id, az = s.availability_zone }] : module.network[0].private_subnets + pod_subnets = var.vpc_id != null ? [for s in data.aws_subnet.pod : { subnet_id = s.id, az = s.availability_zone }] : module.network[0].pod_subnets } module "bastion" { @@ -130,7 +145,8 @@ module "eks" { k8s_version = var.k8s_version vpc_id = local.vpc_id private_subnets = local.private_subnets - ssh_pvt_key_path = aws_key_pair.domino.key_name + pod_subnets = local.pod_subnets + ssh_key_pair_name = aws_key_pair.domino.key_name bastion_security_group_id = try(module.bastion[0].security_group_id, "") create_bastion_sg = var.bastion != null kubeconfig_path = local.kubeconfig_path @@ -139,28 +155,12 @@ module "eks" { node_iam_policies = [module.storage.s3_policy] efs_security_group = module.storage.efs_security_group update_kubeconfig_extra_args = var.update_kubeconfig_extra_args + eks_master_role_names = var.eks_master_role_names + ssh_pvt_key_path = local.ssh_pvt_key_path + bastion_user = local.bastion_user + bastion_public_ip = try(module.bastion[0].public_ip, "") depends_on = [ module.network ] } - -data "aws_iam_role" "eks_master_roles" { - for_each = var.bastion != null ? toset(var.eks_master_role_names) : [] - name = each.key -} - -module "k8s_setup" { - count = var.bastion != null ? 1 : 0 - source = "./submodules/k8s" - ssh_pvt_key_path = local.ssh_pvt_key_path - bastion_user = local.bastion_user - bastion_public_ip = try(module.bastion[0].public_ip, "") - eks_node_role_arns = [for r in module.eks.eks_node_roles : r.arn] - eks_master_role_arns = [for r in concat(values(data.aws_iam_role.eks_master_roles), module.eks.eks_master_roles) : r.arn] - kubeconfig_path = local.kubeconfig_path - depends_on = [ - module.eks, - module.bastion - ] -} diff --git a/submodules/bastion/main.tf b/submodules/bastion/main.tf index 6d33d95b..b284ef4f 100644 --- a/submodules/bastion/main.tf +++ b/submodules/bastion/main.tf @@ -14,7 +14,7 @@ resource "aws_security_group" "bastion" { lifecycle { create_before_destroy = true - ignore_changes = [description] + ignore_changes = [description] } tags = { diff --git a/submodules/eks/README.md b/submodules/eks/README.md index 17830440..e5fad6ac 100644 --- a/submodules/eks/README.md +++ b/submodules/eks/README.md @@ -18,7 +18,9 @@ ## Modules -No modules. +| Name | Source | Version | +|------|--------|---------| +| [k8s\_setup](#module\_k8s\_setup) | ../k8s | n/a | ## Resources @@ -27,6 +29,7 @@ No modules. | [aws_autoscaling_group_tag.tag](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_cloudwatch_log_group.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | | [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | +| [aws_eks_addon.vpc_cni](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_node_group.node_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_policy.custom_eks_node_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | @@ -54,6 +57,7 @@ No modules. | [aws_iam_policy_document.eks_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.kms_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.snapshot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_role.eks_master_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | ## Inputs @@ -61,18 +65,24 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. |
map(object({
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = list(string)
spot = optional(bool, false)
min_per_az = number
max_per_az = number
desired_per_az = number
labels = map(string)
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = object({
size = string
type = string
})
}))
| `{}` | no | +| [bastion\_public\_ip](#input\_bastion\_public\_ip) | Public IP of bastion instance | `string` | `""` | no | | [bastion\_security\_group\_id](#input\_bastion\_security\_group\_id) | Bastion security group id. | `string` | `""` | no | -| [create\_bastion\_sg](#input\_create\_bastion\_sg) | Create bastion access rules toggle. | `bool` | n/a | yes | +| [bastion\_user](#input\_bastion\_user) | Username for bastion instance | `string` | `""` | no | +| [create\_bastion\_sg](#input\_create\_bastion\_sg) | Create bastion access rules toggle. | `bool` | `false` | no | | [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. |
object(
{
compute = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.2xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
platform = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["m5.4xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 1)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "platform"
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
}),
gpu = object(
{
ami = optional(string)
bootstrap_extra_args = optional(string, "")
instance_types = optional(list(string), ["g4dn.xlarge"])
spot = optional(bool, false)
min_per_az = optional(number, 0)
max_per_az = optional(number, 10)
desired_per_az = optional(number, 0)
labels = optional(map(string), {
"dominodatalab.com/node-pool" = "default-gpu"
"nvidia.com/gpu" = true
})
taints = optional(list(object({ key = string, value = optional(string), effect = string })), [])
tags = optional(map(string), {})
volume = optional(object(
{
size = optional(number, 100)
type = optional(string, "gp3")
}),
{
size = 100
type = "gp3"
}
)
})
})
|
{
"compute": {},
"gpu": {},
"platform": {}
}
| no | | [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes | | [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | n/a | yes | -| [eks\_cluster\_addons](#input\_eks\_cluster\_addons) | EKS cluster addons. | `list(string)` |
[
"vpc-cni",
"kube-proxy",
"coredns"
]
| no | +| [eks\_cluster\_addons](#input\_eks\_cluster\_addons) | EKS cluster addons. vpc-cni is installed separately. | `list(string)` |
[
"kube-proxy",
"coredns"
]
| no | +| [eks\_master\_role\_names](#input\_eks\_master\_role\_names) | IAM role names to be added as masters in eks | `list(string)` | `[]` | no | | [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | n/a | yes | | [kubeconfig\_path](#input\_kubeconfig\_path) | Kubeconfig file path. | `string` | `"kubeconfig"` | no | | [node\_iam\_policies](#input\_node\_iam\_policies) | Additional IAM Policy Arns for Nodes | `list(string)` | n/a | yes | +| [pod\_subnets](#input\_pod\_subnets) | List of POD subnets IDs and AZ | `list(object({ subnet_id = string, az = string }))` | n/a | yes | | [private\_subnets](#input\_private\_subnets) | List of Private subnets IDs and AZ | `list(object({ subnet_id = string, az = string }))` | n/a | yes | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | -| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | +| [ssh\_key\_pair\_name](#input\_ssh\_key\_pair\_name) | SSH key pair name. | `string` | n/a | yes | +| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | Path to SSH private key | `string` | `""` | no | +| [update\_kubeconfig\_extra\_args](#input\_update\_kubeconfig\_extra\_args) | Optional extra args when generating kubeconfig | `string` | `""` | no | | [vpc\_id](#input\_vpc\_id) | VPC ID. | `string` | n/a | yes | ## Outputs diff --git a/submodules/eks/cluster.tf b/submodules/eks/cluster.tf index 8c2471fd..fb59c734 100755 --- a/submodules/eks/cluster.tf +++ b/submodules/eks/cluster.tf @@ -46,7 +46,7 @@ resource "aws_security_group" "eks_cluster" { lifecycle { create_before_destroy = true - ignore_changes = [description, name] + ignore_changes = [description, name] } tags = { "Name" = "${local.eks_cluster_name}-eks-cluster" @@ -105,6 +105,12 @@ resource "aws_eks_cluster" "this" { ] } +resource "aws_eks_addon" "vpc_cni" { + cluster_name = aws_eks_cluster.this.name + resolve_conflicts = "OVERWRITE" + addon_name = "vpc-cni" +} + resource "aws_eks_addon" "this" { for_each = toset(var.eks_cluster_addons) cluster_name = aws_eks_cluster.this.name diff --git a/submodules/eks/iam.tf b/submodules/eks/iam.tf index 016e64b9..2095861a 100644 --- a/submodules/eks/iam.tf +++ b/submodules/eks/iam.tf @@ -15,7 +15,7 @@ resource "aws_iam_role" "eks_cluster" { name = "${var.deploy_id}-eks" assume_role_policy = data.aws_iam_policy_document.eks_cluster.json lifecycle { - ignore_changes = [name] + ignore_changes = [name] } } diff --git a/submodules/eks/k8s.tf b/submodules/eks/k8s.tf new file mode 100644 index 00000000..b0826641 --- /dev/null +++ b/submodules/eks/k8s.tf @@ -0,0 +1,20 @@ +data "aws_iam_role" "eks_master_roles" { + for_each = var.create_bastion_sg ? toset(var.eks_master_role_names) : [] + name = each.key +} + +module "k8s_setup" { + count = var.create_bastion_sg ? 1 : 0 + source = "../k8s" + ssh_pvt_key_path = var.ssh_pvt_key_path + bastion_user = var.bastion_user + bastion_public_ip = try(var.bastion_public_ip, "") + eks_node_role_arns = [aws_iam_role.eks_nodes.arn] + eks_master_role_arns = [for r in concat(values(data.aws_iam_role.eks_master_roles), [aws_iam_role.eks_cluster]) : r.arn] + kubeconfig_path = var.kubeconfig_path + + security_group_id = aws_security_group.eks_nodes.id + pod_subnets = var.pod_subnets + + depends_on = [aws_eks_addon.vpc_cni, null_resource.kubeconfig] +} diff --git a/submodules/eks/node-group.tf b/submodules/eks/node-group.tf index d2b9b816..10791c94 100644 --- a/submodules/eks/node-group.tf +++ b/submodules/eks/node-group.tf @@ -86,7 +86,7 @@ resource "aws_launch_template" "node_groups" { for_each = local.node_groups name = "${local.eks_cluster_name}-${each.key}" disable_api_termination = false - key_name = var.ssh_pvt_key_path + key_name = var.ssh_key_pair_name user_data = each.value.ami == null ? null : base64encode(templatefile( "${path.module}/templates/linux_user_data.tpl", { @@ -144,6 +144,7 @@ resource "aws_launch_template" "node_groups" { } resource "aws_eks_node_group" "node_groups" { + depends_on = [module.k8s_setup] for_each = local.node_groups_by_name cluster_name = aws_eks_cluster.this.name node_group_name = "${local.eks_cluster_name}-${each.key}" diff --git a/submodules/eks/variables.tf b/submodules/eks/variables.tf index a4e09fd8..3683dc64 100755 --- a/submodules/eks/variables.tf +++ b/submodules/eks/variables.tf @@ -148,14 +148,23 @@ variable "private_subnets" { } } +variable "pod_subnets" { + description = "List of POD subnets IDs and AZ" + type = list(object({ subnet_id = string, az = string })) + validation { + condition = length(var.pod_subnets) != 1 + error_message = "EKS deployment needs at least 2 subnets. https://docs.aws.amazon.com/eks/latest/userguide/network_reqs.html." + } +} + variable "vpc_id" { type = string description = "VPC ID." } -variable "ssh_pvt_key_path" { +variable "ssh_key_pair_name" { type = string - description = "SSH private key filepath." + description = "SSH key pair name." } variable "bastion_security_group_id" { @@ -166,13 +175,14 @@ variable "bastion_security_group_id" { variable "eks_cluster_addons" { type = list(string) - description = "EKS cluster addons." - default = ["vpc-cni", "kube-proxy", "coredns"] + description = "EKS cluster addons. vpc-cni is installed separately." + default = ["kube-proxy", "coredns"] } variable "create_bastion_sg" { description = "Create bastion access rules toggle." type = bool + default = false } variable "node_iam_policies" { @@ -184,3 +194,27 @@ variable "efs_security_group" { description = "Security Group ID for EFS" type = string } + +variable "eks_master_role_names" { + type = list(string) + description = "IAM role names to be added as masters in eks" + default = [] +} + +variable "ssh_pvt_key_path" { + type = string + description = "Path to SSH private key" + default = "" +} + +variable "bastion_user" { + type = string + description = "Username for bastion instance" + default = "" +} + +variable "bastion_public_ip" { + type = string + description = "Public IP of bastion instance" + default = "" +} diff --git a/submodules/k8s/README.md b/submodules/k8s/README.md index 0f5e43c1..6d2049ad 100644 --- a/submodules/k8s/README.md +++ b/submodules/k8s/README.md @@ -39,6 +39,8 @@ No modules. | [eks\_node\_role\_arns](#input\_eks\_node\_role\_arns) | Roles arns for EKS nodes to be added to aws-auth for api auth. | `list(string)` | n/a | yes | | [k8s\_tunnel\_port](#input\_k8s\_tunnel\_port) | K8s ssh tunnel port | `string` | `"1080"` | no | | [kubeconfig\_path](#input\_kubeconfig\_path) | Kubeconfig filename. | `string` | `"kubeconfig"` | no | +| [pod\_subnets](#input\_pod\_subnets) | Pod subnets and az to setup with vpc-cni | `list(object({ subnet_id = string, az = string }))` | n/a | yes | +| [security\_group\_id](#input\_security\_group\_id) | Security group id for eks cluster. | `string` | n/a | yes | | [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | ## Outputs diff --git a/submodules/k8s/main.tf b/submodules/k8s/main.tf index 621a0518..60923c64 100644 --- a/submodules/k8s/main.tf +++ b/submodules/k8s/main.tf @@ -5,6 +5,8 @@ locals { k8s_pre_setup_sh_template = "k8s-pre-setup.sh.tftpl" aws_auth_filename = "aws-auth.yaml" aws_auth_template = "aws-auth.yaml.tftpl" + eniconfig_filename = length(var.pod_subnets) != 0 ? "eniconfig.yaml" : "" + eniconfig_template = "eniconfig.yaml.tftpl" calico = { operator_url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-operator.yaml" custom_resources_url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-crs.yaml" @@ -20,6 +22,7 @@ locals { kubeconfig_path = var.kubeconfig_path k8s_tunnel_port = var.k8s_tunnel_port aws_auth_yaml = basename(local.aws_auth_filename) + eniconfig_yaml = local.eniconfig_filename != "" ? basename(local.eniconfig_filename) : "" calico_operator_url = local.calico.operator_url calico_custom_resources_url = local.calico.custom_resources_url bastion_user = var.bastion_user @@ -44,11 +47,20 @@ locals { }) } + + eni_config = { + filename = local.eniconfig_filename + content = templatefile("${local.templates_dir}/${local.eniconfig_template}", + { + security_group_id = var.security_group_id + subnets = var.pod_subnets + }) + } } } resource "local_file" "templates" { - for_each = { for k, v in local.templates : k => v } + for_each = { for k, v in local.templates : k => v if v.filename != "" } content = each.value.content filename = "${local.resources_directory}/${each.value.filename}" directory_permission = "0777" diff --git a/submodules/k8s/templates/eniconfig.yaml.tftpl b/submodules/k8s/templates/eniconfig.yaml.tftpl new file mode 100644 index 00000000..737b865e --- /dev/null +++ b/submodules/k8s/templates/eniconfig.yaml.tftpl @@ -0,0 +1,11 @@ +%{ for subnet in subnets ~} +--- +apiVersion: crd.k8s.amazonaws.com/v1alpha1 +kind: ENIConfig +metadata: + name: ${subnet.az} +spec: + securityGroups: + - ${security_group_id} + subnet: ${subnet.subnet_id} +%{ endfor ~} diff --git a/submodules/k8s/templates/k8s-functions.sh.tftpl b/submodules/k8s/templates/k8s-functions.sh.tftpl index 036a1828..c188ff01 100644 --- a/submodules/k8s/templates/k8s-functions.sh.tftpl +++ b/submodules/k8s/templates/k8s-functions.sh.tftpl @@ -32,6 +32,21 @@ set_k8s_auth() { echo } +set_eniconfig() { + local ENICONFIG_YAML="${eniconfig_yaml}" + if [ -z "$ENICONFIG_YAML" ]; then + return + fi + if test -f "$ENICONFIG_YAML"; then + printf "$GREEN Updating $ENICONFIG_YAML... $EC \n" + kubectl_apply "$ENICONFIG_YAML" + else + printf "$RED $ENICONFIG_YAML does not exist. $EC \n" && exit 1 + fi + echo + kubectl_cmd -n kube-system set env daemonset aws-node AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG=true ENI_CONFIG_LABEL_DEF=topology.kubernetes.io/zone +} + install_calico() { local CALICO_OPERATOR_YAML_URL=${calico_operator_url} printf "$GREEN Installing Calico Operator $EC \n" @@ -68,6 +83,15 @@ kubectl_apply() { fi } +kubectl_cmd() { + echo "Running kubectl $@..." + HTTPS_PROXY=socks5://127.0.0.1:${k8s_tunnel_port} kubectl --kubeconfig "${kubeconfig_path}" $@ + if [ $? -ne 0 ]; then + printf "$RED Error running kubectl $@ \n" + exit 1 + fi +} + close_ssh_tunnel_to_k8s_api() { printf "$GREEN Shutting down k8s tunnel ... $EC" ssh -S $TUNNEL_SOCKET_FILE -O exit ${bastion_user}@${bastion_public_ip} diff --git a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl b/submodules/k8s/templates/k8s-pre-setup.sh.tftpl index b5a5aa8f..6602b084 100644 --- a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl +++ b/submodules/k8s/templates/k8s-pre-setup.sh.tftpl @@ -1,5 +1,7 @@ #! /usr/bin/env bash -ex +set -ex + RED="\e[31m" GREEN="\e[32m" EC="\e[0m" @@ -10,6 +12,7 @@ main() { open_ssh_tunnel_to_k8s_api check_kubeconfig set_k8s_auth + set_eniconfig install_calico } diff --git a/submodules/k8s/variables.tf b/submodules/k8s/variables.tf index cafeb7ce..e3b31962 100755 --- a/submodules/k8s/variables.tf +++ b/submodules/k8s/variables.tf @@ -44,3 +44,13 @@ variable "calico_version" { description = "Calico operator version." default = "v1.11.0" } + +variable "security_group_id" { + type = string + description = "Security group id for eks cluster." +} + +variable "pod_subnets" { + type = list(object({ subnet_id = string, az = string })) + description = "Pod subnets and az to setup with vpc-cni" +} diff --git a/submodules/network/README.md b/submodules/network/README.md index 4fca953c..b9c8882d 100644 --- a/submodules/network/README.md +++ b/submodules/network/README.md @@ -28,14 +28,18 @@ No modules. | [aws_flow_log.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/flow_log) | resource | | [aws_internet_gateway.igw](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/internet_gateway) | resource | | [aws_nat_gateway.ngw](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/nat_gateway) | resource | +| [aws_route_table.pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table) | resource | | [aws_route_table.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table) | resource | | [aws_route_table.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table) | resource | +| [aws_route_table_association.pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | | [aws_route_table_association.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | | [aws_route_table_association.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource | +| [aws_subnet.pod](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_subnet.private](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_subnet.public](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource | | [aws_vpc.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc) | resource | | [aws_vpc_endpoint.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint) | resource | +| [aws_vpc_ipv4_cidr_block_association.pod_cidr](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_ipv4_cidr_block_association) | resource | | [aws_network_acls.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/network_acls) | data source | ## Inputs @@ -47,14 +51,18 @@ No modules. | [cidr](#input\_cidr) | The IPv4 CIDR block for the VPC. | `string` | `"10.0.0.0/16"` | no | | [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | `""` | no | | [flow\_log\_bucket\_arn](#input\_flow\_log\_bucket\_arn) | Bucket for vpc flow logging | `object({ arn = string })` | `null` | no | +| [pod\_cidr](#input\_pod\_cidr) | The IPv4 CIDR block for the VPC. | `string` | `"100.64.0.0/16"` | no | +| [pod\_cidrs](#input\_pod\_cidrs) | list of cidrs for the pod subnets | `list(string)` | n/a | yes | | [private\_cidrs](#input\_private\_cidrs) | list of cidrs for the private subnets | `list(string)` | n/a | yes | | [public\_cidrs](#input\_public\_cidrs) | list of cidrs for the public subnets | `list(string)` | n/a | yes | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | +| [use\_pod\_cidr](#input\_use\_pod\_cidr) | Use additional pod CIDR range (ie 100.64.0.0/16) for pod/service networking | `bool` | `true` | no | ## Outputs | Name | Description | |------|-------------| +| [pod\_subnets](#output\_pod\_subnets) | List of pod subnet ID and AZ | | [private\_subnets](#output\_private\_subnets) | List of private subnet ID and AZ | | [public\_subnets](#output\_public\_subnets) | List of public subnet ID and AZ | | [vpc\_id](#output\_vpc\_id) | VPC id. | diff --git a/submodules/network/main.tf b/submodules/network/main.tf index 76d7ad51..257d4a19 100644 --- a/submodules/network/main.tf +++ b/submodules/network/main.tf @@ -12,6 +12,12 @@ locals { vpc_id = aws_vpc.this.id } +resource "aws_vpc_ipv4_cidr_block_association" "pod_cidr" { + count = var.use_pod_cidr ? 1 : 0 + vpc_id = aws_vpc.this.id + cidr_block = var.pod_cidr +} + resource "aws_default_security_group" "default" { vpc_id = local.vpc_id } @@ -23,7 +29,8 @@ resource "aws_vpc_endpoint" "s3" { route_table_ids = concat( [for s in aws_route_table.public : s.id], - [for s in aws_route_table.private : s.id] + [for s in aws_route_table.private : s.id], + [for s in aws_route_table.pod : s.id] ) tags = { @@ -67,7 +74,8 @@ resource "aws_default_network_acl" "default" { subnet_ids = concat( [for s in aws_subnet.public : s.id], - [for s in aws_subnet.private : s.id] + [for s in aws_subnet.private : s.id], + [for s in aws_subnet.pod : s.id] ) lifecycle { diff --git a/submodules/network/outputs.tf b/submodules/network/outputs.tf index ef7cab79..97e33ca7 100644 --- a/submodules/network/outputs.tf +++ b/submodules/network/outputs.tf @@ -3,12 +3,17 @@ output "vpc_id" { value = local.vpc_id } +output "public_subnets" { + description = "List of public subnet ID and AZ" + value = [for cidr, c in local.public_cidrs : { name = c.name, subnet_id = aws_subnet.public[cidr].id, az = c.az }] +} + output "private_subnets" { description = "List of private subnet ID and AZ" value = [for cidr, c in local.private_cidrs : { name = c.name, subnet_id = aws_subnet.private[cidr].id, az = c.az }] } -output "public_subnets" { - description = "List of public subnet ID and AZ" - value = [for cidr, c in local.public_cidrs : { name = c.name, subnet_id = aws_subnet.public[cidr].id, az = c.az }] +output "pod_subnets" { + description = "List of pod subnet ID and AZ" + value = [for cidr, c in local.pod_cidrs : { name = c.name, subnet_id = aws_subnet.pod[cidr].id, az = c.az }] } diff --git a/submodules/network/route-tables.tf b/submodules/network/route-tables.tf index 4ea0f9e0..93bab0c3 100644 --- a/submodules/network/route-tables.tf +++ b/submodules/network/route-tables.tf @@ -37,3 +37,25 @@ resource "aws_route_table_association" "private" { subnet_id = aws_subnet.private[each.key].id route_table_id = aws_route_table.private[each.key].id } + +locals { + pod_public_map = var.use_pod_cidr ? zipmap(keys(local.pod_cidrs), keys(local.public_cidrs)) : {} +} + +resource "aws_route_table" "pod" { + for_each = local.pod_cidrs + route { + cidr_block = "0.0.0.0/0" + nat_gateway_id = aws_nat_gateway.ngw[local.pod_public_map[each.key]].id + } + vpc_id = local.vpc_id + tags = { + "Name" = each.value.name, + } +} + +resource "aws_route_table_association" "pod" { + for_each = local.pod_cidrs + subnet_id = aws_subnet.pod[each.key].id + route_table_id = aws_route_table.pod[each.key].id +} diff --git a/submodules/network/subnets.tf b/submodules/network/subnets.tf index 8b9e1c9c..8eddceb2 100644 --- a/submodules/network/subnets.tf +++ b/submodules/network/subnets.tf @@ -16,6 +16,14 @@ locals { "name" = "${var.deploy_id}-private-${element(local.az, i)}" } } + + ## Get the pod subnets by matching the mask and populating its params + pod_cidrs = { for i, cidr in var.pod_cidrs : cidr => + { + "az" = element(local.az, i) + "name" = "${var.deploy_id}-pod-${element(local.az, i)}" + } + } } resource "aws_subnet" "public" { @@ -53,3 +61,24 @@ resource "aws_subnet" "private" { ignore_changes = [tags] } } + +resource "aws_subnet" "pod" { + for_each = local.pod_cidrs + + availability_zone = each.value.az + vpc_id = local.vpc_id + cidr_block = each.key + tags = merge( + { "Name" : each.value.name }, + var.add_eks_elb_tags ? { + "kubernetes.io/role/internal-elb" = "1" + "kubernetes.io/cluster/${var.deploy_id}" = "shared" + } : {}) + + lifecycle { + ignore_changes = [tags] + } + + ## See https://github.com/hashicorp/terraform-provider-aws/issues/9592 + depends_on = [aws_vpc_ipv4_cidr_block_association.pod_cidr[0]] +} diff --git a/submodules/network/variables.tf b/submodules/network/variables.tf index b2a8c2dc..90b55c19 100644 --- a/submodules/network/variables.tf +++ b/submodules/network/variables.tf @@ -35,6 +35,11 @@ variable "private_cidrs" { description = "list of cidrs for the private subnets" } +variable "pod_cidrs" { + type = list(string) + description = "list of cidrs for the pod subnets" +} + variable "cidr" { type = string default = "10.0.0.0/16" @@ -48,6 +53,25 @@ variable "cidr" { } } +variable "pod_cidr" { + type = string + default = "100.64.0.0/16" + description = "The IPv4 CIDR block for the VPC." + validation { + condition = ( + try(cidrhost(var.pod_cidr, 0), null) == regex("^(.*)/", var.pod_cidr)[0] && + try(cidrnetmask(var.pod_cidr), null) == "255.255.0.0" + ) + error_message = "Argument cidr must be a valid CIDR block." + } +} + +variable "use_pod_cidr" { + type = bool + description = "Use additional pod CIDR range (ie 100.64.0.0/16) for pod/service networking" + default = true +} + ## This is an object in order to be used as a conditional in count, due to https://github.com/hashicorp/terraform/issues/26755 variable "flow_log_bucket_arn" { type = object({ arn = string }) diff --git a/variables.tf b/variables.tf index 29cba13d..cff06ff7 100755 --- a/variables.tf +++ b/variables.tf @@ -82,6 +82,12 @@ variable "private_cidr_network_bits" { default = 19 } +variable "pod_cidr_network_bits" { + type = number + description = "Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs." + default = 19 +} + variable "default_node_groups" { description = "EKS managed node groups definition." type = object( @@ -206,6 +212,25 @@ variable "cidr" { } } +variable "pod_cidr" { + type = string + default = "100.64.0.0/16" + description = "The IPv4 CIDR block for the VPC." + validation { + condition = ( + try(cidrhost(var.pod_cidr, 0), null) == regex("^(.*)/", var.pod_cidr)[0] && + try(cidrnetmask(var.pod_cidr), null) == "255.255.0.0" + ) + error_message = "Argument base_cidr_block must be a valid CIDR block." + } +} + +variable "use_pod_cidr" { + type = bool + description = "Use additional pod CIDR range (ie 100.64.0.0/16) for pod/service networking" + default = true +} + variable "eks_master_role_names" { type = list(string) description = "IAM role names to be added as masters in eks." @@ -230,6 +255,12 @@ variable "private_subnets" { default = null } +variable "pod_subnets" { + type = list(string) + description = "Optional list of pod subnet ids" + default = null +} + variable "bastion" { type = object({ ami = optional(string, null) # default will use the latest 'amazon_linux_2' ami