DOM-52551 gov-cloud support, adds irsa module (#197)

* DOM-52551 gov-cloud support, adds irsa module

.circleci/config.yml

@@ -6,7 +6,7 @@ parameters:
     default: "v3.11.2"
   terraform_version:
     type: string
-    default: "1.4.4"
+    default: "1.6.6"
   hcledit_version:
     type: string
     default: "0.2.9"

.pre-commit-config.yaml

@@ -19,7 +19,7 @@ repos:
       - id: check-dependabot
       - id: check-github-actions
   - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.84.0
+    rev: v1.86.0
     hooks:
       - id: terraform_validate
         # See #4 on https://github.com/antonbabenko/pre-commit-terraform#terraform_validate

.tflint.hcl

@@ -1,6 +1,6 @@
 plugin "aws" {
     enabled = true
     deep_check = false
-    version = "0.21.1"
+    version = "0.29.0"
     source  = "github.com/terraform-linters/tflint-ruleset-aws"
 }

examples/deploy/meta.sh

@@ -9,6 +9,15 @@ declare -a MOD_DIRS=(
   "${BASE_TF_DIR}/nodes"
 )
 
+declare -A COMP_MODS
+COMP_MODS["infra"]="infra"
+COMP_MODS["cluster"]="eks irsa_external_dns irsa_policies"
+COMP_MODS["nodes"]="nodes"
+
+declare -A MOD_ADD
+MOD_ADD["irsa_external_dns"]="irsa"
+MOD_ADD["irsa_policies"]="irsa"
+
 INFRA_DIR="${MOD_DIRS[0]}"
 CLUSTER_DIR="${MOD_DIRS[1]}"
 NODES_DIR="${MOD_DIRS[2]}"
@@ -23,6 +32,8 @@ INFRA_VARS="${BASE_TF_DIR}/infra.tfvars"
 
 export BASE_TF_DIR \
   MOD_DIRS \
+  COMP_MODS \
+  MOD_ADD \
   INFRA_DIR \
   CLUSTER_DIR \
   NODES_DIR \

examples/deploy/set-mod-version.sh

@@ -40,13 +40,15 @@ set_module_version() {
   for dir in "${MOD_DIRS[@]}"; do
     file="${dir}/main.tf"
     echo "Setting module source on: $file"
-    name=$(basename "$dir")
-    if [ $name == "cluster" ]; then
-      name="eks"
-    fi
-    hcledit attribute set "module.${name}.source" \"github.com/dominodatalab/terraform-aws-eks.git//modules/"${name}"?ref="${MOD_VERSION}"\" -f "$file" --update
-  done
 
+    IFS=' ' read -ra MODS <<<"${COMP_MODS[$(basename "$dir")]}"
+    for mod in "${MODS[@]}"; do
+      mod_add=${MOD_ADD[$mod]-"$mod"}
+      MOD_SOURCE=github.com/dominodatalab/terraform-aws-eks.git//modules/"${mod_add}"?ref="${MOD_VERSION}"
+      echo "Setting module source to ref: ${MOD_SOURCE} on ${dir}"
+      hcledit attribute set "module.${mod}.source" "\"${MOD_SOURCE}\"" -u -f "$file"
+    done
+  done
 }
 
 MOD_VERSION="$1"

examples/deploy/terraform/cluster/README.md

@@ -12,31 +12,40 @@
 
 | Name | Version |
 |------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 4.0 |
+| <a name="provider_aws.global"></a> [aws.global](#provider\_aws.global) | >= 4.0 |
 | <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_eks"></a> [eks](#module\_eks) | ./../../../../modules/eks | n/a |
+| <a name="module_irsa_external_dns"></a> [irsa\_external\_dns](#module\_irsa\_external\_dns) | ./../../../../modules/irsa | n/a |
+| <a name="module_irsa_policies"></a> [irsa\_policies](#module\_irsa\_policies) | ./../../../../modules/irsa | n/a |
 
 ## Resources
 
 | Name | Type |
 |------|------|
+| [aws_caller_identity.global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
+| [aws_caller_identity.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source |
 | [terraform_remote_state.infra](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_eks"></a> [eks](#input\_eks) | service\_ipv4\_cidr = CIDR for EKS cluster kubernetes\_network\_config.<br>    creation\_role\_name = Name of the role to import.<br>    k8s\_version = EKS cluster k8s version.<br>    kubeconfig = {<br>      extra\_args = Optional extra args when generating kubeconfig.<br>      path       = Fully qualified path name to write the kubeconfig file.<br>    }<br>    public\_access = {<br>      enabled = Enable EKS API public endpoint.<br>      cidrs   = List of CIDR ranges permitted for accessing the EKS public endpoint.<br>    }<br>    Custom role maps for aws auth configmap<br>    custom\_role\_maps = {<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    }<br>    master\_role\_names  = IAM role names to be added as masters in eks.<br>    cluster\_addons     = EKS cluster addons. vpc-cni is installed separately.<br>    vpc\_cni            = Configuration for AWS VPC CNI<br>    ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.<br>    identity\_providers = Configuration for IDP(Identity Provider).<br>  } | <pre>object({<br>    service_ipv4_cidr  = optional(string)<br>    creation_role_name = optional(string, null)<br>    k8s_version        = optional(string)<br>    kubeconfig = optional(object({<br>      extra_args = optional(string)<br>      path       = optional(string)<br>    }), {})<br>    public_access = optional(object({<br>      enabled = optional(bool)<br>      cidrs   = optional(list(string))<br>    }), {})<br>    custom_role_maps = optional(list(object({<br>      rolearn  = string<br>      username = string<br>      groups   = list(string)<br>    })))<br>    master_role_names  = optional(list(string))<br>    cluster_addons     = optional(list(string))<br>    ssm_log_group_name = optional(string)<br>    vpc_cni = optional(object({<br>      prefix_delegation = optional(bool)<br>      annotate_pod_ip   = optional(bool)<br>    }))<br>    identity_providers = optional(list(object({<br>      client_id                     = string<br>      groups_claim                  = optional(string)<br>      groups_prefix                 = optional(string)<br>      identity_provider_config_name = string<br>      issuer_url                    = optional(string)<br>      required_claims               = optional(string)<br>      username_claim                = optional(string)<br>      username_prefix               = optional(string)<br>    })))<br>  })</pre> | `{}` | no |
+| <a name="input_irsa_external_dns"></a> [irsa\_external\_dns](#input\_irsa\_external\_dns) | Mappings for custom IRSA configurations. | <pre>object({<br>    enabled             = optional(bool, false)<br>    hosted_zone_name    = optional(string, null)<br>    namespace           = optional(string, null)<br>    serviceaccount_name = optional(string, null)<br>  })</pre> | `{}` | no |
+| <a name="input_irsa_policies"></a> [irsa\_policies](#input\_irsa\_policies) | Mappings for custom IRSA configurations. | <pre>list(object({<br>    name                = string<br>    namespace           = string<br>    serviceaccount_name = string<br>    policy              = string #json<br>  }))</pre> | `[]` | no |
 | <a name="input_kms_info"></a> [kms\_info](#input\_kms\_info) | Overrides the KMS key information. Meant for migrated configurations.<br>    {<br>      key\_id  = KMS key id.<br>      key\_arn = KMS key arn.<br>      enabled = KMS key is enabled.<br>    } | <pre>object({<br>    key_id  = string<br>    key_arn = string<br>    enabled = bool<br>  })</pre> | `null` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | <a name="output_eks"></a> [eks](#output\_eks) | EKS details. |
+| <a name="output_external_dns_irsa_role_arn"></a> [external\_dns\_irsa\_role\_arn](#output\_external\_dns\_irsa\_role\_arn) | "External\_dns info"<br>  {<br>    irsa\_role = irsa role arn.<br>    zone\_id   = hosted zone id for external\_dns Iam policy<br>    zone\_name = hosted zone name for external\_dns Iam policy<br>  } |
 | <a name="output_infra"></a> [infra](#output\_infra) | Infra details. |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/deploy/terraform/cluster/main.tf

@@ -25,13 +25,54 @@ module "eks" {
   bastion_info        = local.infra.bastion
   create_eks_role_arn = local.infra.create_eks_role_arn
   tags                = local.infra.tags
+}
+data "aws_caller_identity" "global" {
+  provider = aws.global
+}
+
+data "aws_caller_identity" "this" {}
+
+locals {
+  # Determine if the EKS cluster is in the same account as the hosted zone
+  is_eks_account_same = data.aws_caller_identity.this.account_id == data.aws_caller_identity.global.account_id
+}
 
+# If you are enabling the IRSA configuration for external-dns.
+# You will need to add the role created(module.irsa_external_dns.irsa_role) to
+# the following annotation to the `external-dns` service account:
+# `eks.amazonaws.com/role-arn: <<module.irsa_external_dns.irsa_role>>`
+module "irsa_external_dns" {
+  count               = var.irsa_external_dns != null && var.irsa_external_dns.enabled ? 1 : 0
+  source              = "./../../../../modules/irsa"
+  use_cluster_odc_idp = local.is_eks_account_same
+  eks_info            = module.eks.info
+  external_dns        = var.irsa_external_dns
+
+  providers = {
+    aws = aws.global
+  }
+}
+
+module "irsa_policies" {
+  count                   = var.irsa_policies != null ? 1 : 0
+  source                  = "./../../../../modules/irsa"
+  use_cluster_odc_idp     = true
+  eks_info                = module.eks.info
+  additional_irsa_configs = var.irsa_policies
 }
 
+# Provider configuration for the account where the hosted zone is defined.
+# Useful in configurations where accounts do not have a public hosted zone(i.e us-gov regions) and internet routing(public DNS)
+# is instead defined in a different account. Configure the `global` aws alias accordingly,
+# by specifying the profile belonging to the account pertaining to the hosted zone.
 provider "aws" {
-  region = var.infra.region
+  alias = "global"
+  # profile = << profile with credentials to account where the hosted zone resides>>
 }
 
+provider "aws" {
+  region = local.infra.region
+}
 terraform {
   required_version = ">= 1.4.0"
   required_providers {

examples/deploy/terraform/cluster/outputs.tf

@@ -7,3 +7,15 @@ output "eks" {
   description = "EKS details."
   value       = module.eks.info
 }
+
+output "external_dns_irsa_role_arn" {
+  description = <<EOF
+  "External_dns info"
+  {
+    irsa_role = irsa role arn.
+    zone_id   = hosted zone id for external_dns Iam policy
+    zone_name = hosted zone name for external_dns Iam policy
+  }
+  EOF
+  value       = module.irsa_external_dns
+}

examples/deploy/terraform/cluster/variables.tf

@@ -82,3 +82,27 @@ variable "kms_info" {
   })
   default = null
 }
+
+variable "irsa_policies" {
+  description = "Mappings for custom IRSA configurations."
+  type = list(object({
+    name                = string
+    namespace           = string
+    serviceaccount_name = string
+    policy              = string #json
+  }))
+
+  default = []
+}
+
+variable "irsa_external_dns" {
+  description = "Mappings for custom IRSA configurations."
+  type = object({
+    enabled             = optional(bool, false)
+    hosted_zone_name    = optional(string, null)
+    namespace           = optional(string, null)
+    serviceaccount_name = optional(string, null)
+  })
+
+  default = {}
+}

examples/deploy/terraform/infra/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.4.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.6.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.0 |
 
 ## Providers

examples/deploy/terraform/infra/main.tf

@@ -23,7 +23,7 @@ provider "aws" {
 }
 
 terraform {
-  required_version = ">= 1.4.0"
+  required_version = ">= 1.6.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"

examples/tfvars/minimal-wo-bastion.tfvars

@@ -18,3 +18,5 @@ default_node_groups = {
     enabled = false
   }
 }
+
+route53_hosted_zone_name = "deploys-delta.domino.tech"

examples/tfvars/single-node.tfvars

@@ -39,3 +39,5 @@ storage = {
   },
   costs_enabled = false
 }
+
+route53_hosted_zone_name = "deploys-delta.domino.tech"

modules/eks/main.tf

@@ -227,6 +227,11 @@ locals {
       oidc = {
         arn = aws_iam_openid_connect_provider.oidc_provider.arn
         url = aws_iam_openid_connect_provider.oidc_provider.url
+        cert = {
+          thumbprint_list = data.tls_certificate.cluster_tls_certificate.certificates[*].sha1_fingerprint
+          url             = data.tls_certificate.cluster_tls_certificate.url
+
+        }
       }
     }
     nodes = {

modules/infra/README.md

@@ -5,7 +5,7 @@
 
 | Name | Version |
 |------|---------|
-| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.4.0 |
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.6.0 |
 | <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5 |
 | <a name="requirement_local"></a> [local](#requirement\_local) | >= 2.2.0 |
 | <a name="requirement_time"></a> [time](#requirement\_time) | >= 0.9.1 |

modules/infra/main.tf

@@ -12,7 +12,9 @@ locals {
 }
 
 module "cost_usage_report" {
-  count        = var.domino_cur.provision_cost_usage_report ? 1 : 0
+  #https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cur_report_definition.html
+  # is only available in us-east-1
+  count        = !strcontains(var.region, "us-gov") && var.domino_cur.provision_cost_usage_report ? 1 : 0
   source       = "./submodules/cost-usage-report"
   deploy_id    = var.deploy_id
   network_info = module.network.info
@@ -101,7 +103,7 @@ locals {
 }
 
 provider "aws" {
-  region = "us-east-1"
+  region = strcontains(var.region, "us-gov") ? "us-gov-east-1" : "us-east-1"
   alias  = "us-east-1"
   default_tags {
     tags = var.tags

modules/infra/versions.tf

@@ -1,5 +1,5 @@
 terraform {
-  required_version = ">= 1.4.0"
+  required_version = ">= 1.6.0"
   required_providers {
     aws = {
       source  = "hashicorp/aws"

modules/irsa/README.md

@@ -0,0 +1,56 @@
+# IRSA
+
+This module is an opinionated implementation of predefined and custom `irsa` roles for EKS.
+
+## Predefined IRSA roles
+
+* `external-dns`
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.6.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | ~> 5.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | ~> 5.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_iam_openid_connect_provider.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource |
+| [aws_iam_policy.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_role.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
+| [aws_iam_role_policy_attachment.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_policy_document.external_dns](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_additional_irsa_configs"></a> [additional\_irsa\_configs](#input\_additional\_irsa\_configs) | Input for additional irsa configurations | <pre>list(object({<br>    name                = string<br>    namespace           = string<br>    serviceaccount_name = string<br>    policy              = string #json<br>  }))</pre> | `[]` | no |
+| <a name="input_eks_info"></a> [eks\_info](#input\_eks\_info) | cluster = {<br>      specs {<br>        name            = Cluster name.<br>      }<br>      oidc = {<br>        arn = OIDC provider ARN.<br>        url = OIDC provider url.<br>        cert = {<br>          thumbprint\_list = OIDC cert thumbprints.<br>          url             = OIDC cert URL.<br>      }<br>    } | <pre>object({<br>    cluster = object({<br>      specs = object({<br>        name = string<br>      })<br>      oidc = object({<br>        arn = string<br>        url = string<br>        cert = object({<br>          thumbprint_list = list(string)<br>          url             = string<br>        })<br>      })<br>    })<br>  })</pre> | n/a | yes |
+| <a name="input_external_dns"></a> [external\_dns](#input\_external\_dns) | Config to enable irsa for external-dns | <pre>object({<br>    enabled             = optional(bool, false)<br>    hosted_zone_name    = optional(string, null)<br>    hosted_zone_private = optional(string, false)<br>    namespace           = optional(string, "domino-platform")<br>    serviceaccount_name = optional(string, "external-dns")<br>  })</pre> | `{}` | no |
+| <a name="input_use_cluster_odc_idp"></a> [use\_cluster\_odc\_idp](#input\_use\_cluster\_odc\_idp) | Toogle to uset the oidc idp connector in the trust policy.<br>    Set to `true` if the cluster and the hosted zone are in different aws accounts. | `bool` | `true` | no |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_external_dns"></a> [external\_dns](#output\_external\_dns) | External\_dns info |
+| <a name="output_roles"></a> [roles](#output\_roles) | Roles mapping info |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

modules/irsa/additional-irsa-configs.tf

@@ -0,0 +1,35 @@
+resource "aws_iam_role" "this" {
+  for_each = { for irsa in var.additional_irsa_configs : irsa.name => irsa }
+
+  name = "${local.name_prefix}-${each.value.name}"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Principal = {
+          Federated = local.oidc_provider_arn
+        }
+        Action = "sts:AssumeRoleWithWebIdentity"
+        Condition : {
+          StringEquals : {
+            "${trimprefix(local.oidc_provider_url, "https://")}:sub" : "system:serviceaccount:${each.value.namespace}:${each.value.serviceaccount_name}"
+          }
+        }
+      }
+    ]
+  })
+}
+
+resource "aws_iam_policy" "this" {
+  for_each = { for irsa in var.additional_irsa_configs : irsa.name => irsa }
+  name     = "${local.name_prefix}-${each.value.name}"
+  path     = "/"
+  policy   = each.value.policy
+}
+
+resource "aws_iam_role_policy_attachment" "this" {
+  for_each   = { for irsa in var.additional_irsa_configs : irsa.name => irsa }
+  role       = aws_iam_role.this[each.key].name
+  policy_arn = aws_iam_policy.this[each.key].arn
+}