From 42d64b49880ca32a14b1f7a48cc6e2f4d2661077 Mon Sep 17 00:00:00 2001 From: miguelhar <98769216+miguelhar@users.noreply.github.com> Date: Tue, 19 Sep 2023 16:07:55 -0400 Subject: [PATCH] Releases v3 (#134) * PLAT-6822: Node pool,cluster, infra isolation (#113) * PLAT-6823: Node pool,cluster, infra segregation * [DOM-49677] Initial commit submodule (#126) * PLAT-7142: CDK to TF migration support. --------- Co-authored-by: Luis De Bello <105391279+ldebello-ddl@users.noreply.github.com> --- .circleci/config.yml | 194 ++++---- .gitignore | 9 + .pre-commit-config.yaml | 10 +- README.md | 428 ++++++++++-------- bin/{ => pre-commit}/check-aws-partition.sh | 0 bin/state-migration/README.md | 80 ++++ bin/state-migration/migrate-states.sh | 158 +++++++ examples/bring-your-vpc/README.md | 74 --- examples/bring-your-vpc/main.tf | 34 -- examples/bring-your-vpc/outputs.tf | 4 - examples/bring-your-vpc/variables.tf | 5 - examples/custom-node-pool/README.md | 71 --- examples/custom-node-pool/main.tf | 40 -- examples/custom-node-pool/outputs.tf | 4 - examples/custom-node-pool/variables.tf | 5 - examples/deploy/README.md | 165 +++++++ examples/deploy/meta.sh | 34 ++ examples/deploy/set-mod-version.sh | 59 +++ examples/deploy/terraform/cluster.tfvars | 2 + examples/deploy/terraform/cluster/README.md | 42 ++ examples/deploy/terraform/cluster/main.tf | 47 ++ examples/deploy/terraform/cluster/outputs.tf | 9 + .../deploy/terraform/cluster/variables.tf | 81 ++++ examples/deploy/terraform/infra.tfvars | 84 ++++ examples/deploy/terraform/infra/README.md | 48 ++ examples/deploy/terraform/infra/main.tf | 32 ++ examples/deploy/terraform/infra/outputs.tf | 62 +++ examples/deploy/terraform/infra/variables.tf | 340 ++++++++++++++ examples/deploy/terraform/nodes.tfvars | 2 + examples/deploy/terraform/nodes/README.md | 41 ++ examples/deploy/terraform/nodes/main.tf | 42 ++ examples/deploy/terraform/nodes/outputs.tf | 4 + examples/deploy/terraform/nodes/variables.tf | 106 +++++ examples/deploy/tf.sh | 238 ++++++++++ examples/examples.pem | 51 --- examples/kms-additional-policy/README.md | 52 --- .../kms-additional-policy/kms-policy.json | 12 - examples/kms-additional-policy/main.tf | 21 - examples/kms-additional-policy/outputs.tf | 4 - examples/kms-additional-policy/variables.tf | 5 - examples/kms-additional-policy/versions.tf | 21 - examples/kms/README.md | 55 --- examples/kms/main.tf | 34 -- examples/kms/outputs.tf | 4 - examples/kms/variables.tf | 5 - examples/kms/versions.tf | 21 - examples/minimal-with-bastion/README.md | 54 --- examples/minimal-with-bastion/main.tf | 22 - examples/minimal-with-bastion/outputs.tf | 4 - examples/minimal-with-bastion/variables.tf | 11 - examples/minimal-with-bastion/versions.tf | 21 - examples/nodes-custom-ami/README.md | 168 ------- examples/nodes-custom-ami/main.tf | 84 ---- examples/nodes-custom-ami/outputs.tf | 4 - examples/nodes-custom-ami/variables.tf | 5 - examples/nodes-custom-ami/versions.tf | 21 - examples/public-access/README.md | 58 --- examples/public-access/main.tf | 28 -- examples/public-access/outputs.tf | 4 - examples/public-access/variables.tf | 5 - examples/public-access/versions.tf | 21 - examples/tf-plan-test.sh | 85 ---- examples/tfvars/bring-your-vpc.tfvars | 31 ++ examples/tfvars/custom-node-pool.tfvars | 37 ++ examples/tfvars/kms-add-policies.tfvars | 34 ++ examples/tfvars/kms-byok.tfvars | 24 + examples/tfvars/minimal-with-bastion.tfvars | 20 + examples/tfvars/minimal-wo-bastion.tfvars | 20 + examples/tfvars/nodes-custom-ami.tfvars | 22 + examples/tfvars/private-link.tfvars | 23 + examples/tfvars/public-access.tfvars | 23 + {submodules => modules}/eks/README.md | 27 +- {submodules => modules}/eks/cluster.tf | 20 +- .../eks/eks-bastion-access.tf | 0 {submodules => modules}/eks/iam.tf | 4 +- modules/eks/k8s.tf | 32 ++ {submodules => modules}/eks/main.tf | 10 +- modules/eks/node-group.tf | 62 +++ modules/eks/outputs.tf | 12 + modules/eks/privatelink.tf | 9 + .../eks/submodules}/k8s/README.md | 2 +- .../eks/submodules}/k8s/main.tf | 6 +- .../eks/submodules}/k8s/outputs.tf | 4 +- .../k8s/templates/aws-auth.yaml.tftpl | 0 .../k8s/templates/eniconfig.yaml.tftpl | 0 .../k8s/templates/k8s-functions.sh.tftpl | 17 +- .../k8s/templates/k8s-pre-setup.sh.tftpl | 0 .../eks/submodules}/k8s/variables.tf | 0 .../eks/submodules}/k8s/versions.tf | 0 modules/eks/submodules/privatelink/README.md | 51 +++ ...-balancer-controller_2.5.4_iam_policy.json | 241 ++++++++++ modules/eks/submodules/privatelink/iam.tf | 51 +++ modules/eks/submodules/privatelink/main.tf | 1 + modules/eks/submodules/privatelink/outputs.tf | 6 + .../eks/submodules/privatelink/variables.tf | 100 ++++ .../eks/submodules/privatelink/versions.tf | 9 + .../privatelink/vpc-endpoint-services.tf | 84 ++++ {submodules => modules}/eks/variables.tf | 125 ++--- {submodules => modules}/eks/versions.tf | 20 +- .../iam-bootstrap}/README.md | 0 .../iam-bootstrap}/bootstrap-0.json | 1 - .../iam-bootstrap}/bootstrap-1.json | 22 + .../iam-bootstrap}/main.tf | 0 .../iam-bootstrap}/outputs.tf | 0 .../iam-bootstrap}/render.sh | 0 .../iam-bootstrap}/variables.tf | 0 .../iam-bootstrap}/versions.tf | 0 modules/infra/README.md | 91 ++++ iam.tf => modules/infra/iam.tf | 8 +- kms.tf => modules/infra/kms.tf | 0 main.tf => modules/infra/main.tf | 37 +- modules/infra/outputs.tf | 82 ++++ .../infra/submodules}/bastion/README.md | 0 .../infra/submodules}/bastion/main.tf | 0 .../infra/submodules}/bastion/outputs.tf | 0 .../templates/install-binaries.sh.tftpl | 0 .../infra/submodules}/bastion/variables.tf | 0 .../infra/submodules}/bastion/versions.tf | 0 .../infra/submodules}/network/README.md | 0 .../submodules}/network/internet-gateway.tf | 0 .../infra/submodules}/network/main.tf | 1 - .../infra/submodules}/network/nat-gateway.tf | 0 .../infra/submodules}/network/outputs.tf | 1 + .../infra/submodules}/network/route-tables.tf | 0 .../infra/submodules}/network/subnets.tf | 0 .../infra/submodules}/network/variables.tf | 0 .../infra/submodules}/network/versions.tf | 0 .../infra/submodules}/network/vpc.tf | 0 .../infra/submodules}/storage/README.md | 0 .../infra/submodules}/storage/ecr.tf | 0 .../infra/submodules}/storage/efs.tf | 0 .../submodules}/storage/efs_backup_vault.tf | 2 + .../infra/submodules}/storage/iam.tf | 0 .../infra/submodules}/storage/main.tf | 0 .../infra/submodules}/storage/outputs.tf | 0 .../infra/submodules}/storage/s3.tf | 1 - .../infra/submodules}/storage/variables.tf | 0 .../infra/submodules}/storage/versions.tf | 0 variables.tf => modules/infra/variables.tf | 44 +- .../infra}/versions.tf | 9 +- modules/nodes/README.md | 59 +++ modules/nodes/main.tf | 91 ++++ .../node-group.tf => modules/nodes/nodes.tf | 139 +----- modules/nodes/outputs.tf | 4 + .../nodes}/templates/linux_user_data.tpl | 0 modules/nodes/variables.tf | 303 +++++++++++++ .../nodes}/versions.tf | 14 +- outputs.tf | 33 -- submodules/eks/k8s.tf | 47 -- submodules/eks/outputs.tf | 4 - tests/README.md | 86 ---- tests/deploy/ci-deploy.sh | 180 ++++++++ tests/deploy/infra-ci.tfvars.tftpl | 69 +++ .../{ => deploy/legacy-test}/ci.tfvars.tftpl | 5 +- tests/{ => deploy/legacy-test}/main.tf.json | 19 +- tests/deploy/meta.sh | 16 + .../plan}/create-kms-key/README.md | 0 .../plan}/create-kms-key/main.tf | 0 .../plan}/create-kms-key/outputs.tf | 0 .../plan}/create-kms-key/variables.tf | 0 .../plan/create-kms-key}/versions.tf | 0 tests/plan/terraform/README.md | 50 ++ tests/plan/terraform/main.tf | 52 +++ tests/plan/terraform/outputs.tf | 0 tests/plan/terraform/variables.tf | 391 ++++++++++++++++ tests/plan/terraform/versions.tf | 13 + tests/plan/tf-plan-test.sh | 122 +++++ versions.tf | 43 -- 168 files changed, 4687 insertions(+), 1913 deletions(-) rename bin/{ => pre-commit}/check-aws-partition.sh (100%) create mode 100644 bin/state-migration/README.md create mode 100755 bin/state-migration/migrate-states.sh delete mode 100644 examples/bring-your-vpc/README.md delete mode 100644 examples/bring-your-vpc/main.tf delete mode 100644 examples/bring-your-vpc/outputs.tf delete mode 100644 examples/bring-your-vpc/variables.tf delete mode 100644 examples/custom-node-pool/README.md delete mode 100644 examples/custom-node-pool/main.tf delete mode 100644 examples/custom-node-pool/outputs.tf delete mode 100644 examples/custom-node-pool/variables.tf create mode 100644 examples/deploy/README.md create mode 100644 examples/deploy/meta.sh create mode 100755 examples/deploy/set-mod-version.sh create mode 100644 examples/deploy/terraform/cluster.tfvars create mode 100644 examples/deploy/terraform/cluster/README.md create mode 100644 examples/deploy/terraform/cluster/main.tf create mode 100644 examples/deploy/terraform/cluster/outputs.tf create mode 100644 examples/deploy/terraform/cluster/variables.tf create mode 100644 examples/deploy/terraform/infra.tfvars create mode 100644 examples/deploy/terraform/infra/README.md create mode 100644 examples/deploy/terraform/infra/main.tf create mode 100644 examples/deploy/terraform/infra/outputs.tf create mode 100644 examples/deploy/terraform/infra/variables.tf create mode 100644 examples/deploy/terraform/nodes.tfvars create mode 100644 examples/deploy/terraform/nodes/README.md create mode 100644 examples/deploy/terraform/nodes/main.tf create mode 100644 examples/deploy/terraform/nodes/outputs.tf create mode 100644 examples/deploy/terraform/nodes/variables.tf create mode 100755 examples/deploy/tf.sh delete mode 100644 examples/examples.pem delete mode 100644 examples/kms-additional-policy/README.md delete mode 100644 examples/kms-additional-policy/kms-policy.json delete mode 100644 examples/kms-additional-policy/main.tf delete mode 100644 examples/kms-additional-policy/outputs.tf delete mode 100644 examples/kms-additional-policy/variables.tf delete mode 100644 examples/kms-additional-policy/versions.tf delete mode 100644 examples/kms/README.md delete mode 100644 examples/kms/main.tf delete mode 100644 examples/kms/outputs.tf delete mode 100644 examples/kms/variables.tf delete mode 100644 examples/kms/versions.tf delete mode 100644 examples/minimal-with-bastion/README.md delete mode 100644 examples/minimal-with-bastion/main.tf delete mode 100644 examples/minimal-with-bastion/outputs.tf delete mode 100644 examples/minimal-with-bastion/variables.tf delete mode 100644 examples/minimal-with-bastion/versions.tf delete mode 100644 examples/nodes-custom-ami/README.md delete mode 100644 examples/nodes-custom-ami/main.tf delete mode 100644 examples/nodes-custom-ami/outputs.tf delete mode 100644 examples/nodes-custom-ami/variables.tf delete mode 100644 examples/nodes-custom-ami/versions.tf delete mode 100644 examples/public-access/README.md delete mode 100644 examples/public-access/main.tf delete mode 100644 examples/public-access/outputs.tf delete mode 100644 examples/public-access/variables.tf delete mode 100644 examples/public-access/versions.tf delete mode 100755 examples/tf-plan-test.sh create mode 100644 examples/tfvars/bring-your-vpc.tfvars create mode 100644 examples/tfvars/custom-node-pool.tfvars create mode 100644 examples/tfvars/kms-add-policies.tfvars create mode 100644 examples/tfvars/kms-byok.tfvars create mode 100644 examples/tfvars/minimal-with-bastion.tfvars create mode 100644 examples/tfvars/minimal-wo-bastion.tfvars create mode 100644 examples/tfvars/nodes-custom-ami.tfvars create mode 100644 examples/tfvars/private-link.tfvars create mode 100644 examples/tfvars/public-access.tfvars rename {submodules => modules}/eks/README.md (64%) rename {submodules => modules}/eks/cluster.tf (87%) rename {submodules => modules}/eks/eks-bastion-access.tf (100%) rename {submodules => modules}/eks/iam.tf (99%) create mode 100644 modules/eks/k8s.tf rename {submodules => modules}/eks/main.tf (93%) create mode 100644 modules/eks/node-group.tf create mode 100644 modules/eks/outputs.tf create mode 100644 modules/eks/privatelink.tf rename {submodules => modules/eks/submodules}/k8s/README.md (98%) rename {submodules => modules/eks/submodules}/k8s/main.tf (89%) rename {submodules => modules/eks/submodules}/k8s/outputs.tf (75%) rename {submodules => modules/eks/submodules}/k8s/templates/aws-auth.yaml.tftpl (100%) rename {submodules => modules/eks/submodules}/k8s/templates/eniconfig.yaml.tftpl (100%) rename {submodules => modules/eks/submodules}/k8s/templates/k8s-functions.sh.tftpl (89%) rename {submodules => modules/eks/submodules}/k8s/templates/k8s-pre-setup.sh.tftpl (100%) rename {submodules => modules/eks/submodules}/k8s/variables.tf (100%) rename {submodules => modules/eks/submodules}/k8s/versions.tf (100%) create mode 100644 modules/eks/submodules/privatelink/README.md create mode 100644 modules/eks/submodules/privatelink/aws-load-balancer-controller_2.5.4_iam_policy.json create mode 100644 modules/eks/submodules/privatelink/iam.tf create mode 100644 modules/eks/submodules/privatelink/main.tf create mode 100644 modules/eks/submodules/privatelink/outputs.tf create mode 100644 modules/eks/submodules/privatelink/variables.tf create mode 100644 modules/eks/submodules/privatelink/versions.tf create mode 100644 modules/eks/submodules/privatelink/vpc-endpoint-services.tf rename {submodules => modules}/eks/variables.tf (67%) rename {submodules => modules}/eks/versions.tf (54%) rename {iam-bootstrap => modules/iam-bootstrap}/README.md (100%) rename {iam-bootstrap => modules/iam-bootstrap}/bootstrap-0.json (97%) rename {iam-bootstrap => modules/iam-bootstrap}/bootstrap-1.json (84%) rename {iam-bootstrap => modules/iam-bootstrap}/main.tf (100%) rename {iam-bootstrap => modules/iam-bootstrap}/outputs.tf (100%) rename {iam-bootstrap => modules/iam-bootstrap}/render.sh (100%) rename {iam-bootstrap => modules/iam-bootstrap}/variables.tf (100%) rename {examples/create-kms-key => modules/iam-bootstrap}/versions.tf (100%) create mode 100644 modules/infra/README.md rename iam.tf => modules/infra/iam.tf (90%) rename kms.tf => modules/infra/kms.tf (100%) rename main.tf => modules/infra/main.tf (73%) create mode 100644 modules/infra/outputs.tf rename {submodules => modules/infra/submodules}/bastion/README.md (100%) rename {submodules => modules/infra/submodules}/bastion/main.tf (100%) rename {submodules => modules/infra/submodules}/bastion/outputs.tf (100%) rename {submodules => modules/infra/submodules}/bastion/templates/install-binaries.sh.tftpl (100%) rename {submodules => modules/infra/submodules}/bastion/variables.tf (100%) rename {submodules => modules/infra/submodules}/bastion/versions.tf (100%) rename {submodules => modules/infra/submodules}/network/README.md (100%) rename {submodules => modules/infra/submodules}/network/internet-gateway.tf (100%) rename {submodules => modules/infra/submodules}/network/main.tf (99%) rename {submodules => modules/infra/submodules}/network/nat-gateway.tf (100%) rename {submodules => modules/infra/submodules}/network/outputs.tf (92%) rename {submodules => modules/infra/submodules}/network/route-tables.tf (100%) rename {submodules => modules/infra/submodules}/network/subnets.tf (100%) rename {submodules => modules/infra/submodules}/network/variables.tf (100%) rename {submodules => modules/infra/submodules}/network/versions.tf (100%) rename {submodules => modules/infra/submodules}/network/vpc.tf (100%) rename {submodules => modules/infra/submodules}/storage/README.md (100%) rename {submodules => modules/infra/submodules}/storage/ecr.tf (100%) rename {submodules => modules/infra/submodules}/storage/efs.tf (100%) rename {submodules => modules/infra/submodules}/storage/efs_backup_vault.tf (99%) rename {submodules => modules/infra/submodules}/storage/iam.tf (100%) rename {submodules => modules/infra/submodules}/storage/main.tf (100%) rename {submodules => modules/infra/submodules}/storage/outputs.tf (100%) rename {submodules => modules/infra/submodules}/storage/s3.tf (99%) rename {submodules => modules/infra/submodules}/storage/variables.tf (100%) rename {submodules => modules/infra/submodules}/storage/versions.tf (100%) rename variables.tf => modules/infra/variables.tf (90%) rename {examples/bring-your-vpc => modules/infra}/versions.tf (69%) create mode 100644 modules/nodes/README.md create mode 100644 modules/nodes/main.tf rename submodules/eks/node-group.tf => modules/nodes/nodes.tf (57%) create mode 100644 modules/nodes/outputs.tf rename {submodules/eks => modules/nodes}/templates/linux_user_data.tpl (100%) create mode 100644 modules/nodes/variables.tf rename {examples/custom-node-pool => modules/nodes}/versions.tf (63%) delete mode 100644 outputs.tf delete mode 100644 submodules/eks/k8s.tf delete mode 100644 submodules/eks/outputs.tf delete mode 100644 tests/README.md create mode 100755 tests/deploy/ci-deploy.sh create mode 100644 tests/deploy/infra-ci.tfvars.tftpl rename tests/{ => deploy/legacy-test}/ci.tfvars.tftpl (63%) rename tests/{ => deploy/legacy-test}/main.tf.json (89%) create mode 100644 tests/deploy/meta.sh rename {examples => tests/plan}/create-kms-key/README.md (100%) rename {examples => tests/plan}/create-kms-key/main.tf (100%) rename {examples => tests/plan}/create-kms-key/outputs.tf (100%) rename {examples => tests/plan}/create-kms-key/variables.tf (100%) rename {iam-bootstrap => tests/plan/create-kms-key}/versions.tf (100%) create mode 100644 tests/plan/terraform/README.md create mode 100644 tests/plan/terraform/main.tf create mode 100644 tests/plan/terraform/outputs.tf create mode 100644 tests/plan/terraform/variables.tf create mode 100644 tests/plan/terraform/versions.tf create mode 100755 tests/plan/tf-plan-test.sh delete mode 100644 versions.tf diff --git a/.circleci/config.yml b/.circleci/config.yml index f5f51033..69ae9137 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,15 +1,15 @@ version: 2.1 parameters: - workspace: - type: string - default: circleci-<< pipeline.number >> helm_version: type: string default: "v3.11.2" terraform_version: type: string default: "1.4.4" + hcledit_version: + type: string + default: "0.2.9" GHA_Actor: type: string default: "" @@ -26,14 +26,6 @@ orbs: envsubst: sawadashota/envsubst@1.1.0 commands: - set_tf_vars: - description: "Sets Terraform variables" - steps: - - run: - name: Configure terraform vars - working_directory: tests - command: | - envsubst < ci.tfvars.tftpl | tee terraform.tfvars install_tf: description: "Install Terraform" parameters: @@ -42,6 +34,45 @@ commands: steps: - terraform/install: terraform_version: << parameters.terraform_version >> + install_hcledit: + description: "Install HCL edit" + parameters: + hcledit_version: + type: string + default: "0.2.9" + steps: + - run: + name: Install HCL edit + working_directory: tests/deploy + environment: + HCLEDIT_VERSION: << parameters.hcledit_version >> + command: bash ci-deploy.sh install_hcledit + set_current_mod_source: + description: "Set up module source to current branch." + steps: + - run: + name: "Set module source to current branch" + working_directory: tests/deploy + command: bash ci-deploy.sh set_mod_src_circle_branch + set_aws_creds: + description: "Sets short-lived creds" + steps: + - aws-cli/setup: + role-arn: "${AWS_IAM_ROLE}" + session-duration: "900" + set_tf_vars: + description: "Sets Terraform variables" + steps: + - envsubst/install + - install_hcledit + - set_aws_creds + - run: + name: Bootstrap terraform module/vars. + working_directory: tests/deploy + command: | + bash ci-deploy.sh setup_modules + bash ci-deploy.sh set_tf_vars + - set_current_mod_source install_helm: description: "Install Helm" parameters: @@ -50,71 +81,67 @@ commands: steps: - run: name: Install Helm + working_directory: tests/deploy environment: HELM_VERSION: << parameters.helm_version >> - command: | - if [ -z "$HELM_VERSION" ]; then - echo "HELM_VERSION environment variable not set, exiting." - exit 1 - fi - echo "Installing Helm version: ${HELM_VERSION}" - curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - chmod +x get_helm.sh - ./get_helm.sh --version "${HELM_VERSION}" - rm ./get_helm.sh - helm version --short - gen_pvt_key: - description: "Generates ssh key" - steps: - - run: - name: Generate pvt key - command: ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f tests/domino.pem + command: bash ci-deploy.sh install_helm tf_init_apply: description: "Terraform init" steps: - - aws-cli/setup: - role-arn: "${AWS_IAM_ROLE}" - session-duration: "43200" - run: name: Terraform init/validate/apply - working_directory: tests + working_directory: tests/deploy command: | - echo "Current dir: $(pwd)" - terraform init - terraform validate - terraform workspace new ${WORKSPACE} - terraform apply -auto-approve + ## Most of this verbiage will go away after release. + set -e + echo "Current dir: $PWD" + + if [ -f "migrated.txt" ]; then + echo "This legacy deployment has been migrated" + bash ci-deploy.sh deploy + elif [ -n "$MAJOR_MOD_VERSION" ] && (( $MAJOR_MOD_VERSION < 3 )) ; then + + source "${PWD}/meta.sh" + echo "CI determined this is a legacy deploy." + + echo "Generating Deployment pvt key." + export LEGACY_PVT_KEY="${LEGACY_DIR}/domino.pem" + envsubst < "${LEGACY_DIR}/ci.tfvars.tftpl" | tee "${LEGACY_DIR}/terraform.tfvars" + ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f "$LEGACY_PVT_KEY" + + echo "Running legacy monolithic deploy" + terraform -chdir="$LEGACY_DIR" init + terraform -chdir="$LEGACY_DIR" validate + terraform -chdir="$LEGACY_DIR" apply --auto-approve --input=false + + echo "Running state migration on CI." + bash -xp ../../bin/state-migration/migrate-states.sh + else + echo "Running ci-deploy.sh deploy" + bash ci-deploy.sh deploy + fi + tf_deploy: + description: "Terraform deploy" + steps: + - aws-cli/setup: + role-arn: "${AWS_IAM_ROLE}" + session-duration: "43200" + - tf_init_apply tf_destroy: description: "Terraform destroy" steps: - run: name: Terraform destroy - working_directory: tests - command: | - echo "Current dir: $(pwd)" - terraform destroy --auto-approve || terraform destroy --auto-approve --refresh=false - when: always - tf_ws_delete: - description: "Terraform workspace delete" - steps: - - run: - name: Terraform workspace delete - working_directory: tests - command: | - echo "Current dir: $(pwd)" - terraform workspace select default - terraform workspace delete ${WORKSPACE} + working_directory: tests/deploy + command: bash ci-deploy.sh destroy when: always tf_plan_test: steps: - - aws-cli/setup: - role-arn: "${AWS_IAM_ROLE}" - session-duration: "900" + - set_aws_creds - run: name: Terraform plan test - working_directory: examples - command: | - bash tf-plan-test.sh + working_directory: tests/plan + command: bash tf-plan-test.sh jobs: tf-plan-test: docker: @@ -131,93 +158,68 @@ jobs: docker: - image: cimg/aws:2023.04.1 parameters: - workspace: - type: string terraform_version: type: string helm_version: type: string - environment: - WORKSPACE: << parameters.workspace >> steps: - checkout - install_tf: terraform_version: << parameters.terraform_version >> - install_helm: helm_version: << parameters.helm_version >> - - envsubst/install - set_tf_vars - - gen_pvt_key - - tf_init_apply + - tf_deploy - tf_destroy - - tf_ws_delete test-upgrade: docker: - image: cimg/aws:2023.04.1 parameters: - workspace: - type: string terraform_version: type: string helm_version: type: string - environment: - WORKSPACE: << parameters.workspace >> + hcledit_version: + type: string steps: - checkout - install_tf: terraform_version: << parameters.terraform_version >> - install_helm: helm_version: << parameters.helm_version >> - - envsubst/install + - install_hcledit - set_tf_vars - - gen_pvt_key - run: name: "Set module source to latest published release" - working_directory: tests - command: | - latest_release_tag=$(jq -r '.tag_name' <(curl -sSfL -H "X-GitHub-Api-Version: 2022-11-28" -H "Accept: application/vnd.github+json" https://api.github.com/repos/dominodatalab/terraform-aws-eks/releases/latest)) - export MOD_SOURCE="github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git?ref=${latest_release_tag}" - echo "Latest published release tag is: ${latest_release_tag}" - echo "Setting module source to: ${MOD_SOURCE}" - cat \<<< $(jq --arg mod_source "${MOD_SOURCE}" '.module[0].domino_eks.source = $mod_source' main.tf.json) >main.tf.json + working_directory: tests/deploy + command: bash ci-deploy.sh set_mod_src_latest_rel + - tf_deploy + - set_current_mod_source - tf_init_apply - - run: - name: "Upgrade module by applying this commit" - working_directory: tests - command: | - echo "Testing terraform module upgrade" - export MOD_SOURCE="./.." - echo "Setting module source to local ref: ${MOD_SOURCE}" - cat \<<< $(jq --arg mod_source "${MOD_SOURCE}" '.module[0].domino_eks.source = $mod_source' main.tf.json) >main.tf.json - terraform init --reconfigure --upgrade - terraform validate - terraform apply --auto-approve - tf_destroy - - tf_ws_delete workflows: test-deploy-workflow: when: - equal: ["test-deploy-workflow", << pipeline.parameters.GHA_Action >> ] + equal: ["test-deploy-workflow", << pipeline.parameters.GHA_Action >>] jobs: - test-deploy: context: aws-oidc terraform_version: << pipeline.parameters.terraform_version >> helm_version: << pipeline.parameters.helm_version >> - workspace: << pipeline.parameters.workspace >> test-upgrade-workflow: when: - equal: ["test-upgrade-workflow", << pipeline.parameters.GHA_Action >> ] + equal: ["test-upgrade-workflow", << pipeline.parameters.GHA_Action >>] jobs: - test-upgrade: context: aws-oidc terraform_version: << pipeline.parameters.terraform_version >> helm_version: << pipeline.parameters.helm_version >> - workspace: << pipeline.parameters.workspace >> + hcledit_version: << pipeline.parameters.hcledit_version >> examples-plan-test-workflow: when: - equal: ["examples-plan-test-workflow", << pipeline.parameters.GHA_Action >> ] + equal: + ["examples-plan-test-workflow", << pipeline.parameters.GHA_Action >>] jobs: - tf-plan-test: context: aws-oidc diff --git a/.gitignore b/.gitignore index 88fa915b..779aa0ea 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,13 @@ *.tfstate *.tfstate.* +*infra.outputs* +*cluster.outputs* +*nodes.outputs* +**migrated.txt +**/deploy-test/* +**terraform.plan + **.terraform.lock.hcl* **.terraform.lock.hcl # Crash log files @@ -31,6 +38,7 @@ override.tf.json # example: *tfplan* aws-auth.yaml domino.pem +plan-test.pem* domino.pem.pub k8s-functions.sh k8s-pre-setup.sh @@ -41,3 +49,4 @@ terraform_tfsec.txt eniconfig.yaml kubeconfig-proxy terraform.tfvars +k8s-proxy-tunnel.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e48bf66e..c67a307e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -19,11 +19,11 @@ repos: - id: check-dependabot - id: check-github-actions - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.79.1 + rev: v1.81.0 hooks: - id: terraform_validate # See #4 on https://github.com/antonbabenko/pre-commit-terraform#terraform_validate - exclude: submodules/eks/[^/]+$ + exclude: modules/eks/[^/]+$ args: - "--hook-config=--retry-once-with-cleanup=true" - id: terraform_providers_lock @@ -61,7 +61,7 @@ repos: args: - "--args=--compact" - "--args=--quiet" - - "--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19" + - "--args=--skip-check CKV_CIRCLECIPIPELINES_2,CKV_CIRCLECIPIPELINES_6,CKV2_AWS_11,CKV2_AWS_12,CKV2_AWS_6,CKV_AWS_109,CKV_AWS_111,CKV_AWS_135,CKV_AWS_144,CKV_AWS_145,CKV_AWS_158,CKV_AWS_18,CKV_AWS_184,CKV_AWS_19,CKV_AWS_21,CKV_AWS_66,CKV_AWS_88,CKV2_GHA_1,CKV_AWS_163,CKV_AWS_39,CKV_AWS_38,CKV2_AWS_61,CKV2_AWS_62,CKV_AWS_136,CKV_AWS_329,CKV_AWS_338,CKV_AWS_339,CKV_AWS_341,CKV_AWS_356,CKV2_AWS_19,CKV2_AWS_5,CKV_AWS_150,CKV_AWS_123" - id: terraform_tfsec args: - "--args=-e aws-s3-specify-public-access-block,aws-cloudwatch-log-group-customer-key,aws-s3-enable-bucket-logging,aws-s3-enable-versioning,aws-s3-no-public-buckets,aws-ec2-require-vpc-flow-logs-for-all-vpcs,aws-s3-encryption-customer-key,aws-ec2-no-public-egress-sgr,aws-iam-no-policy-wildcards,aws-s3-block-public-acls,aws-s3-block-public-policy,aws-s3-enable-bucket-encryption,aws-s3-ignore-public-acls,aws-ec2-no-public-ingress-sgr,aws-ecr-repository-customer-key,aws-ecr-enable-image-scans,aws-eks-no-public-cluster-access,aws-eks-no-public-cluster-access-to-cidr" @@ -74,6 +74,6 @@ repos: hooks: - id: check_aws_partition name: Check for hard coded AWS partition - entry: ./bin/check-aws-partition.sh + entry: ./bin/pre-commit/check-aws-partition.sh language: script - exclude: "^(examples|bin)" + exclude: "^(bin|examples)" diff --git a/README.md b/README.md index 8a2ff1e1..07ec801d 100644 --- a/README.md +++ b/README.md @@ -1,224 +1,284 @@ -# terraform-aws-eks +# Terraform Module for EKS Setup. Optimized for Domino Installation [![CircleCI](https://dl.circleci.com/status-badge/img/gh/dominodatalab/terraform-aws-eks/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/dominodatalab/terraform-aws-eks/tree/main) -:warning: **Starting on `v2.0.0` of this module kms is enabled by default. If you used an earlier version and did not set the `kms.enabled` variable, you will need to set `kms.enabled = false` to maintain the current state. Otherwise you will rely on the default `kms.enabled = true` which may destroy existing infrastructure and result in data loss.** +:warning: Important: If you have existing infrastructure created with a version of this module < `v3.0.0` you will need to migrate your state before updating the module to versions >= `v3.0.0`. See [state-migration](./bin/state-migration/README.md#terraform-state-migration-guide) for more details. -## Create SSH Key pair -### Prerequisites -* Host with `ssh-keygen` installed +:warning: Important: Starting from version v2.0.0, this module has KMS enabled by default. If you utilized a prior version without setting the kms.enabled variable, ensure you define kms.enabled = false to preserve your existing state. Failing to do so will default to kms.enabled = true, potentially causing the destruction of your existing infrastructure and possible data loss. + + +## Repository Structure + +* examples/: + * Purpose: Acts as an intuitive guide for module users. + * Contents: Features end-user Terraform configurations along with illustrative `tfvars` samples, showcasing potential setups. + +* modules/: + * Purpose: Houses the primary modules that orchestrate the provisioning process. + * Contents: These modules act as the main entry points for the setup of the underlying infrastructure beneath the **EKS** cluster and its associated components. + +* tests/: + * Purpose: Ensures the integrity and functionality of the module. + * Contents: Contains automation-driven tests intended for validation and continuous integration (CI) checks. + +* bin/state-migration/: + * Purpose: Contains automation to perform terraform state migration, from a monolithic module to a multi-module structure. + * Contents: Script and documentation to perform terraform state migration. + +Always refer to each section's respective README or documentation for detailed information and usage guidelines. + +## Prerequisites +* A host with `ssh-keygen` installed * [awscli](https://docs.aws.amazon.com/cli/latest/userguide/getting-started-install.html) * [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli?in=terraform/aws-get-started#install-terraform) >= v1.3.0 -* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) cli >= 1.24.0 +* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) cli >= 1.25.0 +* [helm](https://helm.sh/docs/intro/install/) >= 3.9 +* [hcledit](https://github.com/minamijoyo/hcledit) +* bash >= 4.0 + + +## Bootstrap module +We first need to setup the module structure. + +### 1. Set your desired module version and deployment directory: +Set up the following environment variables. +Update the following values(Using `v3.0.0` and `domino-deploy` as an example): +```bash +MOD_VERSION='v3.0.0' +DEPLOY_DIR='domino-deploy' +``` +:warning: Ensure the `DEPLOY_DIR` does not exist or is currently empty. + +Create the `DEPLOY_DIR` and use terraform to bootstrap the module from source. + +```bash +mkdir -p "$DEPLOY_DIR" +terraform -chdir="$DEPLOY_DIR" init -backend=false -from-module="github.com/dominodatalab/terraform-aws-eks.git//examples/deploy?ref=${MOD_VERSION}" +``` +Ignore this message: + +``` +Terraform initialized in an empty directory! + +The directory has no Terraform configuration files. You may begin working +with Terraform immediately by creating Terraform configuration files. +``` + +:white_check_mark: If successful, you should get a structure similar to this: + +```bash +domino-deploy +├── README.md +├── meta.sh +├── set-mod-version.sh +├── terraform +│   ├── cluster +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   ├── cluster.tfvars +│   ├── infra +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   ├── infra.tfvars +│   ├── nodes +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   └── nodes.tfvars +└── tf.sh +``` + +**Note**: It's recommended to go through the README.md within the `DEPLOY_DIR` for further details. + +### 2. Update modules version +You can update the modules version using a script or manually. + +#### Using script +You can update the modules version using the `set-mod-version.sh` + +##### Prerequisites +* [hcledit](https://github.com/minamijoyo/hcledit) + +#### Command + +```bash +./set-mod-version.sh "$MOD_VERSION" +``` + +#### Manually +Update the modules' source with the `MOD_VERSION` value. + +For example if `MOD_VERSION=v3.0.0` + +* **infra/main.tf** : Update `module.infra.source` from `"./../../../../modules/infra"` to `github.com/dominodatalab/terraform-aws-eks.git//modules/infra?ref=v3.0.0` +* **cluster/main.tf** : Update `module.eks.source` from `"./../../../../modules/eks"` to `github.com/dominodatalab/terraform-aws-eks.git//modules/eks?ref=v3.0.0` +* **nodes/main.tf** : Update `module.nodes.source` from `"./../../../../modules/nodes"` to `github.com/dominodatalab/terraform-aws-eks.git//modules/nodes?ref=v3.0.0` + + +### 3. Review and Configure `tfvars` + +Consult available variables within each of the modules `variables.tf` + +* `domino-deploy/terraform/infra/variables.tf` + * `deploy_id` + * `region` + * `tags` + * `network` + * `default_node_groups` + * `additional_node_groups` + * `storage` + * `kms` + * `eks` + * `ssh_pvt_key_path` + * `route53_hosted_zone_name` + * `bastion` + +* `domino-deploy/terraform/cluster/variables.tf` + * `eks` + * `kms_info`: :warning: Variable is only intended for migrating infrastructure, it is not recommended to set it. + +* `domino-deploy/terraform/nodes/variables.tf` + * `default_node_groups` + * `additional_node_groups` + +Configure terraform variables at: +* `domino-deploy/terraform/infra.tfvars` +* `domino-deploy/terraform/cluster.tfvars` +* `domino-deploy/terraform/nodes.tfvars` -### Command +**NOTE**: The `eks` configuration is required in both the `infra` and `cluster` modules because the Kubernetes version is used for installing the `kubectl` binary on the bastion host. Similarly, `default_node_groups` and `additional_node_groups` must be defined in both the `infra` and `nodes` modules, as the `availability zones` for the `nodes` are necessary for setting up the network infrastructure. +The `eks` module will source its information from the `infra` outputs if it is not configured on `cluster.tfvars`, as will the `nodes` module if the variables are not configured on `nodes.tfvars`. We recommended setting the variables in `eks` and `nodes` from the beggining as future kubernetes upgrades will be driven from `cluster.tfvars` and `nodes.tfvars`. + + +### 4. Create SSH Key pair +The deployment requires an SSH key. Update the `ssh_pvt_key_path` variable in `domino-deploy/terraform/infra.tfvars` with the full path of your key (we recommend you place your key under the `domino-deploy/terraform` directory). + +If you don't have an SSH key, you can create one using: +```bash + ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem && chmod 600 domino.pem +``` + +### 5. Deploy +#### 1. Set `AWS` credentials and verify. ```bash - ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem && chmod 400 domino.pem +aws sts get-caller-identity ``` -## Create terraform remote state bucket(OPTIONAL) -* Authenticate with aws, make sure that environment variables: `AWS_REGION`, `AWS_ACCESS_KEY_ID` ,`AWS_SECRET_ACCESS_KEY` are set. If your account has MFA set up you will also need `AWS_SESSION_TOKEN`. +#### 2. Change into `domino-deploy`(or whatever your `DEPLOY_DIR` is) -#### 1. Create Bucket(if you already have a bucket just set the `AWS_TERRAFORM_REMOTE_STATE_BUCKET` to its name, and skip this step): ```bash -export AWS_ACCOUNT="$(aws sts get-caller-identity | jq -r .Account)" -export AWS_TERRAFORM_REMOTE_STATE_BUCKET="domino-terraform-rs-${AWS_ACCOUNT}-${AWS_REGION}" +cd domino-deploy +``` + +#### 3. Plan and Apply. +:warning: It is recommended to become familiar with the `tf.sh` [usage](./examples/deploy/README.md#usage). + +At this point all requirements should be set to provision the infrastructure. + +For each of the modules, run `init`, `plan`, inspect the plan, then `apply` in the following order: + +1. `infra` +2. `cluster` +3. `nodes` -aws s3api create-bucket \ - --bucket "${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" \ - --region ${AWS_REGION} \ - --create-bucket-configuration LocationConstraint="${AWS_REGION}" | jq . +Note: You can use `all` instead but it is recommended that the `plan` and `apply` be done one at a time, so that the plans can be carefully examined. + +1. Init all + +```bash +./tf.sh all init ``` -#### Verify bucket exists +2. `infra` plan. ```bash -aws s3api head-bucket --bucket "${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" +./tf.sh infra plan ``` -You should NOT see an error. +3. :exclamation: Carefully inspect the actions detailed in the `infra` plan for correctness, before proceeding. -## 2. Initialize the terraform remote-state -Create a file called terraform.tf(the name does not matter) with the following content -```hcl -terraform { - backend "s3" {} -} +4. `infra` apply + +```bash +./tf.sh infra apply ``` +5. `cluster` plan + ```bash -### Set the deploy id. This will be used later as well. -export TF_VAR_deploy_id="domino-eks-1" ## <-- Feel free to rename. -terraform init -migrate-state \ - -backend-config="bucket=${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" \ - -backend-config="key=domino-eks/${TF_VAR_deploy_id}" \ - -backend-config="region=${AWS_REGION}" +./tf.sh cluster plan ``` +6. :exclamation: Carefully inspect the actions detailed in the `cluster` plan for correctness, before proceeding. +7. `cluster` apply -## If you need to delete the bucket +```bash +./tf.sh cluster apply +``` + +8. nodes plan ```bash +./tf.sh nodes plan +``` +9. :exclamation: Carefully inspect the actions detailed in the `nodes` plan for correctness, before proceeding. + +10. `nodes` apply -aws s3 rb s3://"${AWS_TERRAFORM_REMOTE_STATE_BUCKET}" --force +```bash +./tf.sh nodes apply ``` -# Terraform-docs +### At this point the infrastructure has been created. + + +### Interacting with Kubernetes +To interact with the EKS Control Plane using kubectl or helm commands, you'll need to set up both the appropriate AWS credentials and the KUBECONFIG environment variable. If your EKS cluster is private, you can use mechanisms provided by this module to establish an SSH tunnel through a Bastion host. However, if your EKS endpoint is publicly accessible, you only need to follow steps 1-3 below. - -## Requirements +For ease of setup, use the k8s-functions.sh script, which contains helper functions for cluster configuration. -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [time](#requirement\_time) | >= 0.9.1 | -| [tls](#requirement\_tls) | >= 3.4.0 | +#### Steps +1. Verify AWS Credentials: Ensure your AWS credentials are properly configured by running the following command: -## Providers +```bash +aws sts get-caller-identity +``` -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.0 | -| [time](#provider\_time) | >= 0.9.1 | -| [tls](#provider\_tls) | >= 3.4.0 | +2. Import Functions: Source the k8s-functions.sh script to import its functions into your current shell. -## Modules +```bash +source k8s-functions.sh +``` + +3. Set `KUBECONFIG`: Use the check_kubeconfig function to set the `KUBECONFIG` environment variable appropriately. + +```bash +check_kubeconfig +``` -| Name | Source | Version | -|------|--------|---------| -| [bastion](#module\_bastion) | ./submodules/bastion | n/a | -| [eks](#module\_eks) | ./submodules/eks | n/a | -| [network](#module\_network) | ./submodules/network | n/a | -| [storage](#module\_storage) | ./submodules/storage | n/a | +4. Open SSH Tunnel (Optional): If your EKS cluster is private, open an SSH tunnel through the Bastion host by executing: -## Resources +```bash +open_ssh_tunnel_to_k8s_api +``` + +5. Close SSH Tunnel: To close the SSH tunnel, run: + +```bash +close_ssh_tunnel_to_k8s_api +``` + +### Retrieve Configuration Values for `domino.yaml`. +Run the command below to generate a list of infrastructure values. These values are necessary for configuring the domino.yaml file, which is in turn used for installing the Domino product. + +```bash +./tf.sh infra output domino_config_values +``` -| Name | Type | -|------|------| -| [aws_iam_policy.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | -| [aws_iam_role_policy_attachment.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_iam_role_policy_attachment.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_key_pair.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | -| [aws_kms_alias.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | -| [aws_kms_key.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [time_sleep.create_eks_role_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | -| [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | -| [aws_ec2_instance_type.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | -| [aws_iam_policy_document.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.kms_key_global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_kms_key.key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source | -| [tls_public_key.domino](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/public_key) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })), [])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | -| [bastion](#input\_bastion) | enabled = Create bastion host.
ami = Ami id. Defaults to latest 'amazon\_linux\_2' ami.
instance\_type = Instance type.
authorized\_ssh\_ip\_ranges = List of CIDR ranges permitted for the bastion ssh access.
username = Bastion user.
install\_binaries = Toggle to install required Domino binaries in the bastion. | 
object({
    enabled                  = optional(bool, true)
    ami_id                   = optional(string, null) # default will use the latest 'amazon_linux_2' ami
    instance_type            = optional(string, "t3.micro")
    authorized_ssh_ip_ranges = optional(list(string), ["0.0.0.0/0"])
    username                 = optional(string, "ec2-user")
    install_binaries         = optional(bool, false)
  })
| `{}` | no | -| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      })
  })
| n/a | yes | -| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks"` | no | -| [eks](#input\_eks) | k8s\_version = "EKS cluster k8s version."
kubeconfig = {
extra\_args = "Optional extra args when generating kubeconfig."
path = "Fully qualified path name to write the kubeconfig file."
}
public\_access = {
enabled = "Enable EKS API public endpoint."
cidrs = "List of CIDR ranges permitted for accessing the EKS public endpoint."
}
"Custom role maps for aws auth configmap"
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = "IAM role names to be added as masters in eks."
cluster\_addons = "EKS cluster addons. vpc-cni is installed separately."
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = "CloudWatch log group to send the SSM session logs to."
identity\_providers = "Configuration for IDP(Identity Provider)."
} | 
object({
    k8s_version = optional(string, "1.25")
    kubeconfig = optional(object({
      extra_args = optional(string, "")
      path       = optional(string, "kubeconfig")
    }), {})
    public_access = optional(object({
      enabled = optional(bool, false)
      cidrs   = optional(list(string), [])
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })), [])
    master_role_names  = optional(list(string), [])
    cluster_addons     = optional(list(string), ["kube-proxy", "coredns"])
    ssm_log_group_name = optional(string, "session-manager")
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string, null)
      groups_prefix                 = optional(string, null)
      identity_provider_config_name = string
      issuer_url                    = optional(string, null)
      required_claims               = optional(string, null)
      username_claim                = optional(string, null)
      username_prefix               = optional(string, null)
    })), [])
  })
| `{}` | no | -| [kms](#input\_kms) | enabled = "Toggle, if set use either the specified KMS key\_id or a Domino-generated one"
key\_id = optional(string, null)
additional\_policies = "Allows setting additional KMS key policies when using a Domino-generated key" | 
object({
    enabled             = optional(bool, true)
    key_id              = optional(string, null)
    additional_policies = optional(list(string), [])
  })
| `{}` | no | -| [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. | 
object({
    vpc = optional(object({
      id = optional(string, null)
      subnets = optional(object({
        private = optional(list(string), [])
        public  = optional(list(string), [])
        pod     = optional(list(string), [])
      }), {})
    }), {})
    network_bits = optional(object({
      public  = optional(number, 27)
      private = optional(number, 19)
      pod     = optional(number, 19)
      }
    ), {})
    cidrs = optional(object({
      vpc = optional(string, "10.0.0.0/16")
      pod = optional(string, "100.64.0.0/16")
    }), {})
    use_pod_cidr = optional(bool, true)
  })
| `{}` | no | -| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | -| [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNS zone. | `string` | `null` | no | -| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | -| [storage](#input\_storage) | storage = {
efs = {
access\_point\_path = Filesystem path for efs.
backup\_vault = {
create = Create backup vault for EFS toggle.
force\_destroy = Toggle to allow automatic destruction of all backups when destroying.
backup = {
schedule = Cron-style schedule for EFS backup vault (default: once a day at 12pm).
cold\_storage\_after = Move backup data to cold storage after this many days.
delete\_after = Delete backup data after this many days.
}
}
}
s3 = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets.
}
ecr = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the ECR repositories. if 'false' terraform will NOT be able to delete non-empty repositories.
}
}
} | 
object({
    efs = optional(object({
      access_point_path = optional(string, "/domino")
      backup_vault = optional(object({
        create        = optional(bool, true)
        force_destroy = optional(bool, false)
        backup = optional(object({
          schedule           = optional(string, "0 12 * * ? *")
          cold_storage_after = optional(number, 35)
          delete_after       = optional(number, 125)
        }), {})
      }), {})
    }), {})
    s3 = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
    ecr = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bastion](#output\_bastion) | Bastion details, if it was created. | -| [domino\_key\_pair](#output\_domino\_key\_pair) | Domino key pair | -| [eks](#output\_eks) | EKS details. | -| [hostname](#output\_hostname) | Domino instance URL. | -| [kms](#output\_kms) | KMS key details, if enabled. | -| [network](#output\_network) | Network details. | -| [storage](#output\_storage) | Storage details. | - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | 4.33.0 | -| [tls](#provider\_tls) | 4.0.3 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [bastion](#module\_bastion) | ./submodules/bastion | n/a | -| [eks](#module\_eks) | ./submodules/eks | n/a | -| [network](#module\_network) | ./submodules/network | n/a | -| [storage](#module\_storage) | ./submodules/storage | n/a | - -## Resources - -| Name | Type | -|------|------| -| [aws_iam_policy.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_role_policy_attachment.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_key_pair.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | -| [aws_kms_alias.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | -| [aws_kms_key.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_default_tags.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | -| [aws_ec2_instance_type.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | -| [aws_iam_policy_document.kms_key_global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_iam_policy_document.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | -| [aws_kms_key.key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source | -| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source | -| [tls_public_key.domino](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/public_key) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })), [])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | -| [bastion](#input\_bastion) | ami = Ami id. Defaults to latest 'amazon\_linux\_2' ami.
instance\_type = Instance type.
authorized\_ssh\_ip\_ranges = List of CIDR ranges permitted for the bastion ssh access.
username = Bastion user.
install\_binaries = Toggle to install required Domino binaries in the bastion. | 
object({
    ami_id                   = optional(string, null) # default will use the latest 'amazon_linux_2' ami
    instance_type            = optional(string, "t3.micro")
    authorized_ssh_ip_ranges = optional(list(string), ["0.0.0.0/0"])
    username                 = optional(string, "ec2-user")
    install_binaries         = optional(bool, false)
  })
| `null` | no | -| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      })
  })
| n/a | yes | -| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks"` | no | -| [eks](#input\_eks) | k8s\_version = "EKS cluster k8s version."
kubeconfig = {
extra\_args = "Optional extra args when generating kubeconfig."
path = "Fully qualified path name to write the kubeconfig file."
}
public\_access = {
enabled = "Enable EKS API public endpoint."
cidrs = "List of CIDR ranges permitted for accessing the EKS public endpoint."
}
"Custom role maps for aws auth configmap"
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = "IAM role names to be added as masters in eks."
cluster\_addons = "EKS cluster addons. vpc-cni is installed separately." | 
object({
    k8s_version = optional(string, "1.25")
    kubeconfig = optional(object({
      extra_args = optional(string, "")
      path       = optional(string)
    }), {})
    public_access = optional(object({
      enabled = optional(bool, false)
      cidrs   = optional(list(string), [])
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })), [])
    master_role_names = optional(list(string), [])
    cluster_addons    = optional(list(string), [])
  })
| `{}` | no | -| [kms](#input\_kms) | enabled = "Toggle,if set use either the specified KMS key\_id or a Domino-generated one"
key\_id = optional(string, null) | 
object({
    enabled = optional(bool, false)
    key_id  = optional(string, null)
  })
| `{}` | no | -| [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. | 
object({
    vpc = optional(object({
      id = optional(string, null)
      subnets = optional(object({
        private = optional(list(string), [])
        public  = optional(list(string), [])
        pod     = optional(list(string), [])
      }), {})
    }), {})
    network_bits = optional(object({
      public  = optional(number, 27)
      private = optional(number, 19)
      pod     = optional(number, 19)
      }
    ), {})
    cidrs = optional(object({
      vpc = optional(string, "10.0.0.0/16")
      pod = optional(string, "100.64.0.0/16")
    }), {})
    use_pod_cidr = optional(bool, true)
  })
| `{}` | no | -| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | -| [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNSone. | `string` | `null` | no | -| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | -| [storage](#input\_storage) | storage = {
efs = {
access\_point\_path = Filesystem path for efs.
backup\_vault = {
create = Create backup vault for EFS toggle.
force\_destroy = Toggle to allow automatic destruction of all backups when destroying.
backup = {
schedule = Cron-style schedule for EFS backup vault (default: once a day at 12pm).
cold\_storage\_after = Move backup data to cold storage after this many days.
delete\_after = Delete backup data after this many days.
}
}
}
s3 = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets.
}
ecr = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the ECR repositories. if 'false' terraform will NOT be able to delete non-empty repositories.
}
}
} | 
object({
    efs = optional(object({
      access_point_path = optional(string, "/domino")
      backup_vault = optional(object({
        create        = optional(bool, true)
        force_destroy = optional(bool, false)
        backup = optional(object({
          schedule           = optional(string, "0 12 * * ? *")
          cold_storage_after = optional(number, 35)
          delete_after       = optional(number, 125)
        }), {})
      }), {})
    }), {})
    s3 = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
    ecr = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
  })
| `{}` | no | -| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [bastion](#output\_bastion) | Bastion details, if it was created. | -| [domino\_key\_pair](#output\_domino\_key\_pair) | Domino key pair | -| [eks](#output\_eks) | EKS details. | -| [hostname](#output\_hostname) | Domino instance URL. | -| [kms](#output\_kms) | KMS key details, if enabled. | -| [network](#output\_network) | Network details. | -| [storage](#output\_storage) | Storage details. | - +This command will output a set of key-value pairs, extracted from the infrastructure setup, that can be used as inputs in the domino.yaml configuration file. diff --git a/bin/check-aws-partition.sh b/bin/pre-commit/check-aws-partition.sh similarity index 100% rename from bin/check-aws-partition.sh rename to bin/pre-commit/check-aws-partition.sh diff --git a/bin/state-migration/README.md b/bin/state-migration/README.md new file mode 100644 index 00000000..10eed48f --- /dev/null +++ b/bin/state-migration/README.md @@ -0,0 +1,80 @@ +# Terraform State Migration Guide +## Overview +This script is designed to assist with the migration of a monolithic Terraform(versions < v3.0.0 of this module) state to a modular design (versions >= v3.0.0 of this module). The original implementation provisioned the infrastructure (infra), EKS cluster, and nodes under a single Terraform state. The new design segregates these into three distinct Terraform states: Infrastructure, EKS Cluster, and Nodes. + +## Prerequisites +* Terraform installed. +* jq utility installed. +* Ensure you have the correct permissions to read and write to Terraform states. +* Backup your original Terraform state. + +## Usage: +Ensure all prerequisites are met. + +### Bootstrap the module +* Follow [Module bootstrap](../../README.md#bootstrap-module) +* Ensure `DEPLOY_DIR` is setup in accordance to previous step. + +### Set variables +1. Copy Script: + * Copy the script from bin/state-migration/migrate-states.sh to your deployment directory (DEPLOY_DIR). + ```bash + cp bin/state-migration/migrate-states.sh "$DEPLOY_DIR" + ``` +2. Verify Files: + ```bash + ls "$DEPLOY_DIR" + ``` + * Expected scripts (there should also be a directory called `terraform` and a `README.md`): + * migrate-states.sh + * tf.sh + * meta.sh +3. Append Variables to meta.sh: + * Add the following variables to your meta.sh script: + ```bash + export MOD_NAME="" + export PVT_KEY="" + export LEGACY_DIR="" + export LEGACY_PVT_KEY="" + export LEGACY_STATE="" + ``` + * Here's a brief description of each variable: + * **MOD_NAME**: Name assigned to the module during the deployment that needs migration, i.e `module.domino_eks`. + * **LEGACY_DIR**: The directory containing the deployment you want to migrate. + * **LEGACY_PVT_KEY**: Path to the SSH private key used during the provisioning of the deployment you're migrating. + * **PVT_KEY**: Path to the SSH private key. This will be used to create a copy from `LEGACY_PVT_KEY`. + * **LEGACY_STATE**: Path to the Terraform state file for the deployment you're migrating. This file is typically named `terraform.tfstate`. + +4. Run the script: + * Change into `DEPLOY_DIR` and run the script + ```bash + cd $DEPLOY_DIR + ./migrate-states.sh + ``` + +Monitor the output for any errors. Upon successful completion, you should see the message "State migration completed successfully !!!", and a migrated.txt file would be generated. + +## Detailed Operation +### Step 1: Migrate the EKS Cluster +In the original monolithic state, the EKS module contained both the EKS cluster configuration and the node groups. The first step of the migration process is to separate out the EKS cluster configuration. This is done by the migrate_cluster_state function. + +### Step 2: Migrate Infrastructure +Post the EKS cluster migration, the remaining state primarily consists of infrastructure components. The migrate_infra_state function handles this migration. + +**Note**: During the infrastructure state migration, the IAM role policy attachment specific to route53 ('module.infra.aws_iam_role_policy_attachment.route53[0]') is removed. This policy attachment was later integrated into the EKS module by simply adding it to an existing policy list. + +### Step 3: Migrate Nodes +Following the migration of the EKS cluster, the nodes' state is yet to be segregated. The migrate_nodes_state function is responsible for this. It uses a list of resource definitions (nodes_resource_definitions) as a filter to selectively pull out node-related resources from the EKS module state, and then transfer them to the separate nodes module. + +## Functions: +* **migrate_cluster_state**: Migrates the EKS cluster configuration. +* **migrate_infra_state**: Migrates infrastructure components and removes the route53 IAM role policy attachment. +* **migrate_nodes_state**: Migrates the nodes using a predefined list of resources. +* **copy_files**: Copies essential files, such as private keys. +* **adjust_vars**: Adjusts Terraform variables if necessary. +* **refresh_all**: Refreshes all new Terraform states to ensure they are up-to-date. +* **cleanup**: Deletes any backup files created during the state migration. +* **migrate_all**: A wrapper function to execute the state migration functions. + +## Important Note: +Always maintain backups of your original Terraform states before initiating any state migration. There's a potential risk of corruption or data loss during migration. In case of issues, the backup ensures that you can revert to the original state. diff --git a/bin/state-migration/migrate-states.sh b/bin/state-migration/migrate-states.sh new file mode 100755 index 00000000..2f9dd2c2 --- /dev/null +++ b/bin/state-migration/migrate-states.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +set -euo pipefail + +validate_vars() { + local -n input_vars=$1 + local -n mod_vars=$2 + error="false" + + echo "Validating vars" + + for var in "${input_vars[@]}"; do + if [ -z "${!var// /}" ]; then + echo "Error: $var is expected to be set by the user and its not set or is empty." + error="true" + fi + done + + for var in "${mod_vars[@]}"; do + if [ -z "${!var// /}" ]; then + echo "Error: $var is expected to be sourced from ${DEPLOY_DIR}/meta.sh and its not set or is empty." + error="true" + fi + done + + if test "$error" == "true"; then + return 1 + else + return 0 + fi +} + +migrate_cluster_state() { + + echo "Migrating EKS module state" + mkdir -p "$CLUSTER_DIR" + terraform -chdir="$CLUSTER_DIR" init --reconfigure --upgrade + + terraform state mv \ + -state="$LEGACY_STATE" \ + -state-out="$CLUSTER_STATE" \ + "${MOD_NAME}.module.eks" module.eks + + ls "$CLUSTER_DIR" +} + +migrate_infra_state() { + + echo "Migrating infra state" + mkdir -p "$INFRA_DIR" + terraform -chdir="$INFRA_DIR" init --reconfigure --upgrade + + terraform state mv \ + -state="$LEGACY_STATE" \ + -state-out="$INFRA_STATE" \ + "${MOD_NAME}" module.infra + + terraform state rm -state="$INFRA_STATE" 'module.infra.aws_iam_role_policy_attachment.route53[0]' + + ls "$INFRA_DIR" +} + +migrate_nodes_state() { + + declare -a nodes_resource_definitions=( + 'aws_autoscaling_group_tag.tag' + 'aws_eks_addon.this' + 'aws_eks_node_group.node_groups' + 'aws_launch_template.node_groups' + 'terraform_data.calico_setup' + ) + + declare -a nodes_resources=() + + for resource in "${nodes_resource_definitions[@]}"; do + mapfile -t tmp_array < <(terraform state list -state="$CLUSTER_STATE" | grep "$resource") + nodes_resources+=("${tmp_array[@]}") + done + + echo "Migrating nodes state" + + mkdir -p "$NODES_DIR" + + terraform -chdir="$NODES_DIR" init --reconfigure --upgrade + + for resource in "${nodes_resources[@]}"; do + echo "Migrating nodes resource: $resource" + + terraform state mv \ + -state="$CLUSTER_STATE" \ + -state-out="$NODES_STATE" \ + "$resource" "module.nodes.${resource#module.eks.}" + done +} + +copy_ssh_key() { + echo "Copying pvt ssh key" + cp "$LEGACY_PVT_KEY" "$PVT_KEY" && + { echo "Regenerating pub key" && ssh-keygen -y -f "$PVT_KEY" >"${PVT_KEY}.pub"; } +} + +refresh_all() { + bash "${DEPLOY_DIR}/tf.sh" all refresh +} + +cleanup() { + echo "Deleting terraform state backup files" + find . -type f -name "*.tfstate.*.backup*" -print0 -delete +} + +migrate_all() { + migrate_cluster_state + migrate_infra_state + migrate_nodes_state + +} + +#Set DEPLOY_DIR + +if [ -z "${DEPLOY_DIR// /}" ]; then + echo "DEPLOY_DIR is not set" + exit 1 +fi + +source "${DEPLOY_DIR}/meta.sh" || { echo "${DEPLOY_DIR}/meta.sh is not present" && exit 1; } + +# For CI purposes only `MOD_NAME` must be set by user. +[ "$CI_DEPLOY" == "true" ] && MOD_NAME="module.$(jq -r '.module[0] | keys[0]' ${LEGACY_DIR}/main.tf.json)" + +declare -a required_input_vars=( + "MOD_NAME" + "PVT_KEY" + "LEGACY_DIR" + "LEGACY_PVT_KEY" + "LEGACY_STATE" +) + +declare -a required_module_vars=( + "CLUSTER_DIR" + "CLUSTER_STATE" + "INFRA_DIR" + "INFRA_STATE" + "NODES_DIR" + "NODES_STATE" +) + +if ! validate_vars required_input_vars required_module_vars; then + echo "Variable validation failed." + exit 1 +fi + +echo "Running state migration !!!" + +migrate_all +copy_ssh_key +refresh_all +cleanup + +echo "State migration completed successfully !!!" && touch "migrated.txt" diff --git a/examples/bring-your-vpc/README.md b/examples/bring-your-vpc/README.md deleted file mode 100644 index a0e98809..00000000 --- a/examples/bring-your-vpc/README.md +++ /dev/null @@ -1,74 +0,0 @@ -# Create EKS in existing VPC using existing subnets - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: - -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### The following skips Network/VPC assets and uses the provided instead. -```hcl - network = { - vpc = { - id = "Existing vpc id" - subnets = { - private = "List of existing private subnets ids" - public = "List of existing public subnets ids" - pod = "List of existing subnets ids for pod networking" - } - } - } -``` - -### A separate subnet is recomended for large scale deployments to mitigate IP address starvation. If you dont want to provide separate subnets for pod networking, set `network.use_pod_cidr: false` and ommit the `network.vpc.subnets.pod` value. - -```hcl - network = { - vpc = { - id = "Existing vpc id" - subnets = { - private = "List of existing private subnets ids" - public = "List of existing public subnets ids" - } - } - use_pod_cidr = false - } -``` - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/bring-your-vpc/main.tf b/examples/bring-your-vpc/main.tf deleted file mode 100644 index ff1d0385..00000000 --- a/examples/bring-your-vpc/main.tf +++ /dev/null @@ -1,34 +0,0 @@ - -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks001" - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - - bastion = { - enabled = true - } - network = { - ## The following are not real values - vpc = { - id = "vpc-notrealb8ca349af" - subnets = { - private = ["subnet-notrealvalb2319f", "subnet-notrealval9be2580"] - public = ["subnet-notrealval126cd0", "subnet-notrealval178f224"] - pod = ["subnet-notrealval126cgf4", "subnet-notrealval178f64"] - } - } - use_pod_cidr = false - } -} diff --git a/examples/bring-your-vpc/outputs.tf b/examples/bring-your-vpc/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/bring-your-vpc/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/bring-your-vpc/variables.tf b/examples/bring-your-vpc/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/bring-your-vpc/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/custom-node-pool/README.md b/examples/custom-node-pool/README.md deleted file mode 100644 index 93dcd69f..00000000 --- a/examples/custom-node-pool/README.md +++ /dev/null @@ -1,71 +0,0 @@ - -# Create additional EKS node_group. - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### Create an additional node_group. - -```hcl - additional_node_groups = { - custom-group-0 = { - instance_types = [ - "m5.2xlarge" - ], - min_per_az = 0, - max_per_az = 10, - desired_per_az = 0, - availability_zone_ids = [ - "usw2-az1", - "usw2-az2" - ], - labels = { - "dominodatalab.com/node-pool" = "custom-group-0" - }, - volume = { - size = 100, - type = "gp3" - } - } - } -``` - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/custom-node-pool/main.tf b/examples/custom-node-pool/main.tf deleted file mode 100644 index 47982d5a..00000000 --- a/examples/custom-node-pool/main.tf +++ /dev/null @@ -1,40 +0,0 @@ -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks002" - bastion = {} - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - additional_node_groups = { - custom-group-0 = { - instance_types = [ - "m5.2xlarge" - ], - min_per_az = 0, - max_per_az = 10, - desired_per_az = 0, - availability_zone_ids = [ - "usw2-az1", - "usw2-az2" - ], - labels = { - "dominodatalab.com/node-pool" = "custom-group-0" - }, - volume = { - size = 100, - type = "gp3" - } - } - } - -} diff --git a/examples/custom-node-pool/outputs.tf b/examples/custom-node-pool/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/custom-node-pool/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/custom-node-pool/variables.tf b/examples/custom-node-pool/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/custom-node-pool/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/deploy/README.md b/examples/deploy/README.md new file mode 100644 index 00000000..4ea9cefe --- /dev/null +++ b/examples/deploy/README.md @@ -0,0 +1,165 @@ +# Terraform Multi-Module Management + +## Overview +The `tf.sh` script provides a convenient method to manage multiple Terraform configurations for various components of a system. The primary modules managed by this script include `infra`, `cluster`, and `nodes`. These components might represent different layers of an infrastructure deployment. Similarly the `set-mod-version.sh` script helps to set the source module version on all three modules(`infra`, `cluster`, and `nodes`), see [README](../../README.md#Using_script). + +## Pre-requisites +* Ensure that `terraform` is installed and accessible in your path. +* Ensure that `jq`, a command-line JSON processor, is installed. + +## Directory Structure +The script expects the following directory structure: +``` +deploy +├── README.md +├── meta.sh +├── set-mod-version.sh +├── terraform +│   ├── cluster +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   ├── cluster.tfvars +│   ├── infra +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   ├── infra.tfvars +│   ├── nodes +│   │   ├── README.md +│   │   ├── main.tf +│   │   ├── outputs.tf +│   │   └── variables.tf +│   └── nodes.tfvars +└── tf.sh +``` + +* Each subdirectory under `terraform` (e.g., `infra`, `cluster`, `nodes`) should contain its respective Terraform configurations. +* Each component is expected to have a corresponding `.tfvars` file at the `terraform` directory. For instance, for the `infra` component, there should be an `terraform/infra.tfvars` file. +* Each of component's state and output(when the `output` command is invoked) is saved in the `terraform` directory: + +```bash +└─ deploy/terraform +   ├── cluster.outputs +   ├── cluster.tfstate +   ├── infra.outputs +   ├── infra.tfstate +   ├── nodes.outputs +   └── nodes.tfstate +``` +## Variables structure + +See [README](../../README.md#3-review-and-configure-tfvars) + +## Usage + +To use the script, invoke it with the desired command and component: + +```bash +./tf.sh +``` + +* **component**: The component parameter refers to the specific section of your architecture that you wish to target with a command. Supported components include `infra`, `cluster`, `nodes`, and `all`. Selecting all will execute the command across `infra`, `cluster` and `nodes`. + The script uses the component parameter to identify corresponding Terraform directories and to name both the Terraform variables file (`terraform/${component}.tfvars`) and the Terraform state file (`terraform/${component}.tfstate`). If you create a custom folder named `mydir` that includes your Terraform configuration, setup a terraform variables file(`terraform/mydir.tfstate`), and state file(`terraform/mydir.tfstate`) if existing, then you can utilize the tf.sh script to execute Terraform commands. For example, running `./tf.sh mydir plan`. + + It's important to note that your custom directory, mydir, ***will not*** be included when using the `all` value for components. + +* **command**: Supported commands include: + * `init`: Initializes the Terraform configurations. + * `plan`: Shows the execution plan of Terraform. + * `apply`: Applies the Terraform configurations. + * `destroy`: Destroys the Terraform resources. + * `output`: Shows the output values of your configurations. + * `output` ``: Shows a specific output value. + * `refresh`: Refreshes the Terraform state file. + * `plan_out`: Generates a plan and writes it to `terraform/${component}-terraform.plan`. + * `apply_plan`: Applies plan located at `terraform/${component}-terraform.plan`. + * `roll_nodes`: (Rollout nodes)Runs apply one at a time on `aws_eks_node_group.node_groups` resources. + +## Examples + +* To preview the execution plan of the cluster: + +```bash +./tf.sh cluster plan +``` + +* To create all components: + +```bash +./tf.sh all apply +``` + +* To destroy all components: + +```bash +./tf.sh all destroy +``` + +* To perform a plan and write it to a file(the plan file will be stored at: `terraform/${component}-terraform.plan`): + +```bash +./tf.sh cluster plan_out +``` + +* To apply a a previously generated plan stored at `terraform/${component}-terraform.plan` for this example `terraform/cluster-terraform.plan`: + +```bash +./tf.sh cluster apply_plan +``` + +## Common Operations + +For some frequently performed operations, follow the steps outlined below: + +### Initial install +See the repo's [README](../../README.md#bootstrap-module) for how to bootstrap the module. + +### Updating the modules' version +See `README` [Update modules version](../../README.md#2-update-modules-version) + +### Kubernetes Upgrade: +In order to update Kubernetes we will need to update the `cluster` and the `nodes`. + +1. Set the `eks.k8s_version` variable to desired version(At most it can be 2 minor versions ahead.) +2. Update cluster: + 1. Plan and review the changes: + ```bash + ./tf.sh cluster plan + ``` + 2. Apply the changes: + ```bash + ./tf.sh cluster apply + ``` +3. Update nodes: +Given that the nodes source the k8s version from `eks` we just need to plan and apply. + 1. Plan and review the changes: + ```bash + ./tf.sh nodes plan + ``` + 2. Apply the changes: + ```bash + ./tf.sh nodes apply + ``` + +### Nodes Upgrade: +Given that the nodes module looks for the latest AMI we just need to plan and apply: + +1. Plan and review the changes: + +```bash +./tf.sh nodes plan +``` + +2. :warning: In case of large amount of `node_groups` or if just want to update one `node_group` at a time. + +```bash +./tf.sh nodes roll_nodes +``` +Otherwise to update all `node_groups` in parallel. + +```bash +./tf.sh nodes apply +``` diff --git a/examples/deploy/meta.sh b/examples/deploy/meta.sh new file mode 100644 index 00000000..b3f5a838 --- /dev/null +++ b/examples/deploy/meta.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +BASE_TF_DIR="${SH_DIR}/terraform" +declare -a MOD_DIRS=( + "${BASE_TF_DIR}/infra" + "${BASE_TF_DIR}/cluster" + "${BASE_TF_DIR}/nodes" +) + +INFRA_DIR="${MOD_DIRS[0]}" +CLUSTER_DIR="${MOD_DIRS[1]}" +NODES_DIR="${MOD_DIRS[2]}" + +CLUSTER_STATE="${BASE_TF_DIR}/cluster.tfstate" +NODES_STATE="${BASE_TF_DIR}/nodes.tfstate" +INFRA_STATE="${BASE_TF_DIR}/infra.tfstate" + +CLUSTER_VARS="${BASE_TF_DIR}/cluster.tfvars" +NODES_VARS="${BASE_TF_DIR}/nodes.tfvars" +INFRA_VARS="${BASE_TF_DIR}/infra.tfvars" + +export BASE_TF_DIR \ + MOD_DIRS \ + INFRA_DIR \ + CLUSTER_DIR \ + NODES_DIR \ + CLUSTER_STATE \ + NODES_STATE \ + INFRA_STATE \ + CLUSTER_VARS \ + NODES_VARS \ + INFRA_VARS diff --git a/examples/deploy/set-mod-version.sh b/examples/deploy/set-mod-version.sh new file mode 100755 index 00000000..ec48e21c --- /dev/null +++ b/examples/deploy/set-mod-version.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +set -euo pipefail + +validate_mod_version() { + [[ -n "${MOD_VALIDATION_OFF:-}" ]] && { + echo 'MOD_VALIDATION_OFF is set, skipping module version validation' + return + } + + url="https://api.github.com/repos/dominodatalab/terraform-aws-eks/tags" + + local curl_cmd=("curl" "-s") + + # In case of rate limiting and a `GITHUB_TOKEN` is available + [[ -n "${GITHUB_TOKEN:-}" ]] && curl_cmd+=("-H" "Authorization: token ${GITHUB_TOKEN}") + + while [[ -n "${url:-}" ]]; do + + response=$("${curl_cmd[@]}" -I "$url") + if [[ $? -ne 0 ]]; then + echo "Error fetching tags from $url" + exit 1 + fi + + local tag_array=() + mapfile -t tag_array <<<$("${curl_cmd[@]}" "$url" | jq -r '.[].name | select(test("^v\\d+\\.\\d+\\.\\d+$"))') + + for tag in "${tag_array[@]}"; do + [[ "$MOD_VERSION" == "$tag" ]] && return + done + url=$(echo "$response" | grep -i 'rel="next"' | sed -n 's/.*<\([^>]*\)>; rel="next".*/\1/p' || echo "") + + done + + echo "Error: The MOD_VERSION $MOD_VERSION is not a suitable tag for the modules source." + exit 1 +} + +set_module_version() { + for dir in "${MOD_DIRS[@]}"; do + file="${dir}/main.tf" + echo "Setting module source on: $file" + name=$(basename "$dir") + if [ $name == "cluster" ]; then + name="eks" + fi + hcledit attribute set "module.${name}.source" \"github.com/dominodatalab/terraform-aws-eks.git//modules/"${name}"?ref="${MOD_VERSION}"\" -f "$file" --update + done + +} + +MOD_VERSION="$1" +[ -z "${MOD_VERSION// /}" ] && { echo "Provide a module version in the format $(vX.X.X), ie $(v3.0.0)" && exit 1; } + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +source "${SH_DIR}/meta.sh" + +validate_mod_version +set_module_version diff --git a/examples/deploy/terraform/cluster.tfvars b/examples/deploy/terraform/cluster.tfvars new file mode 100644 index 00000000..6bca19ea --- /dev/null +++ b/examples/deploy/terraform/cluster.tfvars @@ -0,0 +1,2 @@ +eks = null +kms_info = null diff --git a/examples/deploy/terraform/cluster/README.md b/examples/deploy/terraform/cluster/README.md new file mode 100644 index 00000000..c2bccb40 --- /dev/null +++ b/examples/deploy/terraform/cluster/README.md @@ -0,0 +1,42 @@ +# eks + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | >= 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ./../../../../modules/eks | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_remote_state.infra](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [eks](#input\_eks) | creation\_role\_name = Name of the role to import.
k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} | 
object({
    creation_role_name = optional(string, null)
    k8s_version        = optional(string)
    kubeconfig = optional(object({
      extra_args = optional(string)
      path       = optional(string)
    }), {})
    public_access = optional(object({
      enabled = optional(bool)
      cidrs   = optional(list(string))
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })))
    master_role_names  = optional(list(string))
    cluster_addons     = optional(list(string))
    ssm_log_group_name = optional(string)
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string)
      groups_prefix                 = optional(string)
      identity_provider_config_name = string
      issuer_url                    = optional(string)
      required_claims               = optional(string)
      username_claim                = optional(string)
      username_prefix               = optional(string)
    })))
  })
| `null` | no | +| [kms\_info](#input\_kms\_info) | Overrides the KMS key information. Meant for migrated configurations.
{
key\_id = KMS key id.
key\_arn = KMS key arn.
enabled = KMS key is enabled.
} | 
object({
    key_id  = string
    key_arn = string
    enabled = bool
  })
| `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [eks](#output\_eks) | EKS details. | +| [infra](#output\_infra) | Infra details. | + diff --git a/examples/deploy/terraform/cluster/main.tf b/examples/deploy/terraform/cluster/main.tf new file mode 100644 index 00000000..3a0164cf --- /dev/null +++ b/examples/deploy/terraform/cluster/main.tf @@ -0,0 +1,47 @@ +data "terraform_remote_state" "infra" { + backend = "local" + + config = { + path = "${path.module}/../infra.tfstate" + } +} + +locals { + infra = data.terraform_remote_state.infra.outputs.infra + eks = var.eks != null ? var.eks : local.infra.eks + kms = var.kms_info != null ? var.kms_info : local.infra.kms +} + +module "eks" { + source = "./../../../../modules/eks" + deploy_id = local.infra.deploy_id + region = local.infra.region + + ssh_key = local.infra.ssh_key + node_iam_policies = local.infra.node_iam_policies + efs_security_group = local.infra.efs_security_group + eks = local.eks + network_info = local.infra.network + kms_info = local.kms + bastion_info = local.infra.bastion + create_eks_role_arn = local.infra.create_eks_role_arn + tags = local.infra.tags + +} + +provider "aws" { + region = var.region + default_tags { + tags = var.tags + } +} + +terraform { + required_version = ">= 1.4.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = ">= 4.0" + } + } +} diff --git a/examples/deploy/terraform/cluster/outputs.tf b/examples/deploy/terraform/cluster/outputs.tf new file mode 100644 index 00000000..35488e58 --- /dev/null +++ b/examples/deploy/terraform/cluster/outputs.tf @@ -0,0 +1,9 @@ +output "infra" { + description = "Infra details." + value = local.infra +} + +output "eks" { + description = "EKS details." + value = module.eks.info +} diff --git a/examples/deploy/terraform/cluster/variables.tf b/examples/deploy/terraform/cluster/variables.tf new file mode 100644 index 00000000..d643dd67 --- /dev/null +++ b/examples/deploy/terraform/cluster/variables.tf @@ -0,0 +1,81 @@ +## Used to overwrite the `eks` variable passed through the `infra` outputs. + +variable "eks" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | ~> 5.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [infra](#module\_infra) | ./../../../../modules/infra | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string)
    bootstrap_extra_args  = optional(string)
    instance_types        = list(string)
    spot                  = optional(bool)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })))
    tags = optional(map(string))
    gpu  = optional(bool)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | +| [bastion](#input\_bastion) | enabled = Create bastion host.
ami = Ami id. Defaults to latest 'amazon\_linux\_2' ami.
instance\_type = Instance type.
authorized\_ssh\_ip\_ranges = List of CIDR ranges permitted for the bastion ssh access.
username = Bastion user.
install\_binaries = Toggle to install required Domino binaries in the bastion. | 
object({
    enabled                  = optional(bool)
    ami_id                   = optional(string)
    instance_type            = optional(string)
    authorized_ssh_ip_ranges = optional(list(string))
    username                 = optional(string)
    install_binaries         = optional(bool)
  })
| n/a | yes | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string))
          gpu  = optional(bool)
          volume = optional(object({
            size = optional(number)
            type = optional(string)
          }))
      })
  })
| n/a | yes | +| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | n/a | yes | +| [eks](#input\_eks) | creation\_role\_name = Name of the role to import.
k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} | 
object({
    creation_role_name = optional(string, null)
    k8s_version        = optional(string)
    kubeconfig = optional(object({
      extra_args = optional(string)
      path       = optional(string)
    }), {})
    public_access = optional(object({
      enabled = optional(bool)
      cidrs   = optional(list(string))
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })))
    master_role_names  = optional(list(string))
    cluster_addons     = optional(list(string))
    ssm_log_group_name = optional(string)
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string)
      groups_prefix                 = optional(string)
      identity_provider_config_name = string
      issuer_url                    = optional(string)
      required_claims               = optional(string)
      username_claim                = optional(string)
      username_prefix               = optional(string)
    })))
  })
| `{}` | no | +| [kms](#input\_kms) | enabled = Toggle,if set use either the specified KMS key\_id or a Domino-generated one.
key\_id = optional(string, null) | 
object({
    enabled = optional(bool)
    key_id  = optional(string)
  })
| n/a | yes | +| [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. | 
object({
    vpc = optional(object({
      id = optional(string, null)
      subnets = optional(object({
        private = optional(list(string), [])
        public  = optional(list(string), [])
        pod     = optional(list(string), [])
      }), {})
    }), {})
    network_bits = optional(object({
      public  = optional(number, 27)
      private = optional(number, 19)
      pod     = optional(number, 19)
      }
    ), {})
    cidrs = optional(object({
      vpc = optional(string, "10.0.0.0/16")
      pod = optional(string, "100.64.0.0/16")
    }), {})
    use_pod_cidr = optional(bool, true)
  })
| `{}` | no | +| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | +| [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNS zone. | `string` | `null` | no | +| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | +| [storage](#input\_storage) | storage = {
efs = {
access\_point\_path = Filesystem path for efs.
backup\_vault = {
create = Create backup vault for EFS toggle.
force\_destroy = Toggle to allow automatic destruction of all backups when destroying.
backup = {
schedule = Cron-style schedule for EFS backup vault (default: once a day at 12pm).
cold\_storage\_after = Move backup data to cold storage after this many days.
delete\_after = Delete backup data after this many days.
}
}
}
s3 = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets.
}
ecr = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the ECR repositories. if 'false' terraform will NOT be able to delete non-empty repositories.
}
}
} | 
object({
    efs = optional(object({
      access_point_path = optional(string, "/domino")
      backup_vault = optional(object({
        create        = optional(bool, true)
        force_destroy = optional(bool, true)
        backup = optional(object({
          schedule           = optional(string, "0 12 * * ? *")
          cold_storage_after = optional(number, 35)
          delete_after       = optional(number, 125)
        }), {})
      }), {})
    }), {})
    s3 = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
    ecr = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [domino\_config\_values](#output\_domino\_config\_values) | Values used to update the `domino.yml` for installation. | +| [infra](#output\_infra) | Infrastructure outputs. | + diff --git a/examples/deploy/terraform/infra/main.tf b/examples/deploy/terraform/infra/main.tf new file mode 100644 index 00000000..ee00c585 --- /dev/null +++ b/examples/deploy/terraform/infra/main.tf @@ -0,0 +1,32 @@ +module "infra" { + source = "./../../../../modules/infra" + + deploy_id = var.deploy_id + additional_node_groups = var.additional_node_groups + bastion = var.bastion + default_node_groups = var.default_node_groups + + network = var.network + eks = var.eks + kms = var.kms + storage = var.storage + region = var.region + route53_hosted_zone_name = var.route53_hosted_zone_name + ssh_pvt_key_path = var.ssh_pvt_key_path + tags = var.tags +} + + +provider "aws" { + region = var.region +} + +terraform { + required_version = ">= 1.4.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/examples/deploy/terraform/infra/outputs.tf b/examples/deploy/terraform/infra/outputs.tf new file mode 100644 index 00000000..a9c8eb47 --- /dev/null +++ b/examples/deploy/terraform/infra/outputs.tf @@ -0,0 +1,62 @@ +output "infra" { + description = "Infrastructure outputs." + value = module.infra +} + +output "domino_config_values" { + description = "Values used to update the `domino.yml` for installation." + value = { + name = var.deploy_id + autoscaler = { + auto_discovery = { + cluster_name = var.deploy_id + } + aws = { + region = var.region + } + } + internal_docker_registry = { + s3_override = { + region = var.region + bucket = module.infra.storage.s3.buckets.registry.bucket_name + sse_kms_key_id = module.infra.kms.key_arn + } + } + storage_classes = { + block = { + parameters = { + kmsKeyId = module.infra.kms.key_arn + } + } + shared = { + efs = { + region = var.region + filesystem_id = module.infra.storage.efs.file_system.id + access_point_id = module.infra.storage.efs.access_point.id + } + } + blob_storage = { + projects = { + region = var.region + bucket = module.infra.storage.s3.buckets.blobs.bucket_name + sse_kms_key_id = module.infra.kms.key_arn + } + logs = { + region = var.region + bucket = module.infra.storage.s3.buckets.logs.bucket_name + sse_kms_key_id = module.infra.kms.key_arn + + } + backups = { + region = var.region + bucket = module.infra.storage.s3.buckets.backups.bucket_name + sse_kms_key_id = module.infra.kms.key_arn + } + monitoring = { + region = var.region + bucket = module.infra.storage.s3.buckets.monitoring.bucket_name + } + } + } + } +} diff --git a/examples/deploy/terraform/infra/variables.tf b/examples/deploy/terraform/infra/variables.tf new file mode 100644 index 00000000..afb1b520 --- /dev/null +++ b/examples/deploy/terraform/infra/variables.tf @@ -0,0 +1,340 @@ +variable "deploy_id" { + description = "Domino Deployment ID." + type = string +} + +variable "region" { + description = "AWS region for the deployment" + type = string +} + +variable "tags" { + description = "Deployment tags." + type = map(string) +} + +variable "network" { + description = < 32 IPs. + private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. + pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. + } + cidrs = { + vpc = The IPv4 CIDR block for the VPC. + pod = The IPv4 CIDR block for the Pod subnets. + } + use_pod_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. + EOF + + type = object({ + vpc = optional(object({ + id = optional(string, null) + subnets = optional(object({ + private = optional(list(string), []) + public = optional(list(string), []) + pod = optional(list(string), []) + }), {}) + }), {}) + network_bits = optional(object({ + public = optional(number, 27) + private = optional(number, 19) + pod = optional(number, 19) + } + ), {}) + cidrs = optional(object({ + vpc = optional(string, "10.0.0.0/16") + pod = optional(string, "100.64.0.0/16") + }), {}) + use_pod_cidr = optional(bool, true) + }) + + default = {} +} + +variable "default_node_groups" { + description = "EKS managed node groups definition." + type = object( + { + compute = object( + { + ami = optional(string, null) + bootstrap_extra_args = optional(string, "") + instance_types = optional(list(string), ["m5.2xlarge"]) + spot = optional(bool, false) + min_per_az = optional(number, 0) + max_per_az = optional(number, 10) + desired_per_az = optional(number, 0) + availability_zone_ids = list(string) + labels = optional(map(string), { + "dominodatalab.com/node-pool" = "default" + }) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + })), []) + tags = optional(map(string), {}) + gpu = optional(bool, null) + volume = optional(object({ + size = optional(number, 1000) + type = optional(string, "gp3") + }), { + size = 1000 + type = "gp3" + } + ) + }), + platform = object( + { + ami = optional(string, null) + bootstrap_extra_args = optional(string, "") + instance_types = optional(list(string), ["m5.2xlarge"]) + spot = optional(bool, false) + min_per_az = optional(number, 1) + max_per_az = optional(number, 10) + desired_per_az = optional(number, 1) + availability_zone_ids = list(string) + labels = optional(map(string), { + "dominodatalab.com/node-pool" = "platform" + }) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + })), []) + tags = optional(map(string), {}) + gpu = optional(bool, null) + volume = optional(object({ + size = optional(number, 100) + type = optional(string, "gp3") + }), { + size = 100 + type = "gp3" + } + ) + }), + gpu = object( + { + ami = optional(string, null) + bootstrap_extra_args = optional(string, "") + instance_types = optional(list(string), ["g4dn.xlarge"]) + spot = optional(bool, false) + min_per_az = optional(number, 0) + max_per_az = optional(number, 10) + desired_per_az = optional(number, 0) + availability_zone_ids = list(string) + labels = optional(map(string), { + "dominodatalab.com/node-pool" = "default-gpu" + "nvidia.com/gpu" = true + }) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + })), [{ + key = "nvidia.com/gpu" + value = "true" + effect = "NO_SCHEDULE" + } + ]) + tags = optional(map(string)) + gpu = optional(bool) + volume = optional(object({ + size = optional(number) + type = optional(string) + })) + }) + }) +} + +variable "additional_node_groups" { + description = "Additional EKS managed node groups definition." + type = map(object({ + ami = optional(string) + bootstrap_extra_args = optional(string) + instance_types = list(string) + spot = optional(bool) + min_per_az = number + max_per_az = number + desired_per_az = number + availability_zone_ids = list(string) + labels = map(string) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + }))) + tags = optional(map(string)) + gpu = optional(bool) + volume = object({ + size = string + type = string + }) + })) + + default = {} +} + +variable "storage" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [terraform](#provider\_terraform) | n/a | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [nodes](#module\_nodes) | ./../../../../modules/nodes | n/a | + +## Resources + +| Name | Type | +|------|------| +| [terraform_remote_state.eks](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | +| [terraform_remote_state.infra](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string)
    bootstrap_extra_args  = optional(string)
    instance_types        = list(string)
    spot                  = optional(bool)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })))
    tags = optional(map(string), {})
    gpu  = optional(bool)
    volume = object({
      size = string
      type = string
    })
  }))
| `null` | no | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string)
          bootstrap_extra_args  = optional(string)
          instance_types        = optional(list(string))
          spot                  = optional(bool)
          min_per_az            = optional(number)
          max_per_az            = optional(number)
          desired_per_az        = optional(number)
          availability_zone_ids = list(string)
          labels                = optional(map(string))
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })))
          tags = optional(map(string))
          gpu  = optional(bool)
          volume = optional(object({
            size = optional(number)
            type = optional(string)
            })
          )
      }),
      platform = object(
        {
          ami                   = optional(string)
          bootstrap_extra_args  = optional(string)
          instance_types        = optional(list(string))
          spot                  = optional(bool)
          min_per_az            = optional(number)
          max_per_az            = optional(number)
          desired_per_az        = optional(number)
          availability_zone_ids = list(string)
          labels                = optional(map(string))
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })))
          tags = optional(map(string))
          gpu  = optional(bool)
          volume = optional(object({
            size = optional(number)
            type = optional(string)
          }))
      }),
      gpu = object(
        {
          ami                   = optional(string)
          bootstrap_extra_args  = optional(string)
          instance_types        = optional(list(string))
          spot                  = optional(bool)
          min_per_az            = optional(number)
          max_per_az            = optional(number)
          desired_per_az        = optional(number)
          availability_zone_ids = list(string)
          labels                = optional(map(string))
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })))
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number)
            type = optional(string)
          }))
      })
  })
| `null` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [info](#output\_info) | Nodes details. | + diff --git a/examples/deploy/terraform/nodes/main.tf b/examples/deploy/terraform/nodes/main.tf new file mode 100644 index 00000000..325a283d --- /dev/null +++ b/examples/deploy/terraform/nodes/main.tf @@ -0,0 +1,42 @@ +data "terraform_remote_state" "infra" { + backend = "local" + + config = { + path = "${path.module}/../infra.tfstate" + } +} + +data "terraform_remote_state" "eks" { + backend = "local" + + config = { + path = "${path.module}/../cluster.tfstate" + } +} + + + +locals { + infra = data.terraform_remote_state.infra.outputs.infra + eks = data.terraform_remote_state.eks.outputs.eks + default_node_groups = var.default_node_groups != null ? var.default_node_groups : local.infra.default_node_groups + additional_node_groups = var.additional_node_groups != null ? var.additional_node_groups : local.infra.additional_node_groups + +} + +module "nodes" { + source = "./../../../../modules/nodes" + region = local.infra.region + + ssh_key = local.infra.ssh_key + default_node_groups = local.default_node_groups + additional_node_groups = local.additional_node_groups + eks_info = local.eks + network_info = local.infra.network + kms_info = local.infra.kms + tags = local.infra.tags +} + +terraform { + required_version = ">= 1.4.0" +} diff --git a/examples/deploy/terraform/nodes/outputs.tf b/examples/deploy/terraform/nodes/outputs.tf new file mode 100644 index 00000000..e8edc31a --- /dev/null +++ b/examples/deploy/terraform/nodes/outputs.tf @@ -0,0 +1,4 @@ +output "info" { + description = "Nodes details." + value = module.nodes.info +} diff --git a/examples/deploy/terraform/nodes/variables.tf b/examples/deploy/terraform/nodes/variables.tf new file mode 100644 index 00000000..243736ea --- /dev/null +++ b/examples/deploy/terraform/nodes/variables.tf @@ -0,0 +1,106 @@ +## Used to overwrite the `default_node_groups` and `additional_node_groups` variables passed through the `infra` outputs. + +variable "default_node_groups" { + description = "EKS managed node groups definition." + type = object( + { + compute = object( + { + ami = optional(string) + bootstrap_extra_args = optional(string) + instance_types = optional(list(string)) + spot = optional(bool) + min_per_az = optional(number) + max_per_az = optional(number) + desired_per_az = optional(number) + availability_zone_ids = list(string) + labels = optional(map(string)) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + }))) + tags = optional(map(string)) + gpu = optional(bool) + volume = optional(object({ + size = optional(number) + type = optional(string) + }) + ) + }), + platform = object( + { + ami = optional(string) + bootstrap_extra_args = optional(string) + instance_types = optional(list(string)) + spot = optional(bool) + min_per_az = optional(number) + max_per_az = optional(number) + desired_per_az = optional(number) + availability_zone_ids = list(string) + labels = optional(map(string)) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + }))) + tags = optional(map(string)) + gpu = optional(bool) + volume = optional(object({ + size = optional(number) + type = optional(string) + })) + }), + gpu = object( + { + ami = optional(string) + bootstrap_extra_args = optional(string) + instance_types = optional(list(string)) + spot = optional(bool) + min_per_az = optional(number) + max_per_az = optional(number) + desired_per_az = optional(number) + availability_zone_ids = list(string) + labels = optional(map(string)) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + }))) + tags = optional(map(string), {}) + gpu = optional(bool, null) + volume = optional(object({ + size = optional(number) + type = optional(string) + })) + }) + }) + default = null +} + +variable "additional_node_groups" { + description = "Additional EKS managed node groups definition." + type = map(object({ + ami = optional(string) + bootstrap_extra_args = optional(string) + instance_types = list(string) + spot = optional(bool) + min_per_az = number + max_per_az = number + desired_per_az = number + availability_zone_ids = list(string) + labels = map(string) + taints = optional(list(object({ + key = string + value = optional(string) + effect = string + }))) + tags = optional(map(string), {}) + gpu = optional(bool) + volume = object({ + size = string + type = string + }) + })) + default = null +} diff --git a/examples/deploy/tf.sh b/examples/deploy/tf.sh new file mode 100755 index 00000000..bc1b01ed --- /dev/null +++ b/examples/deploy/tf.sh @@ -0,0 +1,238 @@ +#!/usr/bin/env bash +set -euo pipefail + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +source "${SH_DIR}/meta.sh" + +state_exists() { + local dir=$1 + local name=$(basename $dir) + local state_path="${BASE_TF_DIR}/${name}.tfstate" + [[ -f $state_path && -s $state_path ]] +} + +has_resources() { + local dir=$1 + local name=$(basename $dir) + local state_path="${BASE_TF_DIR}/${name}.tfstate" + + if [[ ! -f "$state_path" ]]; then + return 1 + fi + local resource_count=$(jq '.resources | length' "$state_path") + [[ $resource_count -gt 0 ]] +} + +check_dependencies() { + local name=$1 + if [[ $name == "cluster" ]]; then + if ! has_resources "${BASE_TF_DIR}/infra"; then + printf "\nERROR: Cannot plan/apply 'cluster' without 'infra' being provisioned !!!\n\n" + exit 1 + fi + elif [[ $name == "nodes" ]]; then + if ! has_resources "${BASE_TF_DIR}/infra" || ! has_resources "${BASE_TF_DIR}/cluster"; then + printf "\nERROR: Cannot plan/apply 'nodes' without 'infra' and 'cluster' being provisioned. being provisioned !!!\n\n" + exit 1 + fi + fi + return 0 +} + +run_tf_command() { + local dir="$1" + local cmd="$2" + + if [[ "$#" == 3 ]]; then + local param=$3 + else + local param='false' + fi + + local name=$(basename "$dir") + + local plan_path="${BASE_TF_DIR}/${name}-terraform.plan" + local state_path="${BASE_TF_DIR}/${name}.tfstate" + + local tfvars_file="${BASE_TF_DIR}/${name}.tfvars" + local tfvars_json_file="${tfvars_file}.json" + + # Skip .tfvars check for init and output commands + if [[ $cmd != "init" && $cmd != "output" ]]; then + if [[ -f "$tfvars_file" && -f "$tfvars_json_file" ]]; then + echo "ERROR: Both ${tfvars_file} and ${tfvars_json_file} exist. Please consolidate variables onto one of them and remove the other." + exit 1 + fi + + if [[ -s "$tfvars_json_file" ]]; then + tfvars_file="$tfvars_json_file" + fi + + if [[ ! -s "$tfvars_file" ]]; then + echo "ERROR: Neither ${tfvars_file} nor ${tfvars_json_file} exists or they are empty." + exit 1 + fi + fi + + update_node_groups() { + echo "Updating EKS Node Pools" + mapfile -t node_pools < <(terraform -chdir="$dir" state list -state="$state_path" | grep 'aws_eks_node_group.node_groups') + + if [[ ${#node_pools[@]} -eq 0 ]]; then + echo "Error: No node pools found to update." + exit 1 + fi + + for np in "${node_pools[@]}"; do + echo "Updating $np" + terraform -chdir="$dir" apply -input=false -state="$state_path" -target="$np" -auto-approve + done + } + + case $cmd in + init) + terraform -chdir="$dir" init + ;; + plan) + check_dependencies $name + terraform -chdir="$dir" plan -input=false -state="$state_path" -var-file="$tfvars_file" + ;; + plan_out) + check_dependencies $name + terraform -chdir="$dir" plan -input=false -state="$state_path" -var-file="$tfvars_file" -out="$plan_path" + echo "Terraform plan for $name saved at: $(realpath $plan_path)" + ;; + apply) + check_dependencies $name + terraform -chdir="$dir" apply -input=false -state="$state_path" -var-file="$tfvars_file" -auto-approve + ;; + apply_plan) + check_dependencies $name + if [ ! -f "$plan_path" ]; then + echo "$plan_path does not exist. Exiting..." + exit 1 + fi + terraform -chdir="$dir" apply -input=false -state="$state_path" -auto-approve "$plan_path" + ;; + refresh) + check_dependencies $name + terraform -chdir="$dir" apply -input=false -state="$state_path" -var-file="$tfvars_file" -refresh-only -auto-approve + ;; + show_plan_json) + check_dependencies $name + terraform -chdir="$dir" show -json "$plan_path" + ;; + validate) + terraform -chdir="$dir" validate + ;; + output) + if state_exists "$dir"; then + if [ $param != "false" ]; then + terraform -chdir="$dir" output -state="$state_path" "$param" + else + terraform -chdir="$dir" output -state="$state_path" | tee "${BASE_TF_DIR}/${name}.outputs" + fi + else + echo "No state found for $name" + fi + ;; + roll_nodes) + if [[ "$name" != "nodes" ]]; then + echo "Invalid component: ${name} .The 'roll_nodes' command is only applicable to 'nodes'" + exit 1 + fi + update_node_groups + ;; + destroy) + if state_exists "$dir" && has_resources "$dir"; then + terraform -chdir="$dir" destroy -input=false -state="$state_path" -var-file="$tfvars_file" -auto-approve || terraform -chdir="$dir" destroy -input=false -state="$state_path" -var-file="$tfvars_file" -auto-approve -refresh=false + else + echo "Nothing to destroy for $name" + fi + ;; + *) + echo "Unsupported command: $cmd" + exit 1 + ;; + esac +} + +usage() { + echo "Usage: ./tf.sh " + + echo -e "\nComponents:" + echo -e " infra \tManage the infrastructure components." + echo -e " cluster \tManage the cluster components." + echo -e " nodes \tManage the node components." + echo -e " all \tManage all components." + echo "Note: If an unlisted component is provided, the script will attempt to execute the given command, assuming the corresponding directory is properly configured." + + echo -e "\nCommands:" + echo -e " init \t\tInitialize the working directory." + echo -e " validate \t\tCheck the syntax and validate the configuration." + echo -e " plan \t\tGenerate an execution plan." + echo -e " plan_out \t\tGenerate a plan and save it to a file." + echo -e " apply \t\tExecute the actions proposed in the Terraform plan." + echo -e " refresh \t\tUpdate local state with remote resources." + echo -e " destroy \t\tDestroy the Terraform-managed infrastructure." + echo -e " output \t\tDisplay outputs from the Terraform state." + echo -e " output \tDisplay a specific output." + echo -e " roll_nodes \t\tUpdates the resources 'aws_eks_node_group.node_groups' one at a time." + echo "Note: This is suitable in case you do not want to update all nodes at the same time." +} + +# Check number of arguments +if [[ "$#" -ne 3 ]] && [[ "$#" -ne 2 ]]; then + usage + exit 1 +fi + +component=$1 +command=$2 + +if [[ "$#" -eq 3 ]]; then + if [[ "$command" != "output" ]]; then + usage + exit 1 + fi + param=$3 +else + param='false' +fi + +case $component in +infra | cluster | nodes) + if [[ "$param" != "false" ]]; then + run_tf_command "${BASE_TF_DIR}/${component}" "$command" "$param" + else + run_tf_command "${BASE_TF_DIR}/${component}" "$command" + fi + ;; +all) + if [[ "$command" == "destroy" ]]; then + for ((idx = ${#MOD_DIRS[@]} - 1; idx >= 0; idx--)); do + run_tf_command "${MOD_DIRS[idx]}" "$command" + done + else + for dir in "${MOD_DIRS[@]}"; do + run_tf_command "$dir" "$command" + done + fi + ;; +*) + echo "Default components: infra, cluster, nodes, all" + if [[ -d "${BASE_TF_DIR}/${component}" ]]; then + if ls "${BASE_TF_DIR}/${component}"/*.tf 1>/dev/null 2>&1; then + echo "Running command $command on non-default component: $component" + echo "Note: Component: all does not include $component" + run_tf_command "${BASE_TF_DIR}/${component}" "$command" + else + echo "Directory exists but no .tf files found in ${BASE_TF_DIR}/${component}" + exit 1 + fi + else + echo "Component: $component Not supported." + exit 1 + fi + ;; +esac diff --git a/examples/examples.pem b/examples/examples.pem deleted file mode 100644 index fe532075..00000000 --- a/examples/examples.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKgIBAAKCAgEAvjEcckID7VtWPTxhhhTjIRzVf1vAF+oTI6kYUt/b8mv7SSED -S5WiIA+XNwlNjUaGvDkwtwkdICFJE09XLUpdedM9POW9oFE2c7Qv+hO0+60YAEUg -WOJfE1ZV3cVuQhV54Djl/jXFCppK629SU//r0NyrrRQDmoYojdl6e5FbeYAd8I+R -yc6pfZZZ1tYa+z+bYF+BHfyChV57ber8+MeMEoSBG/dhtcOzbapf/o3RHh9IBCjp -7IO2X6OXZ7G0Tx9an5UIPq1bsHQRkSvzoRKy9kn5FnfxNeFy658MLoQUsDOMkB/+ -cHanAqGkix7UsBItSwYoMoy7HCw4XRHxyS9CG+YEqcZ/qncVCtT/44VE3GPOKOIN -lj7WDSu337OeTj3ADo8LyowPJDfQK6jfWhQh6T+cqWCnBWGBp/14JnP0cCsntL+R -imV+AyeOc/MlS/iUtWBSnbc+EG3jwgTqBUFbBLr3Vfv4ZtXiAoVSIo+ZZSkYHX1Q -MUGUO+sB/HMhaIl8aRmUhrZo0ztp+Ag/0wqvFIz4wG7OxCQ+97FBijnGnk9txD59 -i4gYCJ2iEY+UGOObnpgVzPHvwItjEMfPhRyXec5Bwk9+Tp6Ao29Onde+mskfC2+t -dw46UHNPxFU5RjswZHrW0Mb1BcS6MoJNBWcQKQ1L6yU2t/5Q/xwGEolTwF8CAwEA -AQKCAgEAj/rDyyd4q55CifcE6Sk0zKN0aIa/xBraxFtFY6++ZSYjxDeus/CiHF1R -H0P2pqWmaCVvYJEQsHsiQ/EDUPEEFK66pDhTtmLksvDMhSeYEhL2oXWzSDaOvSfV -tx64GmUBr6HDUw/HjABECTlr/NU34xS7Djqhg8BeQQxLcfIXasJ3Z954Amtl2gIR -HtdU87EyYeHK8DVxZZ+iFxMW0QrXWklfKBm9qYlucu3Q3/biQUhzD1sdw4PuczZ3 -7vMTSfa1dyuuYyx/xfQIPHsmY2CxhEt5uu2v+BclFYSBB1HMC5vqpEzkJKVeR1qx -IvSDFIueKDaQBbyOBYTgYmysOp6BDyfQO2owYLbnCWu51buzJ7sFvsrx2x6gH+8E -B9gVBz6x609xhfJ/oF/DJGEtHvWi6Nebky2eOOGvUxDYqal4D4Wu3VMH4JYAtzW0 -sLyuC4R9fqQSSt+i2TOUrFD9Be0+1yTqXa38W8SthnpsXTAdiuppEt2Z3PKhyKT3 -IRkB7ko+E8/8wBogtpj6ANU7S1VDFLwX9uk3btpQyf1GmExk9RXesTk+5SpJPAYG -u9/s2dK6/E3yy6z0egMX+YS1iCBu7hD29zgJwTKoZuhWGqS6UlsJsj+XoZZshWdB -pHIhGn1U8coeq3vraDvZMvEV1QdMM4lxB6UAELHgynRtRAVoPGECggEBAPS4+XGQ -uOHWLSoiPgjTo/3cwNzZiooJ9enX9envQBGABAQvuaZo3A3LNvCJaQTu30bGlk00 -6ncFYRTFF0yKkySyfftecMFHI09QJbiZ16faAgahIJ1v7RHnlVee2HGJubZmGU6f -pleYw83jiqdXUEUOS60ljtwZwAmT8f9RX5Xk/BfWCrxjPlncWN+qK3MS2LA5d4bC -qR9k03S01BqUbhzKiPXvKNxH9DYaBOEMR9Y8jsZfcFa3qm9DP9NTioj5qCxESQ0K -qPlrGjvi07dK1AYlKBgZn11sVZNeLIaV5YxxA9vU1pSvsiVo2DFVqtorOnMqYyp2 -MsZQvvht02MeTe8CggEBAMb01JQCcDzE90G9tuCjddzLZ4LW217gukuPX1s/DqYk -fLGgifhA2TaQz2bY7l38tR6MTdIjtSby0+3gUyAJR5rln4YlKPTvU1pXZrdBo/L6 -c3ENfZtaIMmWg5JwiI8rfe1TB+joXAN+WarKfR1IdYliKqdKeTewx890K6aWFlss -5LjiAo8MAyPlhwOTrOy7TUVhyWSjKuMWW0BewlsrLwr5bYf96lXV+yfOtdI1L8kj -JkqC2h4xaNoH3FLxHYuzanHjlylRXPsBGtYX2600iSkSBQT/nLEOkAoXAQuYSSf9 -BxndDLkApFUwGRvR7Zeqwn8sK6cPMLxCDGDqtzUMJJECggEBAJX1Ty0u8Iu6sVxT -NU41WhUyk6ABUep6Zg1QP6mE6O6QUmW6VPnK4aU2NEk6+naCZaE6CVnwJgie7DlM -+JT3iWkAU0CYQ2G8VL/CLgsrgkGhzoH/dDvlHRFOpm3VsW4JZhqF/9X08HpW6w0b -9JfMYiyesPK9xAfNY6a62JyKYZc5hNyaVWPRMKDh7DfKT+uozc1WuY6DdBz/rK+k -KyDKeZr85OZHEfPBx7zrPrgkzVRTMW4Ph4gr7a1WOSMm2wpKaaoOgfES6i5EWgP+ -X5u+p6PIWwRBly0pOFsgZwplig8AJcwsZP0hmHliALBifVpJCP0PKUsYuM5bJlqW -+eSP6lMCggEBAIGWsOvjFrYOWTrB1l9FORzig/6N8f8/gzh6H1MwpFsFQw7zkUpY -+uJ2uhC0VbZ0RoiqPHoWdgTzf6TeS2Z/hNtYm2plw+KeC/25FhdhMuetyAmoDttk -fhyv4+07TUdrva8aOPm0d1JzNvZuw8WSD7S3cy7Z7FCg3dS7Rz9PXSpv+F6QH7iK -y3TTKPjkdGsRqMf7ZJEfUmztf+xSjGeTniWYx4YflkXohZmtI3LS39g0A9nelzG5 -XyE8o36GvIblCv05j5kC/PwlARTYH2cW5AwTR5+02pvc72cTCsRBoczhG4H2IDZU -O7WdCY28U/RI7jylSZ2b4FOO9b8ewsxTp3ECggEAJLrYchLEQcdUCqJEejnO9Itr -nQHK6JQRt/54hxQPIQJ4VsgpvRSgnpN955n4GmCoCVGV45UfqzsyfZbXLq7pmCGb -DeIxWQ1DAx5UjhF2pIM8enG/Lpbfxybw+Bw/Ey+1CEGSD06EVzwm0pwNl0KPliEC -4tIjHugyXNbelELl63PT7hn2vXdcFU3XC2Q3yHstn1PbrsBXH6220/YGH8Nec3YA -cJw6oTBPUo15SA4hjKNUEneVU/vUdz228t0/iB6yUPiL7fBwRLhsSIOL02rPvoIy -SFax7ak138G5Si5A4ntMbjgTEO9VlDpMRHBCbxBmxktx0zQgxzXTY5MiM7H0yQ== ------END RSA PRIVATE KEY----- diff --git a/examples/kms-additional-policy/README.md b/examples/kms-additional-policy/README.md deleted file mode 100644 index c30c660e..00000000 --- a/examples/kms-additional-policy/README.md +++ /dev/null @@ -1,52 +0,0 @@ -# Adds a custom KMS key policy - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### Add a custom KMS key policy - -```hcl - kms = { - additional_policies = [file("${path.module}/kms-policy.json")] - } -``` - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/kms-additional-policy/kms-policy.json b/examples/kms-additional-policy/kms-policy.json deleted file mode 100644 index 2f9ee3da..00000000 --- a/examples/kms-additional-policy/kms-policy.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "AllowAll", - "Effect": "Allow", - "Principal": "arn:aws:iam::123457890:root", - "Action": ["kms:Decrypt"], - "Resource": "*" - } - ] -} diff --git a/examples/kms-additional-policy/main.tf b/examples/kms-additional-policy/main.tf deleted file mode 100644 index 2880e45e..00000000 --- a/examples/kms-additional-policy/main.tf +++ /dev/null @@ -1,21 +0,0 @@ -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks007" - bastion = {} - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - kms = { - additional_policies = [file("${path.module}/kms-policy.json")] - } -} diff --git a/examples/kms-additional-policy/outputs.tf b/examples/kms-additional-policy/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/kms-additional-policy/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/kms-additional-policy/variables.tf b/examples/kms-additional-policy/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/kms-additional-policy/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/kms-additional-policy/versions.tf b/examples/kms-additional-policy/versions.tf deleted file mode 100644 index 308816cb..00000000 --- a/examples/kms-additional-policy/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/examples/kms/README.md b/examples/kms/README.md deleted file mode 100644 index 71ffcb2a..00000000 --- a/examples/kms/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# EKS with kms_key encryption enabled. - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### Enabling kms encryption in the module. -```hcl - kms = { - enabled = true # Enables kms encryption. - key_id = # If provided, the key will be used. Otherwise the module will generate and use a kms_key. - } -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [terraform](#provider\_terraform) | n/a | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -| Name | Type | -|------|------| -| [terraform_remote_state.kms_key](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/data-sources/remote_state) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/kms/main.tf b/examples/kms/main.tf deleted file mode 100644 index bbd979e6..00000000 --- a/examples/kms/main.tf +++ /dev/null @@ -1,34 +0,0 @@ -data "terraform_remote_state" "kms_key" { - backend = "local" - - config = { - path = "${path.module}/../create-kms-key/terraform.tfstate" - } -} - - -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks003" - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - - bastion = { - enabled = true - } - kms = { - enabled = true - key_id = data.terraform_remote_state.kms_key.outputs.kms_key_id - } -} diff --git a/examples/kms/outputs.tf b/examples/kms/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/kms/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/kms/variables.tf b/examples/kms/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/kms/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/kms/versions.tf b/examples/kms/versions.tf deleted file mode 100644 index 308816cb..00000000 --- a/examples/kms/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/examples/minimal-with-bastion/README.md b/examples/minimal-with-bastion/README.md deleted file mode 100644 index 33a6cd18..00000000 --- a/examples/minimal-with-bastion/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# Minimal EKS with bastion. - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: - -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### Creating a bastion as part of the deployment. -#### An empty dict creates the bastion with defaults. -```hcl - - bastion = { - enabled = true - } -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deploy\_id](#input\_deploy\_id) | Unique name for deployment | `string` | `"dominoeks004"` | no | -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/minimal-with-bastion/main.tf b/examples/minimal-with-bastion/main.tf deleted file mode 100644 index f3863126..00000000 --- a/examples/minimal-with-bastion/main.tf +++ /dev/null @@ -1,22 +0,0 @@ - -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = var.deploy_id - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - bastion = { - enabled = true - install_binaries = true - } -} diff --git a/examples/minimal-with-bastion/outputs.tf b/examples/minimal-with-bastion/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/minimal-with-bastion/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/minimal-with-bastion/variables.tf b/examples/minimal-with-bastion/variables.tf deleted file mode 100644 index aaf3987a..00000000 --- a/examples/minimal-with-bastion/variables.tf +++ /dev/null @@ -1,11 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} - -variable "deploy_id" { - description = "Unique name for deployment" - type = string - default = "dominoeks004" -} diff --git a/examples/minimal-with-bastion/versions.tf b/examples/minimal-with-bastion/versions.tf deleted file mode 100644 index 308816cb..00000000 --- a/examples/minimal-with-bastion/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/examples/nodes-custom-ami/README.md b/examples/nodes-custom-ami/README.md deleted file mode 100644 index d17c14cb..00000000 --- a/examples/nodes-custom-ami/README.md +++ /dev/null @@ -1,168 +0,0 @@ -# Specify AMI for the EKS nodes. - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: - -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` - -### You can provide the ami id for any or all of the node_groups, Note that the `gpu` node_group needs a GPU specific AMI. By default EKS will determine the latest ami. - -```hcl -data "aws_ami" "eks_node" { - most_recent = true - - filter { - name = "name" - values = ["amazon-eks-node-1.25-*"] - } - - filter { - name = "owner-id" - values = ["602401143452"] # Amazon EKS AMI account ID - } - - filter { - name = "state" - values = ["available"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - -# GPU -data "aws_ami" "eks_gpu_node" { - most_recent = true - - filter { - name = "name" - values = ["amazon-eks-gpu-node-1.25-*"] - } - - filter { - name = "owner-id" - values = ["602401143452"] # Amazon EKS AMI account ID - } - - filter { - name = "state" - values = ["available"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} -``` - -#### The AMIs are specified in each of `the default_node_groups` `ami` field. -```hcl -default_node_groups = { - compute = { - ami = data.aws_ami.eks_node.image_id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - ami = data.aws_ami.eks_node.image_id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - ami = data.aws_ami.eks_gpu_node.id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } -``` - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | 4.60.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -| Name | Type | -|------|------| -| [aws_ami.eks_gpu_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_ami.eks_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | - -## Inputs - -No inputs. - -## Outputs - -No outputs. - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws](#provider\_aws) | >= 4.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -| Name | Type | -|------|------| -| [aws_ami.eks_gpu_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | -| [aws_ami.eks_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/nodes-custom-ami/main.tf b/examples/nodes-custom-ami/main.tf deleted file mode 100644 index c196b5d1..00000000 --- a/examples/nodes-custom-ami/main.tf +++ /dev/null @@ -1,84 +0,0 @@ -data "aws_ami" "eks_node" { - most_recent = true - - filter { - name = "name" - values = ["amazon-eks-node-1.25-*"] - } - - filter { - name = "owner-id" - values = ["602401143452"] # Amazon EKS AMI account ID - } - - filter { - name = "state" - values = ["available"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - -# GPU -data "aws_ami" "eks_gpu_node" { - most_recent = true - - filter { - name = "name" - values = ["amazon-eks-gpu-node-1.25-*"] - } - - filter { - name = "owner-id" - values = ["602401143452"] # Amazon EKS AMI account ID - } - - filter { - name = "state" - values = ["available"] - } - - filter { - name = "architecture" - values = ["x86_64"] - } - - filter { - name = "virtualization-type" - values = ["hvm"] - } -} - -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks005" - default_node_groups = { - compute = { - ami = data.aws_ami.eks_node.image_id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - ami = data.aws_ami.eks_node.image_id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - ami = data.aws_ami.eks_gpu_node.id - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - - bastion = { - enabled = true - install_binaries = true - } -} diff --git a/examples/nodes-custom-ami/outputs.tf b/examples/nodes-custom-ami/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/nodes-custom-ami/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/nodes-custom-ami/variables.tf b/examples/nodes-custom-ami/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/nodes-custom-ami/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/nodes-custom-ami/versions.tf b/examples/nodes-custom-ami/versions.tf deleted file mode 100644 index 308816cb..00000000 --- a/examples/nodes-custom-ami/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/examples/public-access/README.md b/examples/public-access/README.md deleted file mode 100644 index 552b3d60..00000000 --- a/examples/public-access/README.md +++ /dev/null @@ -1,58 +0,0 @@ -# EKS with public access - -### Provide full path for existing ssh key: `ssh_pvt_key_path` -### Otherwise generate using: - -```bash -ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f domino.pem -``` -### By default the module does not enable public access. -#### Enable EKS public endpoint. - -```hcl - eks = { - public_access = { - enabled = true - ## Replace "0.0.0.0/0" with ranges you want to allow access to the EKS API. - cidrs = ["0.0.0.0/0"] - } - } -``` - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [local](#requirement\_local) | >= 2.2.0 | -| [tls](#requirement\_tls) | >= 3.4.0 | - -## Providers - -No providers. - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./../.. | n/a | - -## Resources - -No resources. - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [region](#input\_region) | AWS region for deployment. | `string` | `"us-west-2"` | no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | Module domino\_eks output | - diff --git a/examples/public-access/main.tf b/examples/public-access/main.tf deleted file mode 100644 index 3fdf2721..00000000 --- a/examples/public-access/main.tf +++ /dev/null @@ -1,28 +0,0 @@ - -module "domino_eks" { - source = "./../.." - region = var.region - ssh_pvt_key_path = "./../examples.pem" - deploy_id = "dominoeks006" - default_node_groups = { - compute = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - platform = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - gpu = { - availability_zone_ids = ["usw2-az1", "usw2-az2"] - } - } - bastion = { - enabled = false - } - eks = { - public_access = { - enabled = true - cidrs = ["108.214.49.0/24"] # Replace this with the desired CIDR range - - } - } -} diff --git a/examples/public-access/outputs.tf b/examples/public-access/outputs.tf deleted file mode 100644 index 99ed4cdb..00000000 --- a/examples/public-access/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "domino_eks" { - description = "Module domino_eks output" - value = module.domino_eks -} diff --git a/examples/public-access/variables.tf b/examples/public-access/variables.tf deleted file mode 100644 index b78602f5..00000000 --- a/examples/public-access/variables.tf +++ /dev/null @@ -1,5 +0,0 @@ -variable "region" { - description = "AWS region for deployment." - type = string - default = "us-west-2" -} diff --git a/examples/public-access/versions.tf b/examples/public-access/versions.tf deleted file mode 100644 index 308816cb..00000000 --- a/examples/public-access/versions.tf +++ /dev/null @@ -1,21 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - } -} - -provider "aws" { - region = var.region -} diff --git a/examples/tf-plan-test.sh b/examples/tf-plan-test.sh deleted file mode 100755 index 489a5726..00000000 --- a/examples/tf-plan-test.sh +++ /dev/null @@ -1,85 +0,0 @@ -#! /usr/bin/env bash - -exclude=("bring-your-vpc" "examples.pem" "tf-plan-test.sh" "create-kms-key" "kms") - -failed_dirs=() -success_dirs=() - -verify_terraform() { - if ! [ -x "$(command -v terraform)" ]; then - printf "\n\033[0;31mError: Terraform is not installed!!!\033[0m\n" - exit 1 - else - terraform_version=$(terraform --version | awk '/Terraform/ {print $2}') - printf "\033[0;32mTerraform version $terraform_version is installed.\033[0m\n" - fi -} - -tf_plan() { - local dir="${1}" - terraform -chdir="$dir" init -upgrade - terraform -chdir="$dir" plan - - if [ "$?" != "0" ]; then - printf "\033[0;31mERROR: terraform plan failed in $dir\033[0m.\n" - failed_dirs+=("${dir%/}") - else - printf "\033[0;32mSUCCESS: terraform plan succeeded in $dir\033[0m.\n" - success_dirs+=("${dir%/}") - fi - -} - -run_terraform_plans() { - for dir in */; do - [[ " ${exclude[*]} " == *" ${dir%/} "* ]] && continue - printf "\n\033[0;33mRunning terraform plan in ${dir%/}\033[0m:\n" - tf_plan "${dir}" - done -} - -create_kms_key() { - local dir="create-kms-key" - - printf "\n\033[0;33mCreating KMS key\033[0m\n" - terraform -chdir="$dir" init - if ! terraform -chdir="$dir" apply --auto-approve; then - printf "\n\033[0;31mFailed to create kms key!!!\033[0m\n" - failed_dirs+=("kms") - else - printf "\n\033[0;32mKMS key created successfully\033[0m\n" - fi - -} - -destroy_kms_key() { - local dir="create-kms-key" - - printf "\n\033[0;33mDestroying KMS key\033[0m\n" - terraform -chdir="$dir" destroy --auto-approve || terraform -chdir="$dir" destroy --auto-approve --refresh=false -} - -test_kms() { - create_kms_key - tf_plan "kms" -} - -finish() { - destroy_kms_key - - if [ "${#success_dirs[@]}" != "0" ]; then - printf "\n\033[0;32mThe following examples ran the terraform plan successfully:\033[0m\n" - printf '\033[0;32m%s\n\033[0m' "${success_dirs[@]}" - fi - - if [ "${#failed_dirs[@]}" != "0" ]; then - printf "\n\033[0;31mThe following examples failed the terraform plan:\033[0m\n" - printf '\033[0;31m%s\n\033[0m' "${failed_dirs[@]} " - exit 1 - fi -} - -trap finish EXIT ERR INT TERM -verify_terraform -run_terraform_plans -test_kms diff --git a/examples/tfvars/bring-your-vpc.tfvars b/examples/tfvars/bring-your-vpc.tfvars new file mode 100644 index 00000000..69b5979e --- /dev/null +++ b/examples/tfvars/bring-your-vpc.tfvars @@ -0,0 +1,31 @@ +deploy_id = "plantest001" +region = "us-west-2" +ssh_pvt_key_path = "domino.pem" + +default_node_groups = { + compute = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + gpu = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + platform = { + "availability_zone_ids" = ["usw2-az1", "usw2-az2"] + } +} + +bastion = { + enabled = true +} +network = { + ## The following are not real values + vpc = { + id = "vpc-notrealb8ca349af" + subnets = { + private = ["subnet-notrealvalb2319f", "subnet-notrealval9be2580"] + public = ["subnet-notrealval126cd0", "subnet-notrealval178f224"] + pod = ["subnet-notrealval126cgf4", "subnet-notrealval178f64"] + } + } + use_pod_cidr = false +} diff --git a/examples/tfvars/custom-node-pool.tfvars b/examples/tfvars/custom-node-pool.tfvars new file mode 100644 index 00000000..52291c36 --- /dev/null +++ b/examples/tfvars/custom-node-pool.tfvars @@ -0,0 +1,37 @@ +deploy_id = "plantest002" +region = "us-west-2" +ssh_pvt_key_path = "domino.pem" + +default_node_groups = { + compute = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + platform = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + gpu = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } +} + +additional_node_groups = { + custom-0 = { + instance_types = [ + "m5.2xlarge" + ], + min_per_az = 0, + max_per_az = 10, + desired_per_az = 0, + availability_zone_ids = [ + "usw2-az1", + "usw2-az2" + ], + labels = { + "dominodatalab.com/node-pool" = "custom-group-0" + }, + volume = { + size = 100, + type = "gp3" + } + } +} diff --git a/examples/tfvars/kms-add-policies.tfvars b/examples/tfvars/kms-add-policies.tfvars new file mode 100644 index 00000000..f680448b --- /dev/null +++ b/examples/tfvars/kms-add-policies.tfvars @@ -0,0 +1,34 @@ +deploy_id = "plantest010" +region = "us-west-2" +ssh_pvt_key_path = "domino.pem" + +default_node_groups = { + compute = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + gpu = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + platform = { + "availability_zone_ids" = ["usw2-az1", "usw2-az2"] + } +} + +kms = { + enabled = true + additional_policies = [< [terraform](#requirement\_terraform) | >= 1.4.0 | -| [aws](#requirement\_aws) | >= 4.0 | +| [aws](#requirement\_aws) | ~> 5.0 | | [null](#requirement\_null) | >= 3.1.0 | | [tls](#requirement\_tls) | >= 4.0 | @@ -14,8 +14,8 @@ | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.0 | -| [aws.eks](#provider\_aws.eks) | >= 4.0 | +| [aws](#provider\_aws) | ~> 5.0 | +| [aws.eks](#provider\_aws.eks) | ~> 5.0 | | [null](#provider\_null) | >= 3.1.0 | | [terraform](#provider\_terraform) | n/a | | [tls](#provider\_tls) | >= 4.0 | @@ -24,19 +24,17 @@ | Name | Source | Version | |------|--------|---------| -| [k8s\_setup](#module\_k8s\_setup) | ../k8s | n/a | +| [k8s\_setup](#module\_k8s\_setup) | ./submodules/k8s | n/a | +| [privatelink](#module\_privatelink) | ./submodules/privatelink | n/a | ## Resources | Name | Type | |------|------| -| [aws_autoscaling_group_tag.tag](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | | [aws_cloudwatch_log_group.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | -| [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_addon.vpc_cni](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | | [aws_eks_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_cluster) | resource | | [aws_eks_identity_provider_config.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_identity_provider_config) | resource | -| [aws_eks_node_group.node_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource | | [aws_iam_policy.custom_eks_node_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | @@ -44,7 +42,6 @@ | [aws_iam_role_policy_attachment.aws_eks_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.custom_eks_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_launch_template.node_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | | [aws_security_group.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.eks_nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.bastion_eks](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | @@ -52,13 +49,10 @@ | [aws_security_group_rule.eks_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [null_resource.kubeconfig](https://registry.terraform.io/providers/hashicorp/null/latest/docs/resources/resource) | resource | -| [terraform_data.calico_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | | [terraform_data.run_k8s_pre_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | -| [aws_ami.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | | [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_caller_identity.aws_eks_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | -| [aws_ec2_instance_type_offerings.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | -| [aws_eks_addon_version.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | +| [aws_eks_addon_version.default_vpc_cni](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | | [aws_iam_policy_document.autoscaler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.custom_eks_node_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.ebs_csi](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | @@ -69,8 +63,6 @@ | [aws_iam_role.master_roles](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_role) | data source | | [aws_iam_session_context.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source | | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | -| [aws_ssm_parameter.eks_ami_release_version](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | -| [aws_ssm_parameter.eks_gpu_ami_release_version](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | | [tls_certificate.cluster_tls_certificate](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/certificate) | data source | ## Inputs @@ -78,19 +70,22 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [bastion\_info](#input\_bastion\_info) | user = Bastion username.
public\_ip = Bastion public ip.
security\_group\_id = Bastion sg id.
ssh\_bastion\_command = Command to ssh onto bastion. | 
object({
    user                = string
    public_ip           = string
    security_group_id   = string
    ssh_bastion_command = string
  })
| n/a | yes | +| [create\_eks\_role\_arn](#input\_create\_eks\_role\_arn) | Role arn to assume during the EKS cluster creation. | `string` | n/a | yes | | [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes | | [efs\_security\_group](#input\_efs\_security\_group) | Security Group ID for EFS | `string` | n/a | yes | -| [eks](#input\_eks) | k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file. Defaults to '/kubeconfig'
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
List of Custom role maps for aws auth configmap
custom\_role\_maps = [{
rolearn = string
username = string
groups = list(string)
}]
master\_role\_names = IAM role names to be added as masters in EKS.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
ssm\_log\_group\_name = "CloudWatch log group to send the SSM session logs to."
vpc\_cni = Configuration for AWS VPC CNI
identity\_providers = "Configuration for IDP(Identity Provider)." | 
object({
    k8s_version = optional(string)
    kubeconfig = optional(object({
      extra_args = optional(string)
      path       = optional(string)
    }))
    public_access = optional(object({
      enabled = optional(bool)
      cidrs   = optional(list(string))
    }))
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })))
    master_role_names  = optional(list(string))
    cluster_addons     = optional(list(string))
    ssm_log_group_name = optional(string)
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string, null)
      groups_prefix                 = optional(string, null)
      identity_provider_config_name = string
      issuer_url                    = optional(string, null)
      required_claims               = optional(string, null)
      username_claim                = optional(string, null)
      username_prefix               = optional(string, null)
    })), [])
  })
| n/a | yes | +| [eks](#input\_eks) | creation\_role\_name = Name of the role to import.
k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} | 
object({
    creation_role_name = optional(string, null)
    k8s_version        = optional(string, "1.27")
    kubeconfig = optional(object({
      extra_args = optional(string, "")
      path       = optional(string, null)
    }), {})
    public_access = optional(object({
      enabled = optional(bool, false)
      cidrs   = optional(list(string), [])
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })), [])
    master_role_names  = optional(list(string), [])
    cluster_addons     = optional(list(string), ["kube-proxy", "coredns"])
    ssm_log_group_name = optional(string, "session-manager")
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string, null)
      groups_prefix                 = optional(string, null)
      identity_provider_config_name = string
      issuer_url                    = optional(string, null)
      required_claims               = optional(string, null)
      username_claim                = optional(string, null)
      username_prefix               = optional(string, null)
    })), [])
  })
| `{}` | no | | [kms\_info](#input\_kms\_info) | key\_id = KMS key id.
key\_arn = KMS key arn.
enabled = KMS key is enabled | 
object({
    key_id  = string
    key_arn = string
    enabled = bool
  })
| n/a | yes | | [network\_info](#input\_network\_info) | id = VPC ID.
subnets = {
public = List of public Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
private = List of private Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
pod = List of pod Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
} | 
object({
    vpc_id = string
    subnets = object({
      public = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
      private = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
      pod = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
    })
  })
| n/a | yes | -| [node\_groups](#input\_node\_groups) | EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints                = optional(list(object({ key = string, value = optional(string), effect = string })), [])
    tags                  = optional(map(string), {})
    instance_tags         = optional(map(string), {})
    gpu                   = optional(bool, false)
    volume = object({
      size = string
      type = string
    })
  }))
| n/a | yes | | [node\_iam\_policies](#input\_node\_iam\_policies) | Additional IAM Policy Arns for Nodes | `list(string)` | n/a | yes | +| [privatelink](#input\_privatelink) | {
enabled = Enable Private Link connections.
namespace = Namespace for IAM Policy conditions.
monitoring\_bucket = Bucket for NLBs monitoring.
route53\_hosted\_zone\_name = Hosted zone for External DNS zone.
vpc\_endpoint\_services = [{
name = Name of the VPC Endpoint Service.
ports = List of ports exposing the VPC Endpoint Service. i.e [8080, 8081]
cert\_arn = Certificate ARN used by the NLB associated for the given VPC Endpoint Service.
private\_dns = Private DNS for the VPC Endpoint Service.
}]
} | 
object({
    enabled                  = optional(bool, false)
    namespace                = optional(string, "domino-platform")
    monitoring_bucket        = optional(string, null)
    route53_hosted_zone_name = optional(string, null)
    vpc_endpoint_services = optional(list(object({
      name        = optional(string)
      ports       = optional(list(number))
      cert_arn    = optional(string)
      private_dns = optional(string)
    })), [])
  })
| `{}` | no | | [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | | [ssh\_key](#input\_ssh\_key) | path = SSH private key filepath.
key\_pair\_name = AWS key\_pair name. | 
object({
    path          = string
    key_pair_name = string
  })
| n/a | yes | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | ## Outputs | Name | Description | |------|-------------| | [info](#output\_info) | EKS information | +| [privatelink](#output\_privatelink) | Private Link Info | diff --git a/submodules/eks/cluster.tf b/modules/eks/cluster.tf similarity index 87% rename from submodules/eks/cluster.tf rename to modules/eks/cluster.tf index 9f14c353..948b7ec8 100644 --- a/submodules/eks/cluster.tf +++ b/modules/eks/cluster.tf @@ -85,16 +85,15 @@ resource "aws_iam_openid_connect_provider" "oidc_provider" { } -data "aws_eks_addon_version" "default" { - for_each = toset(concat(["vpc-cni"], var.eks.cluster_addons)) - addon_name = each.key +data "aws_eks_addon_version" "default_vpc_cni" { + addon_name = "vpc-cni" kubernetes_version = aws_eks_cluster.this.version } resource "aws_eks_addon" "vpc_cni" { cluster_name = aws_eks_cluster.this.name addon_name = "vpc-cni" - addon_version = data.aws_eks_addon_version.default["vpc-cni"].version + addon_version = data.aws_eks_addon_version.default_vpc_cni.version resolve_conflicts_on_create = "OVERWRITE" resolve_conflicts_on_update = "OVERWRITE" configuration_values = jsonencode({ @@ -105,19 +104,6 @@ resource "aws_eks_addon" "vpc_cni" { }) } -resource "aws_eks_addon" "this" { - for_each = toset(var.eks.cluster_addons) - cluster_name = aws_eks_cluster.this.name - addon_name = each.key - addon_version = data.aws_eks_addon_version.default[each.key].version - resolve_conflicts_on_create = "OVERWRITE" - resolve_conflicts_on_update = "OVERWRITE" - - depends_on = [ - aws_eks_node_group.node_groups, - ] -} - resource "null_resource" "kubeconfig" { provisioner "local-exec" { when = create diff --git a/submodules/eks/eks-bastion-access.tf b/modules/eks/eks-bastion-access.tf similarity index 100% rename from submodules/eks/eks-bastion-access.tf rename to modules/eks/eks-bastion-access.tf diff --git a/submodules/eks/iam.tf b/modules/eks/iam.tf similarity index 99% rename from submodules/eks/iam.tf rename to modules/eks/iam.tf index 2c3bd5bf..12c532ec 100644 --- a/submodules/eks/iam.tf +++ b/modules/eks/iam.tf @@ -244,13 +244,13 @@ resource "aws_iam_policy" "custom_eks_node_policy" { } locals { - eks_aws_node_iam_policies = toset([ + eks_aws_node_iam_policies = [ "AmazonEKSWorkerNodePolicy", "AmazonEKS_CNI_Policy", "AmazonEC2ContainerRegistryReadOnly", "AmazonSSMManagedInstanceCore", "AmazonElasticFileSystemReadOnlyAccess", - ]) + ] custom_node_policies = concat([aws_iam_policy.custom_eks_node_policy.arn], var.node_iam_policies) } diff --git a/modules/eks/k8s.tf b/modules/eks/k8s.tf new file mode 100644 index 00000000..5bb32e00 --- /dev/null +++ b/modules/eks/k8s.tf @@ -0,0 +1,32 @@ +locals { + run_setup = var.bastion_info != null || var.eks.public_access.enabled ? 1 : 0 + k8s_pre_setup_sh_file = local.run_setup != 0 ? module.k8s_setup[0].filepath : null +} + +module "k8s_setup" { + count = local.run_setup + + source = "./submodules/k8s" + ssh_key = var.ssh_key + bastion_info = var.bastion_info + network_info = var.network_info + eks_info = local.eks_info + + depends_on = [aws_eks_addon.vpc_cni, null_resource.kubeconfig] +} + +resource "terraform_data" "run_k8s_pre_setup" { + count = local.run_setup + + triggers_replace = [ + module.k8s_setup[0].change_hash + ] + + provisioner "local-exec" { + command = "bash ./${basename(module.k8s_setup[0].filepath)} set_k8s_auth set_eniconfig" + interpreter = ["bash", "-c"] + working_dir = dirname(module.k8s_setup[0].filepath) + } + + depends_on = [module.k8s_setup] +} diff --git a/submodules/eks/main.tf b/modules/eks/main.tf similarity index 93% rename from submodules/eks/main.tf rename to modules/eks/main.tf index c0a22729..c3a37495 100644 --- a/submodules/eks/main.tf +++ b/modules/eks/main.tf @@ -16,7 +16,7 @@ data "aws_iam_session_context" "create_eks_role" { } locals { - kubeconfig_path = try(abspath(pathexpand(var.eks.kubeconfig.path)), "${path.cwd}/kubeconfig") + kubeconfig_path = var.eks.kubeconfig.path != null ? abspath(pathexpand(var.eks.kubeconfig.path)) : "${path.cwd}/kubeconfig" kubeconfig_args_list = split(" ", chomp(trimspace(var.eks.kubeconfig.extra_args))) kubeconfig_args_list_parsed = contains(local.kubeconfig_args_list, "--role-arn") ? local.kubeconfig_args_list : concat(local.kubeconfig_args_list, ["--role-arn", data.aws_iam_session_context.create_eks_role.issuer_arn]) kubeconfig_args = join(" ", local.kubeconfig_args_list_parsed) @@ -172,8 +172,16 @@ locals { } + eks_info = { cluster = { + specs = { + name = aws_eks_cluster.this.name + endpoint = aws_eks_cluster.this.endpoint + certificate_authority = aws_eks_cluster.this.certificate_authority + kubernetes_network_config = aws_eks_cluster.this.kubernetes_network_config + } + addons = var.eks.cluster_addons version = aws_eks_cluster.this.version public_access = var.eks.public_access arn = aws_eks_cluster.this.arn diff --git a/modules/eks/node-group.tf b/modules/eks/node-group.tf new file mode 100644 index 00000000..cd0f015a --- /dev/null +++ b/modules/eks/node-group.tf @@ -0,0 +1,62 @@ +## EKS Nodes +data "aws_iam_policy_document" "eks_nodes" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["ec2.${local.dns_suffix}"] + } + } +} + +resource "aws_iam_role" "eks_nodes" { + name = "${local.eks_cluster_name}-eks-nodes" + assume_role_policy = data.aws_iam_policy_document.eks_nodes.json +} + +resource "aws_security_group" "eks_nodes" { + name = "${local.eks_cluster_name}-nodes" + description = "EKS cluster Nodes security group" + vpc_id = var.network_info.vpc_id + + lifecycle { + create_before_destroy = true + ignore_changes = [ + name, + description + ] + } + tags = { + "Name" = "${local.eks_cluster_name}-eks-nodes" + "kubernetes.io/cluster/${local.eks_cluster_name}" = "owned" + } +} + +resource "aws_security_group_rule" "node" { + for_each = local.node_security_group_rules + + # Required + security_group_id = aws_security_group.eks_nodes.id + protocol = each.value.protocol + from_port = each.value.from_port + to_port = each.value.to_port + type = each.value.type + description = each.value.description + cidr_blocks = try(each.value.cidr_blocks, null) + self = try(each.value.self, null) + source_security_group_id = try( + each.value.source_security_group_id, + try(each.value.source_cluster_security_group, false) ? aws_security_group.eks_cluster.id : null + ) +} + +resource "aws_security_group_rule" "efs" { + security_group_id = var.efs_security_group + protocol = "tcp" + from_port = 2049 + to_port = 2049 + type = "ingress" + description = "EFS access" + source_security_group_id = aws_security_group.eks_nodes.id +} diff --git a/modules/eks/outputs.tf b/modules/eks/outputs.tf new file mode 100644 index 00000000..e7eec64f --- /dev/null +++ b/modules/eks/outputs.tf @@ -0,0 +1,12 @@ +output "info" { + description = "EKS information" + value = merge(local.eks_info, { + k8s_pre_setup_sh_file = local.k8s_pre_setup_sh_file + } + ) +} + +output "privatelink" { + description = "Private Link Info" + value = var.privatelink.enabled ? module.privatelink[0].info : null +} \ No newline at end of file diff --git a/modules/eks/privatelink.tf b/modules/eks/privatelink.tf new file mode 100644 index 00000000..86b2f583 --- /dev/null +++ b/modules/eks/privatelink.tf @@ -0,0 +1,9 @@ +module "privatelink" { + count = var.privatelink.enabled ? 1 : 0 + source = "./submodules/privatelink" + deploy_id = var.deploy_id + region = var.region + network_info = var.network_info + privatelink = var.privatelink + oidc_provider_id = aws_iam_openid_connect_provider.oidc_provider.id +} \ No newline at end of file diff --git a/submodules/k8s/README.md b/modules/eks/submodules/k8s/README.md similarity index 98% rename from submodules/k8s/README.md rename to modules/eks/submodules/k8s/README.md index f1085cf3..e164a26f 100644 --- a/submodules/k8s/README.md +++ b/modules/eks/submodules/k8s/README.md @@ -43,6 +43,6 @@ No modules. | Name | Description | |------|-------------| | [change\_hash](#output\_change\_hash) | Hash of all templated files | -| [filename](#output\_filename) | Filename of primary script | +| [filepath](#output\_filepath) | Filename of primary script | | [resources\_directory](#output\_resources\_directory) | Directory for provisioned scripts and templated files | diff --git a/submodules/k8s/main.tf b/modules/eks/submodules/k8s/main.tf similarity index 89% rename from submodules/k8s/main.tf rename to modules/eks/submodules/k8s/main.tf index 8b0c26bf..20cead37 100644 --- a/submodules/k8s/main.tf +++ b/modules/eks/submodules/k8s/main.tf @@ -6,7 +6,7 @@ resource "random_integer" "port" { locals { k8s_functions_sh_filename = "k8s-functions.sh" k8s_functions_sh_template = "k8s-functions.sh.tftpl" - k8s_pre_setup_sh_filename = "k8s-pre-setup.sh" + k8s_pre_setup_sh_file = "k8s-pre-setup.sh" k8s_pre_setup_sh_template = "k8s-pre-setup.sh.tftpl" aws_auth_filename = "aws-auth.yaml" aws_auth_template = "aws-auth.yaml.tftpl" @@ -32,7 +32,7 @@ locals { } k8s_presetup = { - filename = local.k8s_pre_setup_sh_filename + filename = local.k8s_pre_setup_sh_file content = templatefile("${local.templates_dir}/${local.k8s_pre_setup_sh_template}", { k8s_functions_sh_filename = local.k8s_functions_sh_filename }) @@ -69,5 +69,5 @@ resource "local_file" "templates" { } locals { - change_hash = "${join("-", [for file in ["k8s_presetup", "k8s_functions_sh", "aws_auth"] : md5(local_file.templates[file].content)])}-${try(md5(local_file.templates["eni_config"].content), "none")}" + change_hash = join("-", [for tpl in sort(keys(local_file.templates)) : local_file.templates[tpl].content_md5]) } diff --git a/submodules/k8s/outputs.tf b/modules/eks/submodules/k8s/outputs.tf similarity index 75% rename from submodules/k8s/outputs.tf rename to modules/eks/submodules/k8s/outputs.tf index 54edc844..68c9d4be 100644 --- a/submodules/k8s/outputs.tf +++ b/modules/eks/submodules/k8s/outputs.tf @@ -3,9 +3,9 @@ output "change_hash" { value = local.change_hash } -output "filename" { +output "filepath" { description = "Filename of primary script" - value = basename(local_file.templates["k8s_presetup"].filename) + value = "${local.resources_directory}/${local.k8s_pre_setup_sh_file}" } output "resources_directory" { diff --git a/submodules/k8s/templates/aws-auth.yaml.tftpl b/modules/eks/submodules/k8s/templates/aws-auth.yaml.tftpl similarity index 100% rename from submodules/k8s/templates/aws-auth.yaml.tftpl rename to modules/eks/submodules/k8s/templates/aws-auth.yaml.tftpl diff --git a/submodules/k8s/templates/eniconfig.yaml.tftpl b/modules/eks/submodules/k8s/templates/eniconfig.yaml.tftpl similarity index 100% rename from submodules/k8s/templates/eniconfig.yaml.tftpl rename to modules/eks/submodules/k8s/templates/eniconfig.yaml.tftpl diff --git a/submodules/k8s/templates/k8s-functions.sh.tftpl b/modules/eks/submodules/k8s/templates/k8s-functions.sh.tftpl similarity index 89% rename from submodules/k8s/templates/k8s-functions.sh.tftpl rename to modules/eks/submodules/k8s/templates/k8s-functions.sh.tftpl index b77c3ca7..200dec09 100644 --- a/submodules/k8s/templates/k8s-functions.sh.tftpl +++ b/modules/eks/submodules/k8s/templates/k8s-functions.sh.tftpl @@ -7,13 +7,21 @@ EC="\e[0m" KUBECONFIG="${kubeconfig_path}" KUBECONFIG_PROXY="$KUBECONFIG-proxy" TUNNEL_SOCKET_FILE=$${TUNNEL_SOCKET_FILE:-/tmp/k8s-tunnel-socket-${k8s_tunnel_port}} +kubeconfig="" open_ssh_tunnel_to_k8s_api() { if [[ -n "${bastion_public_ip}" && -n "${bastion_user}" ]]; then + close_ssh_tunnel_to_k8s_api >/dev/null 2>&1 || true printf "$GREEN Opening k8s tunnel ... $EC \n" - ssh-keygen -R "${bastion_public_ip}" || true + ssh-keygen -R "${bastion_public_ip}" >/dev/null 2>&1 || true + chmod 400 "${ssh_pvt_key_path}" - ssh -q -N -f -M -o "IdentitiesOnly=yes" -o "StrictHostKeyChecking=no" -o "ExitOnForwardFailure=yes" -i "${ssh_pvt_key_path}" -D ${k8s_tunnel_port} -S "$TUNNEL_SOCKET_FILE" ${bastion_user}@${bastion_public_ip} + if ssh -v -q -N -f -M -o "IdentitiesOnly=yes" -o "StrictHostKeyChecking=no" -o "ExitOnForwardFailure=yes" -i "${ssh_pvt_key_path}" -D ${k8s_tunnel_port} -S "$TUNNEL_SOCKET_FILE" ${bastion_user}@${bastion_public_ip}; then + printf "$GREEN Established k8s ssh tunnel. $EC\n" + else + printf "$RED Failed to establish k8s ssh tunnel. $EC\n" + exit 1 + fi else printf "$GREEN No bastion, no tunnel needed... $EC \n" fi @@ -26,13 +34,14 @@ check_kubeconfig() { echo "$KUBECONFIG exists, creating $KUBECONFIG_PROXY for proxy use." cp $KUBECONFIG $KUBECONFIG_PROXY kubectl --kubeconfig $KUBECONFIG_PROXY config set "clusters.${eks_cluster_arn}.proxy-url" "socks5://127.0.0.1:${k8s_tunnel_port}" - export kubeconfig=$KUBECONFIG_PROXY + kubeconfig=$KUBECONFIG_PROXY else - export kubeconfig=$KUBECONFIG + kubeconfig=$KUBECONFIG fi else echo "$KUBECONFIG does not exist." && exit 1 fi + export KUBECONFIG="$kubeconfig" echo } diff --git a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl b/modules/eks/submodules/k8s/templates/k8s-pre-setup.sh.tftpl similarity index 100% rename from submodules/k8s/templates/k8s-pre-setup.sh.tftpl rename to modules/eks/submodules/k8s/templates/k8s-pre-setup.sh.tftpl diff --git a/submodules/k8s/variables.tf b/modules/eks/submodules/k8s/variables.tf similarity index 100% rename from submodules/k8s/variables.tf rename to modules/eks/submodules/k8s/variables.tf diff --git a/submodules/k8s/versions.tf b/modules/eks/submodules/k8s/versions.tf similarity index 100% rename from submodules/k8s/versions.tf rename to modules/eks/submodules/k8s/versions.tf diff --git a/modules/eks/submodules/privatelink/README.md b/modules/eks/submodules/privatelink/README.md new file mode 100644 index 00000000..2bfb6cbb --- /dev/null +++ b/modules/eks/submodules/privatelink/README.md @@ -0,0 +1,51 @@ + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.0 | +| [aws](#requirement\_aws) | ~> 5.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_lb.nlbs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb) | resource | +| [aws_lb_listener.listeners](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_listener) | resource | +| [aws_lb_target_group.target_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lb_target_group) | resource | +| [aws_route53_record.service_endpoint_private_dns_verification](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route53_record) | resource | +| [aws_vpc_endpoint_service.vpc_endpoint_services](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_endpoint_service) | resource | +| [aws_iam_policy_document.load_balancer_controller](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.load_balancer_controller_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID | `string` | n/a | yes | +| [network\_info](#input\_network\_info) | {
vpc\_id = VPC Id.
subnets = {
private = Private subnets.
public = Public subnets.
pod = Pod subnets.
}), {})
}), {}) | 
object({
    vpc_id = string
    subnets = object({
      private = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
      public = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
      pod = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
    })
  })
| n/a | yes | +| [oidc\_provider\_id](#input\_oidc\_provider\_id) | OIDC Provider ID | `string` | n/a | yes | +| [privatelink](#input\_privatelink) | {
enabled = Enable Private Link connections.
namespace = Namespace for IAM Policy conditions.
monitoring\_bucket = Bucket for NLBs monitoring.
route53\_hosted\_zone\_name = Hosted zone for External DNS zone.
vpc\_endpoint\_services = [{
name = Name of the VPC Endpoint Service.
ports = List of ports exposing the VPC Endpoint Service. i.e [8080, 8081]
cert\_arn = Certificate ARN used by the NLB associated for the given VPC Endpoint Service.
private\_dns = Private DNS for the VPC Endpoint Service.
}]
} | 
object({
    enabled                  = optional(bool, false)
    namespace                = optional(string, "domino-platform")
    monitoring_bucket        = optional(string, null)
    route53_hosted_zone_name = optional(string, null)
    vpc_endpoint_services = optional(list(object({
      name        = optional(string)
      ports       = optional(list(number))
      cert_arn    = optional(string)
      private_dns = optional(string)
    })), [])
  })
| `{}` | no | +| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| [info](#output\_info) | Target groups... | ++ \ No newline at end of file diff --git a/modules/eks/submodules/privatelink/aws-load-balancer-controller_2.5.4_iam_policy.json b/modules/eks/submodules/privatelink/aws-load-balancer-controller_2.5.4_iam_policy.json new file mode 100644 index 00000000..5b34ad79 --- /dev/null +++ b/modules/eks/submodules/privatelink/aws-load-balancer-controller_2.5.4_iam_policy.json @@ -0,0 +1,241 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "iam:CreateServiceLinkedRole" + ], + "Resource": "*", + "Condition": { + "StringEquals": { + "iam:AWSServiceName": "elasticloadbalancing.amazonaws.com" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeAvailabilityZones", + "ec2:DescribeInternetGateways", + "ec2:DescribeVpcs", + "ec2:DescribeVpcPeeringConnections", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeInstances", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeTags", + "ec2:GetCoipPoolUsage", + "ec2:DescribeCoipPools", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:DescribeTags" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "cognito-idp:DescribeUserPoolClient", + "acm:ListCertificates", + "acm:DescribeCertificate", + "iam:ListServerCertificates", + "iam:GetServerCertificate", + "waf-regional:GetWebACL", + "waf-regional:GetWebACLForResource", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + "wafv2:GetWebACL", + "wafv2:GetWebACLForResource", + "wafv2:AssociateWebACL", + "wafv2:DisassociateWebACL", + "shield:GetSubscriptionState", + "shield:DescribeProtection", + "shield:CreateProtection", + "shield:DeleteProtection" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateSecurityGroup" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags" + ], + "Resource": "arn:${partition}:ec2:*:*:security-group/*", + "Condition": { + "StringEquals": { + "ec2:CreateAction": "CreateSecurityGroup" + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:CreateTags", + "ec2:DeleteTags" + ], + "Resource": "arn:${partition}:ec2:*:*:security-group/*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:RevokeSecurityGroupIngress", + "ec2:DeleteSecurityGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:DeleteRule" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:${partition}:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:${partition}:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:${partition}:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "true", + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags", + "elasticloadbalancing:RemoveTags" + ], + "Resource": [ + "arn:${partition}:elasticloadbalancing:*:*:listener/net/*/*/*", + "arn:${partition}:elasticloadbalancing:*:*:listener/app/*/*/*", + "arn:${partition}:elasticloadbalancing:*:*:listener-rule/net/*/*/*", + "arn:${partition}:elasticloadbalancing:*:*:listener-rule/app/*/*/*" + ] + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:DeleteTargetGroup" + ], + "Resource": "*", + "Condition": { + "Null": { + "aws:ResourceTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:AddTags" + ], + "Resource": [ + "arn:${partition}:elasticloadbalancing:*:*:targetgroup/*/*", + "arn:${partition}:elasticloadbalancing:*:*:loadbalancer/net/*/*", + "arn:${partition}:elasticloadbalancing:*:*:loadbalancer/app/*/*" + ], + "Condition": { + "StringEquals": { + "elasticloadbalancing:CreateAction": [ + "CreateTargetGroup", + "CreateLoadBalancer" + ] + }, + "Null": { + "aws:RequestTag/elbv2.k8s.aws/cluster": "false" + } + } + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:DeregisterTargets" + ], + "Resource": "arn:${partition}:elasticloadbalancing:*:*:targetgroup/*/*" + }, + { + "Effect": "Allow", + "Action": [ + "elasticloadbalancing:SetWebAcl", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:ModifyRule" + ], + "Resource": "*" + } + ] +} \ No newline at end of file diff --git a/modules/eks/submodules/privatelink/iam.tf b/modules/eks/submodules/privatelink/iam.tf new file mode 100644 index 00000000..c7631a1a --- /dev/null +++ b/modules/eks/submodules/privatelink/iam.tf @@ -0,0 +1,51 @@ +locals { + oidc_provider_prefix = replace(var.oidc_provider_id, "/arn:.*:oidc-provider//", "") +} + +data "aws_iam_policy_document" "load_balancer_controller" { + statement { + sid = "" + effect = "Allow" + actions = ["sts:AssumeRoleWithWebIdentity"] + + condition { + test = "StringEquals" + variable = "${local.oidc_provider_prefix}:sub" + values = ["system:serviceaccount:${var.privatelink.namespace}:${var.deploy_id}-load-balancer-controller"] + } + + condition { + test = "StringEquals" + variable = "${local.oidc_provider_prefix}:aud" + values = ["sts.amazonaws.com"] + } + + principals { + type = "Federated" + identifiers = [var.oidc_provider_id] + } + } +} + +data "aws_iam_policy_document" "load_balancer_controller_policy" { + source_policy_documents = [ + templatefile("${path.module}/aws-load-balancer-controller_2.5.4_iam_policy.json", { + partition = data.aws_partition.current.partition + }) + ] +} + +resource "aws_iam_role" "load_balancer_controller" { + name = "${var.deploy_id}-load-balancer-controller" + assume_role_policy = data.aws_iam_policy_document.load_balancer_controller.json +} + +resource "aws_iam_policy" "load_balancer_controller" { + name = "${var.deploy_id}-load-balancer-controller" + policy = data.aws_iam_policy_document.load_balancer_controller_policy.json +} + +resource "aws_iam_role_policy_attachment" "load_balancer_controller" { + role = aws_iam_role.load_balancer_controller.name + policy_arn = aws_iam_policy.load_balancer_controller.arn +} \ No newline at end of file diff --git a/modules/eks/submodules/privatelink/main.tf b/modules/eks/submodules/privatelink/main.tf new file mode 100644 index 00000000..d9adb625 --- /dev/null +++ b/modules/eks/submodules/privatelink/main.tf @@ -0,0 +1 @@ +data "aws_partition" "current" {} \ No newline at end of file diff --git a/modules/eks/submodules/privatelink/outputs.tf b/modules/eks/submodules/privatelink/outputs.tf new file mode 100644 index 00000000..63b36769 --- /dev/null +++ b/modules/eks/submodules/privatelink/outputs.tf @@ -0,0 +1,6 @@ +output "info" { + description = "Target groups..." + value = { + target_groups = { for k, v in aws_lb_target_group.target_groups : k => v.arn } + } +} \ No newline at end of file diff --git a/modules/eks/submodules/privatelink/variables.tf b/modules/eks/submodules/privatelink/variables.tf new file mode 100644 index 00000000..b9312637 --- /dev/null +++ b/modules/eks/submodules/privatelink/variables.tf @@ -0,0 +1,100 @@ +variable "deploy_id" { + type = string + description = "Domino Deployment ID" + + validation { + condition = can(regex("^[a-z-0-9]{3,32}$", var.deploy_id)) + error_message = "Argument deploy_id must: start with a letter, contain lowercase alphanumeric characters(can contain hyphens[-]) with length between 3 and 32 characters." + } +} + +variable "region" { + type = string + description = "AWS region for the deployment" + nullable = false + validation { + condition = can(regex("(us(-gov)?|ap|ca|cn|eu|sa|me|af|il)-(central|(north|south)?(east|west)?)-[0-9]", var.region)) + error_message = "The provided region must follow the format of AWS region names, e.g., us-west-2, us-gov-west-1." + } +} + +variable "network_info" { + description = < service.private_dns } + + listeners = distinct(flatten([ + for service in var.privatelink.vpc_endpoint_services : [ + for port in service.ports : { + service = service.name + port = port + cert_arn = service.cert_arn + } + ] + ])) +} + +data "aws_route53_zone" "hosted" { + name = var.privatelink.route53_hosted_zone_name + private_zone = false +} + +resource "aws_lb" "nlbs" { + for_each = local.endpoint_services + + name = "${var.deploy_id}-${each.key}" + load_balancer_type = "network" + + subnets = [for subnet in var.network_info.subnets.public : subnet.subnet_id] + + enable_deletion_protection = false + enable_cross_zone_load_balancing = true + + access_logs { + bucket = var.privatelink.monitoring_bucket + enabled = true + } +} + +resource "aws_lb_target_group" "target_groups" { + for_each = local.endpoint_services + + name = "${var.deploy_id}-${each.key}" + port = 80 # Not used but required + protocol = "TCP" + vpc_id = var.network_info.vpc_id +} + +resource "aws_lb_listener" "listeners" { + for_each = { for entry in local.listeners : "${entry.service}.${entry.port}" => entry } + + load_balancer_arn = aws_lb.nlbs[each.value.service].arn + port = each.value.port + protocol = "TLS" + ssl_policy = "ELBSecurityPolicy-TLS-1-2-2017-01" + certificate_arn = each.value.cert_arn + + default_action { + type = "forward" + target_group_arn = aws_lb_target_group.target_groups[each.value.service].arn + } +} + +resource "aws_vpc_endpoint_service" "vpc_endpoint_services" { + for_each = local.endpoint_services + + acceptance_required = false + network_load_balancer_arns = [aws_lb.nlbs[each.key].arn] + + private_dns_name = each.value + + tags = { + "Name" = "${var.deploy_id}-${each.key}" + } +} + +resource "aws_route53_record" "service_endpoint_private_dns_verification" { + for_each = local.endpoint_services + + zone_id = data.aws_route53_zone.hosted.zone_id + name = each.value + type = "TXT" + ttl = 1800 + records = [ + aws_vpc_endpoint_service.vpc_endpoint_services[each.key].private_dns_name_configuration[0].value + ] +} \ No newline at end of file diff --git a/submodules/eks/variables.tf b/modules/eks/variables.tf similarity index 67% rename from submodules/eks/variables.tf rename to modules/eks/variables.tf index 050efc45..ab7f2b7e 100644 --- a/submodules/eks/variables.tf +++ b/modules/eks/variables.tf @@ -18,29 +18,6 @@ variable "region" { } } -variable "node_groups" { - description = "EKS managed node groups definition." - type = map(object({ - ami = optional(string, null) - bootstrap_extra_args = optional(string, "") - instance_types = list(string) - spot = optional(bool, false) - min_per_az = number - max_per_az = number - desired_per_az = number - availability_zone_ids = list(string) - labels = map(string) - taints = optional(list(object({ key = string, value = optional(string), effect = string })), []) - tags = optional(map(string), {}) - instance_tags = optional(map(string), {}) - gpu = optional(bool, false) - volume = object({ - size = string - type = string - }) - })) -} - variable "network_info" { description = </kubeconfig' + path = Fully qualified path name to write the kubeconfig file. } public_access = { enabled = Enable EKS API public endpoint. cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint. } - List of Custom role maps for aws auth configmap - custom_role_maps = [{ + Custom role maps for aws auth configmap + custom_role_maps = { rolearn = string username = string groups = list(string) - }] - master_role_names = IAM role names to be added as masters in EKS. + } + master_role_names = IAM role names to be added as masters in eks. cluster_addons = EKS cluster addons. vpc-cni is installed separately. - ssm_log_group_name = "CloudWatch log group to send the SSM session logs to." vpc_cni = Configuration for AWS VPC CNI - identity_providers = "Configuration for IDP(Identity Provider)." + ssm_log_group_name = CloudWatch log group to send the SSM session logs to. + identity_providers = Configuration for IDP(Identity Provider). + } EOF type = object({ - k8s_version = optional(string) + creation_role_name = optional(string, null) + k8s_version = optional(string, "1.27") kubeconfig = optional(object({ - extra_args = optional(string) - path = optional(string) - })) + extra_args = optional(string, "") + path = optional(string, null) + }), {}) public_access = optional(object({ - enabled = optional(bool) - cidrs = optional(list(string)) - })) + enabled = optional(bool, false) + cidrs = optional(list(string), []) + }), {}) custom_role_maps = optional(list(object({ rolearn = string username = string groups = list(string) - }))) - master_role_names = optional(list(string)) - cluster_addons = optional(list(string)) - ssm_log_group_name = optional(string) + })), []) + master_role_names = optional(list(string), []) + cluster_addons = optional(list(string), ["kube-proxy", "coredns"]) + ssm_log_group_name = optional(string, "session-manager") vpc_cni = optional(object({ prefix_delegation = optional(bool) })) @@ -197,19 +177,7 @@ variable "eks" { })), []) }) - validation { - condition = var.eks.public_access.enabled ? length(var.eks.public_access.cidrs) > 0 : true - error_message = "eks.public_access.cidrs must be configured when public access is enabled" - } - - validation { - condition = !var.eks.public_access.enabled ? true : alltrue([ - for cidr in var.eks.public_access.cidrs : - try(cidrhost(cidr, 0), null) == regex("^(.*)/", cidr)[0] && - try(cidrnetmask(cidr), null) != null - ]) - error_message = "All elements in eks.public_access.cidrs list must be valid CIDR blocks" - } + default = {} } variable "ssh_key" { @@ -222,3 +190,52 @@ variable "ssh_key" { key_pair_name = string }) } + +variable "create_eks_role_arn" { + description = "Role arn to assume during the EKS cluster creation." + type = string +} + +variable "tags" { + type = map(string) + description = "Deployment tags." + default = {} +} + +variable "privatelink" { + description = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | < 5.17 | +| [local](#requirement\_local) | >= 2.2.0 | +| [time](#requirement\_time) | >= 0.9.1 | +| [tls](#requirement\_tls) | >= 3.4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | < 5.17 | +| [time](#provider\_time) | >= 0.9.1 | +| [tls](#provider\_tls) | >= 3.4.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [bastion](#module\_bastion) | ./submodules/bastion | n/a | +| [network](#module\_network) | ./submodules/network | n/a | +| [storage](#module\_storage) | ./submodules/storage | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_iam_policy.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_role.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | +| [aws_iam_role_policy_attachment.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_key_pair.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/key_pair) | resource | +| [aws_kms_alias.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_alias) | resource | +| [aws_kms_key.domino](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [time_sleep.create_eks_role_30_seconds](https://registry.terraform.io/providers/hashicorp/time/latest/docs/resources/sleep) | resource | +| [aws_caller_identity.aws_account](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | +| [aws_default_tags.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | +| [aws_ec2_instance_type.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_iam_policy_document.create_eks_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.kms_key_global](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.route53](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_kms_key.key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/kms_key) | data source | +| [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | +| [aws_route53_zone.hosted](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/route53_zone) | data source | +| [tls_public_key.domino](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/data-sources/public_key) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })), [])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | +| [bastion](#input\_bastion) | enabled = Create bastion host.
ami = Ami id. Defaults to latest 'amazon\_linux\_2' ami.
instance\_type = Instance type.
authorized\_ssh\_ip\_ranges = List of CIDR ranges permitted for the bastion ssh access.
username = Bastion user.
install\_binaries = Toggle to install required Domino binaries in the bastion. | 
object({
    enabled                  = optional(bool, true)
    ami_id                   = optional(string, null) # default will use the latest 'amazon_linux_2' ami
    instance_type            = optional(string, "t3.micro")
    authorized_ssh_ip_ranges = optional(list(string), ["0.0.0.0/0"])
    username                 = optional(string, "ec2-user")
    install_binaries         = optional(bool, false)
  })
| `{}` | no | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      })
  })
| n/a | yes | +| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks"` | no | +| [eks](#input\_eks) | creation\_role\_name = Name of the role to import.
k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} | 
object({
    creation_role_name = optional(string, null)
    k8s_version        = optional(string, "1.27")
    kubeconfig = optional(object({
      extra_args = optional(string, "")
      path       = optional(string, null)
    }), {})
    public_access = optional(object({
      enabled = optional(bool, false)
      cidrs   = optional(list(string), [])
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })), [])
    master_role_names  = optional(list(string), [])
    cluster_addons     = optional(list(string), ["kube-proxy", "coredns"])
    ssm_log_group_name = optional(string, "session-manager")
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string, null)
      groups_prefix                 = optional(string, null)
      identity_provider_config_name = string
      issuer_url                    = optional(string, null)
      required_claims               = optional(string, null)
      username_claim                = optional(string, null)
      username_prefix               = optional(string, null)
    })), [])
  })
| `{}` | no | +| [kms](#input\_kms) | enabled = "Toggle, if set use either the specified KMS key\_id or a Domino-generated one"
key\_id = optional(string, null)
additional\_policies = "Allows setting additional KMS key policies when using a Domino-generated key" | 
object({
    enabled             = optional(bool, true)
    key_id              = optional(string, null)
    additional_policies = optional(list(string), [])
  })
| `{}` | no | +| [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. | 
object({
    vpc = optional(object({
      id = optional(string, null)
      subnets = optional(object({
        private = optional(list(string), [])
        public  = optional(list(string), [])
        pod     = optional(list(string), [])
      }), {})
    }), {})
    network_bits = optional(object({
      public  = optional(number, 27)
      private = optional(number, 19)
      pod     = optional(number, 19)
      }
    ), {})
    cidrs = optional(object({
      vpc = optional(string, "10.0.0.0/16")
      pod = optional(string, "100.64.0.0/16")
    }), {})
    use_pod_cidr = optional(bool, true)
  })
| `{}` | no | +| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | +| [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNS zone. | `string` | `null` | no | +| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | +| [storage](#input\_storage) | storage = {
efs = {
access\_point\_path = Filesystem path for efs.
backup\_vault = {
create = Create backup vault for EFS toggle.
force\_destroy = Toggle to allow automatic destruction of all backups when destroying.
backup = {
schedule = Cron-style schedule for EFS backup vault (default: once a day at 12pm).
cold\_storage\_after = Move backup data to cold storage after this many days.
delete\_after = Delete backup data after this many days.
}
}
}
s3 = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets.
}
ecr = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the ECR repositories. if 'false' terraform will NOT be able to delete non-empty repositories.
}
}
} | 
object({
    efs = optional(object({
      access_point_path = optional(string, "/domino")
      backup_vault = optional(object({
        create        = optional(bool, true)
        force_destroy = optional(bool, true)
        backup = optional(object({
          schedule           = optional(string, "0 12 * * ? *")
          cold_storage_after = optional(number, 35)
          delete_after       = optional(number, 125)
        }), {})
      }), {})
    }), {})
    s3 = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
    ecr = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [additional\_node\_groups](#output\_additional\_node\_groups) | Additional EKS managed node groups definition. | +| [bastion](#output\_bastion) | Bastion details, if it was created. | +| [create\_eks\_role\_arn](#output\_create\_eks\_role\_arn) | Role arn to assume during the EKS cluster creation. | +| [default\_node\_groups](#output\_default\_node\_groups) | Default nodegroups. | +| [deploy\_id](#output\_deploy\_id) | Domino Deployment ID. | +| [domino\_key\_pair](#output\_domino\_key\_pair) | Domino key pair | +| [efs\_security\_group](#output\_efs\_security\_group) | Security Group ID for EFS | +| [eks](#output\_eks) | EKS variables. | +| [hostname](#output\_hostname) | Domino instance URL. | +| [kms](#output\_kms) | KMS key details, if enabled. | +| [monitoring\_bucket](#output\_monitoring\_bucket) | Monitoring Bucket | +| [network](#output\_network) | Network details. | +| [node\_iam\_policies](#output\_node\_iam\_policies) | Policies attached to EKS nodes role | +| [region](#output\_region) | Deployment Region. | +| [ssh\_key](#output\_ssh\_key) | SSH key path,name. | +| [storage](#output\_storage) | Storage details. | +| [tags](#output\_tags) | Deployment tags. | + diff --git a/iam.tf b/modules/infra/iam.tf similarity index 90% rename from iam.tf rename to modules/infra/iam.tf index 4341356e..ba149ebb 100644 --- a/iam.tf +++ b/modules/infra/iam.tf @@ -32,15 +32,10 @@ resource "aws_iam_policy" "route53" { policy = data.aws_iam_policy_document.route53[0].json } -resource "aws_iam_role_policy_attachment" "route53" { - count = var.route53_hosted_zone_name != null ? length(module.eks.info.nodes.roles) : 0 - policy_arn = aws_iam_policy.route53[0].arn - role = lookup(module.eks.info.nodes.roles[count.index], "name") -} locals { - create_eks_role_name = "${var.deploy_id}-create-eks" + create_eks_role_name = coalesce(var.eks.creation_role_name, "${var.deploy_id}-create-eks") } data "aws_iam_policy_document" "create_eks_role" { @@ -86,7 +81,6 @@ data "aws_iam_policy_document" "create_eks_role" { resources = ["arn:${data.aws_partition.current.partition}:iam::${local.aws_account_id}:role/aws-service-role/*"] effect = "Allow" } - } resource "aws_iam_policy" "create_eks_role" { diff --git a/kms.tf b/modules/infra/kms.tf similarity index 100% rename from kms.tf rename to modules/infra/kms.tf diff --git a/main.tf b/modules/infra/main.tf similarity index 73% rename from main.tf rename to modules/infra/main.tf index eb2a4e97..7c320aa5 100644 --- a/main.tf +++ b/modules/infra/main.tf @@ -9,8 +9,6 @@ locals { key_arn = local.kms_key.arn enabled = var.kms.enabled } - bastion_info = var.bastion.enabled ? module.bastion[0].info : null - } module "storage" { @@ -21,6 +19,10 @@ module "storage" { storage = var.storage } +data "aws_ec2_instance_type" "all" { + for_each = toset(flatten([for ng in merge(var.additional_node_groups, var.default_node_groups) : ng.instance_types])) + instance_type = each.value +} locals { node_groups = { for name, ng in @@ -77,31 +79,8 @@ module "bastion" { bastion = var.bastion } -data "aws_ec2_instance_type" "all" { - for_each = toset(flatten([for ng in merge(var.additional_node_groups, var.default_node_groups) : ng.instance_types])) - instance_type = each.value -} - -module "eks" { - source = "./submodules/eks" - deploy_id = var.deploy_id - region = var.region - ssh_key = local.ssh_key - node_groups = local.node_groups - node_iam_policies = [module.storage.info.s3.iam_policy_arn, module.storage.info.ecr.iam_policy_arn] - efs_security_group = module.storage.info.efs.security_group_id - eks = var.eks - network_info = module.network.info - kms_info = local.kms_info - bastion_info = local.bastion_info - - depends_on = [ - aws_iam_role.create_eks_role, - aws_iam_policy.create_eks_role, - aws_iam_role_policy_attachment.create_eks_role - ] - - providers = { - aws.eks = aws.eks - } +locals { + bastion_info = var.bastion.enabled && length(module.bastion) > 0 ? module.bastion[0].info : null + node_iam_policies_storage = [module.storage.info.s3.iam_policy_arn, module.storage.info.ecr.iam_policy_arn] + node_iam_policies = var.route53_hosted_zone_name != null ? concat(local.node_iam_policies_storage, [aws_iam_policy.route53[0].arn]) : local.node_iam_policies_storage } diff --git a/modules/infra/outputs.tf b/modules/infra/outputs.tf new file mode 100644 index 00000000..162ee1e7 --- /dev/null +++ b/modules/infra/outputs.tf @@ -0,0 +1,82 @@ +output "hostname" { + description = "Domino instance URL." + value = try("${var.deploy_id}.${var.route53_hosted_zone_name}", null) +} +output "domino_key_pair" { + description = "Domino key pair" + value = { name = aws_key_pair.domino.key_name } +} + +output "kms" { + description = "KMS key details, if enabled." + value = local.kms_info +} + +output "network" { + description = "Network details." + value = module.network.info +} + +output "bastion" { + description = "Bastion details, if it was created." + value = local.bastion_info +} + +output "storage" { + description = "Storage details." + value = module.storage.info +} + +output "tags" { + description = "Deployment tags." + value = var.tags +} + +output "deploy_id" { + description = "Domino Deployment ID." + value = var.deploy_id +} + +output "region" { + description = "Deployment Region." + value = var.region +} +output "eks" { + description = "EKS variables." + value = var.eks +} + +output "ssh_key" { + description = "SSH key path,name." + value = local.ssh_key +} + +output "additional_node_groups" { + description = "Additional EKS managed node groups definition." + value = var.additional_node_groups +} + +output "default_node_groups" { + description = "Default nodegroups." + value = var.default_node_groups +} + +output "efs_security_group" { + description = "Security Group ID for EFS" + value = module.storage.info.efs.security_group_id +} + +output "node_iam_policies" { + description = "Policies attached to EKS nodes role" + value = local.node_iam_policies +} + +output "create_eks_role_arn" { + description = "Role arn to assume during the EKS cluster creation." + value = aws_iam_role.create_eks_role.arn +} + +output "monitoring_bucket" { + description = "Monitoring Bucket" + value = module.storage.info.s3.buckets.monitoring.bucket_name +} \ No newline at end of file diff --git a/submodules/bastion/README.md b/modules/infra/submodules/bastion/README.md similarity index 100% rename from submodules/bastion/README.md rename to modules/infra/submodules/bastion/README.md diff --git a/submodules/bastion/main.tf b/modules/infra/submodules/bastion/main.tf similarity index 100% rename from submodules/bastion/main.tf rename to modules/infra/submodules/bastion/main.tf diff --git a/submodules/bastion/outputs.tf b/modules/infra/submodules/bastion/outputs.tf similarity index 100% rename from submodules/bastion/outputs.tf rename to modules/infra/submodules/bastion/outputs.tf diff --git a/submodules/bastion/templates/install-binaries.sh.tftpl b/modules/infra/submodules/bastion/templates/install-binaries.sh.tftpl similarity index 100% rename from submodules/bastion/templates/install-binaries.sh.tftpl rename to modules/infra/submodules/bastion/templates/install-binaries.sh.tftpl diff --git a/submodules/bastion/variables.tf b/modules/infra/submodules/bastion/variables.tf similarity index 100% rename from submodules/bastion/variables.tf rename to modules/infra/submodules/bastion/variables.tf diff --git a/submodules/bastion/versions.tf b/modules/infra/submodules/bastion/versions.tf similarity index 100% rename from submodules/bastion/versions.tf rename to modules/infra/submodules/bastion/versions.tf diff --git a/submodules/network/README.md b/modules/infra/submodules/network/README.md similarity index 100% rename from submodules/network/README.md rename to modules/infra/submodules/network/README.md diff --git a/submodules/network/internet-gateway.tf b/modules/infra/submodules/network/internet-gateway.tf similarity index 100% rename from submodules/network/internet-gateway.tf rename to modules/infra/submodules/network/internet-gateway.tf diff --git a/submodules/network/main.tf b/modules/infra/submodules/network/main.tf similarity index 99% rename from submodules/network/main.tf rename to modules/infra/submodules/network/main.tf index dc49e81a..15c8b2a9 100644 --- a/submodules/network/main.tf +++ b/modules/infra/submodules/network/main.tf @@ -23,7 +23,6 @@ locals { az_ids = local.provided_vpc ? distinct(data.aws_subnet.private[*].availability_zone_id) : distinct(flatten([for name, ng in var.node_groups : ng.availability_zone_ids])) num_of_azs = length(local.az_ids) - ## Calculating public and private subnets based on the base base cidr and desired network bits base_cidr_network_bits = tonumber(regex("[^/]*$", var.network.cidrs.vpc)) ## We have one Cidr to carve the nw bits for both pvt and public subnets diff --git a/submodules/network/nat-gateway.tf b/modules/infra/submodules/network/nat-gateway.tf similarity index 100% rename from submodules/network/nat-gateway.tf rename to modules/infra/submodules/network/nat-gateway.tf diff --git a/submodules/network/outputs.tf b/modules/infra/submodules/network/outputs.tf similarity index 92% rename from submodules/network/outputs.tf rename to modules/infra/submodules/network/outputs.tf index be0ee70f..c72a30ec 100644 --- a/submodules/network/outputs.tf +++ b/modules/infra/submodules/network/outputs.tf @@ -1,6 +1,7 @@ output "info" { description = "Nework information. vpc_id, subnets..." value = { + az_ids = local.az_ids vpc_id = local.create_vpc ? aws_vpc.this[0].id : data.aws_vpc.provided[0].id subnets = { public = local.public_subnets diff --git a/submodules/network/route-tables.tf b/modules/infra/submodules/network/route-tables.tf similarity index 100% rename from submodules/network/route-tables.tf rename to modules/infra/submodules/network/route-tables.tf diff --git a/submodules/network/subnets.tf b/modules/infra/submodules/network/subnets.tf similarity index 100% rename from submodules/network/subnets.tf rename to modules/infra/submodules/network/subnets.tf diff --git a/submodules/network/variables.tf b/modules/infra/submodules/network/variables.tf similarity index 100% rename from submodules/network/variables.tf rename to modules/infra/submodules/network/variables.tf diff --git a/submodules/network/versions.tf b/modules/infra/submodules/network/versions.tf similarity index 100% rename from submodules/network/versions.tf rename to modules/infra/submodules/network/versions.tf diff --git a/submodules/network/vpc.tf b/modules/infra/submodules/network/vpc.tf similarity index 100% rename from submodules/network/vpc.tf rename to modules/infra/submodules/network/vpc.tf diff --git a/submodules/storage/README.md b/modules/infra/submodules/storage/README.md similarity index 100% rename from submodules/storage/README.md rename to modules/infra/submodules/storage/README.md diff --git a/submodules/storage/ecr.tf b/modules/infra/submodules/storage/ecr.tf similarity index 100% rename from submodules/storage/ecr.tf rename to modules/infra/submodules/storage/ecr.tf diff --git a/submodules/storage/efs.tf b/modules/infra/submodules/storage/efs.tf similarity index 100% rename from submodules/storage/efs.tf rename to modules/infra/submodules/storage/efs.tf diff --git a/submodules/storage/efs_backup_vault.tf b/modules/infra/submodules/storage/efs_backup_vault.tf similarity index 99% rename from submodules/storage/efs_backup_vault.tf rename to modules/infra/submodules/storage/efs_backup_vault.tf index 7900ac3e..63c582cb 100644 --- a/submodules/storage/efs_backup_vault.tf +++ b/modules/infra/submodules/storage/efs_backup_vault.tf @@ -10,6 +10,8 @@ resource "aws_backup_vault" "efs" { kms_key_arn, ] } + + } resource "aws_backup_plan" "efs" { diff --git a/submodules/storage/iam.tf b/modules/infra/submodules/storage/iam.tf similarity index 100% rename from submodules/storage/iam.tf rename to modules/infra/submodules/storage/iam.tf diff --git a/submodules/storage/main.tf b/modules/infra/submodules/storage/main.tf similarity index 100% rename from submodules/storage/main.tf rename to modules/infra/submodules/storage/main.tf diff --git a/submodules/storage/outputs.tf b/modules/infra/submodules/storage/outputs.tf similarity index 100% rename from submodules/storage/outputs.tf rename to modules/infra/submodules/storage/outputs.tf diff --git a/submodules/storage/s3.tf b/modules/infra/submodules/storage/s3.tf similarity index 99% rename from submodules/storage/s3.tf rename to modules/infra/submodules/storage/s3.tf index e5030b53..25972f27 100644 --- a/submodules/storage/s3.tf +++ b/modules/infra/submodules/storage/s3.tf @@ -73,7 +73,6 @@ resource "aws_s3_bucket" "blobs" { bucket = "${var.deploy_id}-blobs" force_destroy = var.storage.s3.force_destroy_on_deletion object_lock_enabled = false - } data "aws_iam_policy_document" "blobs" { diff --git a/submodules/storage/variables.tf b/modules/infra/submodules/storage/variables.tf similarity index 100% rename from submodules/storage/variables.tf rename to modules/infra/submodules/storage/variables.tf diff --git a/submodules/storage/versions.tf b/modules/infra/submodules/storage/versions.tf similarity index 100% rename from submodules/storage/versions.tf rename to modules/infra/submodules/storage/versions.tf diff --git a/variables.tf b/modules/infra/variables.tf similarity index 90% rename from variables.tf rename to modules/infra/variables.tf index 25ef3c62..2063e2fe 100644 --- a/variables.tf +++ b/modules/infra/variables.tf @@ -16,13 +16,13 @@ variable "deploy_id" { validation { condition = length(var.deploy_id) >= 3 && length(var.deploy_id) <= 32 && can(regex("^[a-z]([-a-z0-9]*[a-z0-9])$", var.deploy_id)) - error_message = < +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | ~> 5.0 | +| [null](#requirement\_null) | >= 3.1.0 | +| [tls](#requirement\_tls) | >= 4.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | +| [terraform](#provider\_terraform) | n/a | + +## Modules + +No modules. + +## Resources + +| Name | Type | +|------|------| +| [aws_autoscaling_group_tag.tag](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group_tag) | resource | +| [aws_eks_addon.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_addon) | resource | +| [aws_eks_node_group.node_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/eks_node_group) | resource | +| [aws_launch_template.node_groups](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | +| [terraform_data.calico_setup](https://registry.terraform.io/providers/hashicorp/terraform/latest/docs/resources/data) | resource | +| [aws_ami.custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | +| [aws_default_tags.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/default_tags) | data source | +| [aws_ec2_instance_type.all](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type) | data source | +| [aws_ec2_instance_type_offerings.nodes](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ec2_instance_type_offerings) | data source | +| [aws_eks_addon_version.default](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | +| [aws_ssm_parameter.eks_ami_release_version](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | +| [aws_ssm_parameter.eks_gpu_ami_release_version](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ssm_parameter) | data source | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })), [])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      })
  })
| n/a | yes | +| [eks\_info](#input\_eks\_info) | cluster = {
addons = List of addons
specs = Cluster spes. {
name = Cluster name.
endpoint = Cluster endpont.
kubernetes\_network\_config = Cluster k8s nw config.
}
version = K8s version.
arn = EKS Cluster arn.
security\_group\_id = EKS Cluster security group id.
endpoint = EKS Cluster API endpoint.
roles = Default IAM Roles associated with the EKS cluster. {
name = string
arn = string
}
custom\_roles = Custom IAM Roles associated with the EKS cluster. {
rolearn = string
username = string
groups = list(string)
}
oidc = {
arn = OIDC provider ARN.
url = OIDC provider url.
}
}
nodes = {
security\_group\_id = EKS Nodes security group id.
roles = IAM Roles associated with the EKS Nodes.{
name = string
arn = string
}
}
kubeconfig = Kubeconfig details.{
path = string
extra\_args = string
} | 
object({
    k8s_pre_setup_sh_file = string
    cluster = object({
      addons = list(string)
      specs = object({
        name                      = string
        endpoint                  = string
        kubernetes_network_config = list(map(any))
        certificate_authority     = list(map(any))
      })
      version           = string
      arn               = string
      security_group_id = string
      endpoint          = string
      roles = list(object({
        name = string
        arn  = string
      }))
      custom_roles = list(object({
        rolearn  = string
        username = string
        groups   = list(string)
      }))
      oidc = object({
        arn = string
        url = string
      })
    })
    nodes = object({
      security_group_id = string
      roles = list(object({
        name = string
        arn  = string
      }))
    })
    kubeconfig = object({
      path       = string
      extra_args = string
    })
  })
| n/a | yes | +| [kms\_info](#input\_kms\_info) | key\_id = KMS key id.
key\_arn = KMS key arn.
enabled = KMS key is enabled | 
object({
    key_id  = string
    key_arn = string
    enabled = bool
  })
| n/a | yes | +| [network\_info](#input\_network\_info) | id = VPC ID.
subnets = {
public = List of public Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
private = List of private Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
pod = List of pod Subnets.
[{
name = Subnet name.
subnet\_id = Subnet ud
az = Subnet availability\_zone
az\_id = Subnet availability\_zone\_id
}]
} | 
object({
    vpc_id = string
    subnets = object({
      public = list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      }))
      private = optional(list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      })), [])
      pod = optional(list(object({
        name      = string
        subnet_id = string
        az        = string
        az_id     = string
      })), [])
    })
  })
| n/a | yes | +| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | +| [ssh\_key](#input\_ssh\_key) | path = SSH private key filepath.
key\_pair\_name = AWS key\_pair name. | 
object({
    path          = string
    key_pair_name = string
  })
| n/a | yes | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [info](#output\_info) | Nodes details. | + diff --git a/modules/nodes/main.tf b/modules/nodes/main.tf new file mode 100644 index 00000000..8bcef6d4 --- /dev/null +++ b/modules/nodes/main.tf @@ -0,0 +1,91 @@ +data "aws_default_tags" "this" {} + +data "aws_ec2_instance_type" "all" { + for_each = toset(flatten([for ng in merge(var.additional_node_groups, var.default_node_groups) : ng.instance_types])) + instance_type = each.value +} + +locals { + node_groups = { + for name, ng in + merge(var.additional_node_groups, var.default_node_groups) : + name => merge(ng, { + gpu = ng.gpu != null ? ng.gpu : anytrue([for itype in ng.instance_types : length(data.aws_ec2_instance_type.all[itype].gpus) > 0]), + instance_tags = merge(data.aws_default_tags.this.tags, ng.tags) + }) + } +} + +data "aws_ec2_instance_type_offerings" "nodes" { + for_each = { + for name, ng in local.node_groups : + name => ng.instance_types + } + + filter { + name = "instance-type" + values = each.value + } + + location_type = "availability-zone-id" +} + +data "aws_ami" "custom" { + for_each = toset([for k, v in local.node_groups : v.ami if v.ami != null]) + + filter { + name = "image-id" + values = [each.value] + } +} + +locals { + node_groups_per_zone = flatten([ + for ng_name, ng in local.node_groups : [ + for sb_name, sb in var.network_info.subnets.private : { + ng_name = ng_name + sb_name = sb_name + subnet = sb + node_group = ng + } if contains(ng.availability_zone_ids, sb.az_id) + ] + ]) + node_groups_by_name = { for ngz in local.node_groups_per_zone : "${ngz.ng_name}-${ngz.sb_name}" => ngz } +} + + +data "aws_eks_addon_version" "default" { + for_each = toset(var.eks_info.cluster.addons) + addon_name = each.key + kubernetes_version = var.eks_info.cluster.version +} + +resource "aws_eks_addon" "this" { + for_each = toset(var.eks_info.cluster.addons) + cluster_name = var.eks_info.cluster.specs.name + addon_name = each.key + addon_version = data.aws_eks_addon_version.default[each.key].version + resolve_conflicts_on_create = "OVERWRITE" + resolve_conflicts_on_update = "OVERWRITE" + depends_on = [ + aws_eks_node_group.node_groups, + ] +} + + + +resource "terraform_data" "calico_setup" { + count = try(fileexists(var.eks_info.k8s_pre_setup_sh_file), false) ? 1 : 0 + + triggers_replace = [ + filemd5(var.eks_info.k8s_pre_setup_sh_file) + ] + + provisioner "local-exec" { + command = "bash ./${basename(var.eks_info.k8s_pre_setup_sh_file)} install_calico" + interpreter = ["bash", "-c"] + working_dir = dirname(var.eks_info.k8s_pre_setup_sh_file) + } + + depends_on = [aws_eks_node_group.node_groups] +} diff --git a/submodules/eks/node-group.tf b/modules/nodes/nodes.tf similarity index 57% rename from submodules/eks/node-group.tf rename to modules/nodes/nodes.tf index 95a1a881..adc6c715 100644 --- a/submodules/eks/node-group.tf +++ b/modules/nodes/nodes.tf @@ -1,109 +1,7 @@ -# Validating zone offerings. - -# Check the zones where the instance types are being offered -data "aws_ec2_instance_type_offerings" "nodes" { - for_each = { - for name, ng in var.node_groups : - name => ng.instance_types - } - - filter { - name = "instance-type" - values = each.value - } - - location_type = "availability-zone-id" -} - -## EKS Nodes -data "aws_iam_policy_document" "eks_nodes" { - statement { - actions = ["sts:AssumeRole"] - - principals { - type = "Service" - identifiers = ["ec2.${local.dns_suffix}"] - } - } -} - -resource "aws_iam_role" "eks_nodes" { - name = "${local.eks_cluster_name}-eks-nodes" - assume_role_policy = data.aws_iam_policy_document.eks_nodes.json -} - -resource "aws_security_group" "eks_nodes" { - name = "${local.eks_cluster_name}-nodes" - description = "EKS cluster Nodes security group" - vpc_id = var.network_info.vpc_id - - lifecycle { - create_before_destroy = true - ignore_changes = [ - name, - description - ] - } - tags = { - "Name" = "${local.eks_cluster_name}-eks-nodes" - "kubernetes.io/cluster/${local.eks_cluster_name}" = "owned" - } -} - -resource "aws_security_group_rule" "node" { - for_each = local.node_security_group_rules - - # Required - security_group_id = aws_security_group.eks_nodes.id - protocol = each.value.protocol - from_port = each.value.from_port - to_port = each.value.to_port - type = each.value.type - description = each.value.description - cidr_blocks = try(each.value.cidr_blocks, null) - self = try(each.value.self, null) - source_security_group_id = try( - each.value.source_security_group_id, - try(each.value.source_cluster_security_group, false) ? aws_security_group.eks_cluster.id : null - ) -} - -resource "aws_security_group_rule" "efs" { - security_group_id = var.efs_security_group - protocol = "tcp" - from_port = 2049 - to_port = 2049 - type = "ingress" - description = "EFS access" - source_security_group_id = aws_security_group.eks_nodes.id -} - -locals { - node_groups_per_zone = flatten([ - for ng_name, ng in var.node_groups : [ - for sb_name, sb in var.network_info.subnets.private : { - ng_name = ng_name - sb_name = sb_name - subnet = sb - node_group = ng - } if contains(ng.availability_zone_ids, sb.az_id) - ] - ]) - node_groups_by_name = { for ngz in local.node_groups_per_zone : "${ngz.ng_name}-${ngz.sb_name}" => ngz } -} - -data "aws_ami" "custom" { - for_each = toset([for k, v in var.node_groups : v.ami if v.ami != null]) - - filter { - name = "image-id" - values = [each.value] - } -} resource "aws_launch_template" "node_groups" { - for_each = var.node_groups - name = "${local.eks_cluster_name}-${each.key}" + for_each = local.node_groups + name = "${var.eks_info.cluster.specs.name}-${each.key}" disable_api_termination = false key_name = var.ssh_key.key_pair_name user_data = each.value.ami == null ? null : base64encode(templatefile( @@ -111,16 +9,16 @@ resource "aws_launch_template" "node_groups" { { # https://docs.aws.amazon.com/eks/latest/userguide/launch-templates.html#launch-template-custom-ami # Required to bootstrap node - cluster_name = aws_eks_cluster.this.name - cluster_endpoint = aws_eks_cluster.this.endpoint - cluster_auth_base64 = aws_eks_cluster.this.certificate_authority[0].data + cluster_name = var.eks_info.cluster.specs.name + cluster_endpoint = var.eks_info.cluster.specs.endpoint + cluster_auth_base64 = var.eks_info.cluster.specs.certificate_authority[0].data # Optional - cluster_service_ipv4_cidr = aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr != null ? aws_eks_cluster.this.kubernetes_network_config[0].service_ipv4_cidr : "" + cluster_service_ipv4_cidr = var.eks_info.cluster.specs.kubernetes_network_config[0].service_ipv4_cidr != null ? var.eks_info.cluster.specs.kubernetes_network_config[0].service_ipv4_cidr : "" bootstrap_extra_args = each.value.bootstrap_extra_args pre_bootstrap_user_data = "" post_bootstrap_user_data = "" })) - vpc_security_group_ids = [aws_security_group.eks_nodes.id] + vpc_security_group_ids = [var.eks_info.nodes.security_group_id] image_id = each.value.ami block_device_mappings { @@ -131,7 +29,7 @@ resource "aws_launch_template" "node_groups" { encrypted = true volume_size = each.value.volume.size volume_type = each.value.volume.type - kms_key_id = var.kms_info.enabled ? local.kms_key_arn : null + kms_key_id = var.kms_info.enabled ? var.kms_info.key_arn : null } } @@ -148,17 +46,11 @@ resource "aws_launch_template" "node_groups" { content { resource_type = tag_specifications.value tags = merge(each.value.instance_tags, each.value.tags, { - "Name" = "${local.eks_cluster_name}-${each.key}" + "Name" = "${var.eks_info.cluster.specs.name}-${each.key}" }) } } - depends_on = [ - aws_security_group_rule.node, - aws_iam_role_policy_attachment.aws_eks_nodes, - aws_iam_role_policy_attachment.custom_eks_nodes, - ] - lifecycle { precondition { condition = length(setsubtract(each.value.availability_zone_ids, data.aws_ec2_instance_type_offerings.nodes[each.key].locations)) == 0 @@ -175,21 +67,20 @@ resource "aws_launch_template" "node_groups" { } data "aws_ssm_parameter" "eks_ami_release_version" { - name = "/aws/service/eks/optimized-ami/${aws_eks_cluster.this.version}/amazon-linux-2/recommended/release_version" + name = "/aws/service/eks/optimized-ami/${var.eks_info.cluster.version}/amazon-linux-2/recommended/release_version" } data "aws_ssm_parameter" "eks_gpu_ami_release_version" { - name = "/aws/service/eks/optimized-ami/${aws_eks_cluster.this.version}/amazon-linux-2-gpu/recommended/release_version" + name = "/aws/service/eks/optimized-ami/${var.eks_info.cluster.version}/amazon-linux-2-gpu/recommended/release_version" } resource "aws_eks_node_group" "node_groups" { - depends_on = [terraform_data.run_k8s_pre_setup] for_each = local.node_groups_by_name - cluster_name = aws_eks_cluster.this.name - version = each.value.node_group.ami != null ? null : aws_eks_cluster.this.version + cluster_name = var.eks_info.cluster.specs.name + version = each.value.node_group.ami != null ? null : var.eks_info.cluster.version release_version = each.value.node_group.ami != null ? null : (each.value.node_group.gpu ? nonsensitive(data.aws_ssm_parameter.eks_gpu_ami_release_version.value) : nonsensitive(data.aws_ssm_parameter.eks_ami_release_version.value)) - node_group_name = "${local.eks_cluster_name}-${each.key}" - node_role_arn = aws_iam_role.eks_nodes.arn + node_group_name = "${var.eks_info.cluster.specs.name}-${each.key}" + node_role_arn = var.eks_info.nodes.roles[0].arn subnet_ids = [each.value.subnet.subnet_id] force_update_version = true scaling_config { diff --git a/modules/nodes/outputs.tf b/modules/nodes/outputs.tf new file mode 100644 index 00000000..199743ad --- /dev/null +++ b/modules/nodes/outputs.tf @@ -0,0 +1,4 @@ +output "info" { + description = "Nodes details." + value = aws_eks_node_group.node_groups +} diff --git a/submodules/eks/templates/linux_user_data.tpl b/modules/nodes/templates/linux_user_data.tpl similarity index 100% rename from submodules/eks/templates/linux_user_data.tpl rename to modules/nodes/templates/linux_user_data.tpl diff --git a/modules/nodes/variables.tf b/modules/nodes/variables.tf new file mode 100644 index 00000000..46809108 --- /dev/null +++ b/modules/nodes/variables.tf @@ -0,0 +1,303 @@ + + +variable "ssh_key" { + description = < -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [aws](#requirement\_aws) | >= 4.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws.local](#provider\_aws.local) | >= 4.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | ./.. | n/a | - -## Resources - -| Name | Type | -|------|------| -| [aws_ami.eks_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | n/a | yes | -| [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | `"1.23"` | no | -| [region](#input\_region) | AWS region for the deployment | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Deployment tags. | `map(string)` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | EKS module outputs | - - - -## Requirements - -| Name | Version | -|------|---------| -| [terraform](#requirement\_terraform) | >= 1.3.0 | -| [aws](#requirement\_aws) | >= 4.0 | -| [random](#requirement\_random) | ~> 3.0 | - -## Providers - -| Name | Version | -|------|---------| -| [aws.local](#provider\_aws.local) | 4.33.0 | - -## Modules - -| Name | Source | Version | -|------|--------|---------| -| [domino\_eks](#module\_domino\_eks) | git@github.com:dominodatalab/terraform-aws-eks.git | n/a | - -## Resources - -| Name | Type | -|------|------| -| [aws_ami.eks_node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"dominotfeks"` | no | -| [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | `"1.25"` | no | -| [region](#input\_region) | AWS region for the deployment | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Deployment tags. | `map(string)` | 
{
  "deploy": "test"
}
| no | - -## Outputs - -| Name | Description | -|------|-------------| -| [domino\_eks](#output\_domino\_eks) | EKS module outputs | - diff --git a/tests/deploy/ci-deploy.sh b/tests/deploy/ci-deploy.sh new file mode 100755 index 00000000..6dc2fa43 --- /dev/null +++ b/tests/deploy/ci-deploy.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +set -euo pipefail + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +# ci vars +echo "Getting CI vars" +source "${SH_DIR}/meta.sh" + +# module vars +if test -f "${DEPLOY_DIR}/meta.sh"; then + echo "Getting module vars" + source "${DEPLOY_DIR}/meta.sh" +fi + +# remote module vars +BASE_REMOTE_SRC="github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git" +BASE_REMOTE_MOD_SRC="${BASE_REMOTE_SRC}//modules" + +deploy() { + component=${1:-all} + pushd "$DEPLOY_DIR" >/dev/null + for tf_cmd in 'init' 'validate' 'apply'; do + bash "./tf.sh" "$component" "$tf_cmd" || { + echo "Terraform $tf_cmd failed. Exiting..." + return 1 + } + done + popd >/dev/null 2>&1 +} + +set_ci_branch_name() { + if [[ "$CIRCLE_BRANCH" =~ ^pull/[0-9]+/head$ ]]; then + PR_NUMBER=$(echo "$CIRCLE_BRANCH" | sed -n 's/^pull\/\([0-9]*\)\/head/\1/p') + CI_BRANCH_NAME=$(curl -s \ + "https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/pulls/${PR_NUMBER}" | + jq -r .head.ref) + else + CI_BRANCH_NAME="$CIRCLE_BRANCH" + fi + export CI_BRANCH_NAME +} + +setup_modules() { + mkdir -p "$DEPLOY_DIR" + set_ci_branch_name + echo "Running init from module: -from-module=${BASE_REMOTE_SRC}//examples/deploy?ref=${CI_BRANCH_NAME} at: $DEPLOY_DIR" + terraform -chdir="$DEPLOY_DIR" init -backend=false -from-module="${BASE_REMOTE_SRC}//examples/deploy?ref=${CI_BRANCH_NAME}" +} + +install_helm() { + if [ -z "$HELM_VERSION" ]; then + echo "HELM_VERSION environment variable not set, exiting." + exit 1 + fi + echo "Installing Helm version: ${HELM_VERSION}" + curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 + chmod +x get_helm.sh + ./get_helm.sh --version "${HELM_VERSION}" + rm ./get_helm.sh + helm version --short +} + +install_hcledit() { + hcledit_version="${HCLEDIT_VERSION}" + hcledit_artifact=hcledit_${hcledit_version}_linux_amd64.tar.gz + curl -fsSL -o "${hcledit_artifact}" "https://github.com/minamijoyo/hcledit/releases/download/v${hcledit_version}/${hcledit_artifact}" + tar xvzf "${hcledit_artifact}" + sudo mv hcledit /usr/local/bin/ && rm "${hcledit_artifact}" && hcledit version +} + +set_eks_worker_ami() { + # We can potentially test AMI upgrades in CI. + # 1 is latest. + precedence="$1" + k8s_version="$(grep 'k8s_version' $INFRA_VARS_TPL | awk -F'"' '{print $2}')" + if ! aws sts get-caller-identity; then + echo "Incorrect AWS credentials." + exit 1 + fi + CUSTOM_AMI="$(aws ec2 describe-images --region us-west-2 --owners '602401143452' --filters "Name=owner-alias,Values=amazon" "Name=architecture,Values=x86_64" "Name=name,Values=amazon-eks-node-${k8s_version// /}*" --query "sort_by(Images, &CreationDate) | [-${precedence}].ImageId" --output text)" + export CUSTOM_AMI +} + +set_tf_vars() { + set_eks_worker_ami '1' + if [ -z $CUSTOM_AMI ]; then + echo 'CUSTOM_AMI is not set.' + # We want to test passing an ami. + exit 1 + fi + + [ -f "$PVT_KEY" ] || { ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f "$PVT_KEY" && chmod 600 "$PVT_KEY"; } + local default_nodes + + export CUSTOM_AMI PVT_KEY + default_nodes=$(envsubst <"$INFRA_VARS_TPL" | tee "$INFRA_VARS" | hcledit attribute get default_node_groups) + echo "default_node_groups = $default_nodes" >"$NODES_VARS" + + echo "Infra vars:" && cat "$INFRA_VARS" + echo "Cluster vars:" && cat "$CLUSTER_VARS" + echo "Nodes vars:" && cat "$NODES_VARS" +} + +# Not used atm, but we could test nodes upgrades(OS-patches). +deploy_latest_ami_nodes() { + ## Setting the default_node_groups.compute.ami to null ensures latest. + export CUSTOM_AMI=null + set_tf_vars + deploy 'nodes' +} + +destroy() { + component=${1:-all} + pushd "$DEPLOY_DIR" >/dev/null + bash "./tf.sh" "$component" destroy + popd >/dev/null 2>&1 +} + +set_mod_src() { + local mod_source="$1" + local tf_file="$2" + local name="$3" + + hcledit attribute set "module.${name}.source" "\"${mod_source}\"" -u -f "$tf_file" + cat "$tf_file" +} + +set_all_mod_src() { + local ref="$1" + local base_local_mod_src="./../../../../../modules" + + for dir in "${MOD_DIRS[@]}"; do + local name + if [[ "$dir" == *"cluster"* ]]; then + name="eks" + else + name="$(basename $dir)" + fi + if [ "$ref" == "local" ]; then + MOD_SOURCE="${base_local_mod_src}/${name}" + else + MOD_SOURCE="${BASE_REMOTE_MOD_SRC}/${name}?ref=${ref}" + fi + + echo "Setting module source to ref: ${MOD_SOURCE} on ${dir}" + set_mod_src "$MOD_SOURCE" "${dir}/main.tf" "$name" + done +} + +set_mod_src_circle_branch() { + set_ci_branch_name + set_all_mod_src "$CI_BRANCH_NAME" +} + +set_mod_src_local() { + echo "Updating module source to local." + set_all_mod_src "local" +} + +set_mod_src_latest_rel() { + echo "Updating module source to the latest published release." + latest_release_tag="$(curl -sSfL -H "X-GitHub-Api-Version: 2022-11-28" -H "Accept: application/vnd.github+json" https://api.github.com/repos/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}/releases/latest | jq -r '.tag_name')" + echo "Latest published release tag is: ${latest_release_tag}" + local ROOT_MOD_SOURCE="github.com/${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}.git?ref=${latest_release_tag}" + MAJOR_MOD_VERSION=$(echo "${latest_release_tag}" | awk -F'.' '{print $1}' | sed 's/^v//') + echo "export MAJOR_MOD_VERSION=$MAJOR_MOD_VERSION" >>$BASH_ENV + + if (($MAJOR_MOD_VERSION < 3)); then + echo "Legacy: Setting module source to: ${ROOT_MOD_SOURCE}" + cat <<<$(jq --arg mod_source "${ROOT_MOD_SOURCE}" '.module[0].domino_eks.source = $mod_source' "$LEGACY_TF") >"$LEGACY_TF" + else + set_all_mod_src "$latest_release_tag" + fi + +} + +for arg in "$@"; do + "$arg" +done diff --git a/tests/deploy/infra-ci.tfvars.tftpl b/tests/deploy/infra-ci.tfvars.tftpl new file mode 100644 index 00000000..e68bccb1 --- /dev/null +++ b/tests/deploy/infra-ci.tfvars.tftpl @@ -0,0 +1,69 @@ +deploy_id = "circleci-${CIRCLE_BUILD_NUM}" +region = "us-west-2" +route53_hosted_zone_name = "deploys-delta.domino.tech" +ssh_pvt_key_path = "${PVT_KEY}" + +default_node_groups = { + compute = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + ami = "${CUSTOM_AMI}" + instance_types = [ + "m4.2xlarge", + "m5.2xlarge", + "m5a.2xlarge", + "m5ad.2xlarge", + "m5d.2xlarge", + "m5dn.2xlarge", + "m5n.2xlarge", + "m5zn.2xlarge", + "m6id.2xlarge" + ] + spot = true + } + gpu = { + availability_zone_ids = ["usw2-az1", "usw2-az2"] + } + platform = { + "availability_zone_ids" = ["usw2-az1", "usw2-az2"] + } +} + +additional_node_groups = { + other_az = { + availability_zone_ids = ["usw2-az3"] + desired_per_az = 0 + instance_types = ["m5.2xlarge"] + labels = { + "dominodatalab.com/node-pool" = "other-az" + } + max_per_az = 10 + min_per_az = 0 + volume = { + size = 100 + type = "gp3" + } + } +} + +bastion = { + enabled = true + install_binaries = true +} + +eks = { + k8s_version = "1.27" + master_role_names = ["okta-poweruser", "okta-fulladmin"] +} + +kms = { + "enabled" = true +} + +tags = { + deploy_id = "circleci-${CIRCLE_BUILD_NUM}" + CIRCLE_BUILD_URL = "${CIRCLE_BUILD_URL}" + CIRCLE_BRANCH = "${CIRCLE_BRANCH}" + CIRCLE_JOB = "${CIRCLE_JOB}" + CIRCLE_REPOSITORY_URL = "${CIRCLE_REPOSITORY_URL}" + CIRCLE_BUILD_NUM = "${CIRCLE_BUILD_NUM}" +} diff --git a/tests/ci.tfvars.tftpl b/tests/deploy/legacy-test/ci.tfvars.tftpl similarity index 63% rename from tests/ci.tfvars.tftpl rename to tests/deploy/legacy-test/ci.tfvars.tftpl index 284af7d4..e9894a37 100644 --- a/tests/ci.tfvars.tftpl +++ b/tests/deploy/legacy-test/ci.tfvars.tftpl @@ -1,6 +1,7 @@ -deploy_id = "${WORKSPACE}" +deploy_id = "circleci-${CIRCLE_BUILD_NUM}" +ssh_pvt_key_path = "${LEGACY_PVT_KEY}" tags = { - deploy_id = "${WORKSPACE}" + deploy_id = "circleci-${CIRCLE_BUILD_NUM}" CIRCLE_BUILD_URL = "${CIRCLE_BUILD_URL}" CIRCLE_BRANCH = "${CIRCLE_BRANCH}" CIRCLE_JOB = "${CIRCLE_JOB}" diff --git a/tests/main.tf.json b/tests/deploy/legacy-test/main.tf.json similarity index 89% rename from tests/main.tf.json rename to tests/deploy/legacy-test/main.tf.json index e39c7062..9d136d9f 100644 --- a/tests/main.tf.json +++ b/tests/deploy/legacy-test/main.tf.json @@ -3,7 +3,6 @@ { "aws_ami": { "eks_node": { - "provider": "aws.local", "most_recent": true, "owners": ["amazon"], "filter": [ @@ -19,8 +18,7 @@ "module": [ { "domino_eks": { - "source": "./..", - + "source": "github.com/dominodatalab/terraform-aws-eks.git?ref=v2.1.8", "deploy_id": "${var.deploy_id}", "region": "${var.region}", "route53_hosted_zone_name": "deploys-delta.domino.tech", @@ -35,7 +33,7 @@ "kms": { "enabled": true }, - "ssh_pvt_key_path": "domino.pem", + "ssh_pvt_key_path": "${var.ssh_pvt_key_path}", "tags": "${var.tags}", "default_node_groups": { "compute": { @@ -79,8 +77,13 @@ { "deploy_id": { "type": "string", - "description": "Domino Deployment ID.", - "default": "dominotfeks2" + "description": "Domino Deployment ID." + } + }, + { + "ssh_pvt_key_path": { + "type": "string", + "description": "ssh key path." } }, { @@ -103,7 +106,7 @@ "k8s_version": { "type": "string", "description": "EKS cluster k8s version.", - "default": "1.25" + "default": "1.27" } } ], @@ -114,7 +117,7 @@ { "aws": { "source": "hashicorp/aws", - "version": ">= 4.0" + "version": "~> 5.0" }, "random": { "source": "hashicorp/random", diff --git a/tests/deploy/meta.sh b/tests/deploy/meta.sh new file mode 100644 index 00000000..978fdd99 --- /dev/null +++ b/tests/deploy/meta.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" + +CI_DEPLOY="true" +DEPLOY_DIR="${SH_DIR}/deploy-test" +PVT_KEY="${DEPLOY_DIR}/domino.pem" + +LEGACY_DIR="${SH_DIR}/legacy-test" +LEGACY_TF="${LEGACY_DIR}/main.tf.json" +LEGACY_STATE="${LEGACY_DIR}/terraform.tfstate" +LEGACY_PVT_KEY="${LEGACY_DIR}/domino.pem" + +INFRA_VARS_TPL="${SH_DIR}/infra-ci.tfvars.tftpl" + +export SH_DIR CI_DEPLOY DEPLOY_DIR LEGACY_DIR LEGACY_TF PVT_KEY LEGACY_STATE LEGACY_PVT_KEY INFRA_VARS_TPL diff --git a/examples/create-kms-key/README.md b/tests/plan/create-kms-key/README.md similarity index 100% rename from examples/create-kms-key/README.md rename to tests/plan/create-kms-key/README.md diff --git a/examples/create-kms-key/main.tf b/tests/plan/create-kms-key/main.tf similarity index 100% rename from examples/create-kms-key/main.tf rename to tests/plan/create-kms-key/main.tf diff --git a/examples/create-kms-key/outputs.tf b/tests/plan/create-kms-key/outputs.tf similarity index 100% rename from examples/create-kms-key/outputs.tf rename to tests/plan/create-kms-key/outputs.tf diff --git a/examples/create-kms-key/variables.tf b/tests/plan/create-kms-key/variables.tf similarity index 100% rename from examples/create-kms-key/variables.tf rename to tests/plan/create-kms-key/variables.tf diff --git a/iam-bootstrap/versions.tf b/tests/plan/create-kms-key/versions.tf similarity index 100% rename from iam-bootstrap/versions.tf rename to tests/plan/create-kms-key/versions.tf diff --git a/tests/plan/terraform/README.md b/tests/plan/terraform/README.md new file mode 100644 index 00000000..07041c4d --- /dev/null +++ b/tests/plan/terraform/README.md @@ -0,0 +1,50 @@ +# terraform + +:warning: **DO NOT USE TO PROVISION INFRASTRUCTURE.This implementation is meant for testing purposes ONLY.** + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 1.4.0 | +| [aws](#requirement\_aws) | ~> 5.0 | + +## Providers + +No providers. + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [eks](#module\_eks) | ./../../../modules/eks | n/a | +| [infra](#module\_infra) | ./../../../modules/infra/ | n/a | +| [nodes](#module\_nodes) | ./../../../modules/nodes | n/a | + +## Resources + +No resources. + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [additional\_node\_groups](#input\_additional\_node\_groups) | Additional EKS managed node groups definition. | 
map(object({
    ami                   = optional(string, null)
    bootstrap_extra_args  = optional(string, "")
    instance_types        = list(string)
    spot                  = optional(bool, false)
    min_per_az            = number
    max_per_az            = number
    desired_per_az        = number
    availability_zone_ids = list(string)
    labels                = map(string)
    taints = optional(list(object({
      key    = string
      value  = optional(string)
      effect = string
    })), [])
    tags = optional(map(string), {})
    gpu  = optional(bool, null)
    volume = object({
      size = string
      type = string
    })
  }))
| `{}` | no | +| [bastion](#input\_bastion) | enabled = Create bastion host.
ami = Ami id. Defaults to latest 'amazon\_linux\_2' ami.
instance\_type = Instance type.
authorized\_ssh\_ip\_ranges = List of CIDR ranges permitted for the bastion ssh access.
username = Bastion user.
install\_binaries = Toggle to install required Domino binaries in the bastion. | 
object({
    enabled                  = optional(bool, true)
    ami_id                   = optional(string, null) # default will use the latest 'amazon_linux_2' ami
    instance_type            = optional(string, "t3.micro")
    authorized_ssh_ip_ranges = optional(list(string), ["0.0.0.0/0"])
    username                 = optional(string, "ec2-user")
    install_binaries         = optional(bool, false)
  })
| `{}` | no | +| [default\_node\_groups](#input\_default\_node\_groups) | EKS managed node groups definition. | 
object(
    {
      compute = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      }),
      platform = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["m5.2xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 1)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 1)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "platform"
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
          })), [])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 100)
            type = optional(string, "gp3")
            }), {
            size = 100
            type = "gp3"
            }
          )
      }),
      gpu = object(
        {
          ami                   = optional(string, null)
          bootstrap_extra_args  = optional(string, "")
          instance_types        = optional(list(string), ["g4dn.xlarge"])
          spot                  = optional(bool, false)
          min_per_az            = optional(number, 0)
          max_per_az            = optional(number, 10)
          desired_per_az        = optional(number, 0)
          availability_zone_ids = list(string)
          labels = optional(map(string), {
            "dominodatalab.com/node-pool" = "default-gpu"
            "nvidia.com/gpu"              = true
          })
          taints = optional(list(object({
            key    = string
            value  = optional(string)
            effect = string
            })), [{
            key    = "nvidia.com/gpu"
            value  = "true"
            effect = "NO_SCHEDULE"
            }
          ])
          tags = optional(map(string), {})
          gpu  = optional(bool, null)
          volume = optional(object({
            size = optional(number, 1000)
            type = optional(string, "gp3")
            }), {
            size = 1000
            type = "gp3"
            }
          )
      })
  })
| n/a | yes | +| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks"` | no | +| [eks](#input\_eks) | k8s\_version = EKS cluster k8s version.
kubeconfig = {
extra\_args = Optional extra args when generating kubeconfig.
path = Fully qualified path name to write the kubeconfig file.
}
public\_access = {
enabled = Enable EKS API public endpoint.
cidrs = List of CIDR ranges permitted for accessing the EKS public endpoint.
}
Custom role maps for aws auth configmap
custom\_role\_maps = {
rolearn = string
username = string
groups = list(string)
}
master\_role\_names = IAM role names to be added as masters in eks.
cluster\_addons = EKS cluster addons. vpc-cni is installed separately.
vpc\_cni = Configuration for AWS VPC CNI
ssm\_log\_group\_name = CloudWatch log group to send the SSM session logs to.
identity\_providers = Configuration for IDP(Identity Provider).
} | 
object({
    k8s_version = optional(string, "1.27")
    kubeconfig = optional(object({
      extra_args = optional(string, "")
      path       = optional(string, null)
    }), {})
    public_access = optional(object({
      enabled = optional(bool, false)
      cidrs   = optional(list(string), [])
    }), {})
    custom_role_maps = optional(list(object({
      rolearn  = string
      username = string
      groups   = list(string)
    })), [])
    master_role_names  = optional(list(string), [])
    cluster_addons     = optional(list(string), ["kube-proxy", "coredns"])
    ssm_log_group_name = optional(string, "session-manager")
    vpc_cni = optional(object({
      prefix_delegation = optional(bool)
    }))
    identity_providers = optional(list(object({
      client_id                     = string
      groups_claim                  = optional(string, null)
      groups_prefix                 = optional(string, null)
      identity_provider_config_name = string
      issuer_url                    = optional(string, null)
      required_claims               = optional(string, null)
      username_claim                = optional(string, null)
      username_prefix               = optional(string, null)
    })), [])
  })
| `{}` | no | +| [enable\_private\_link](#input\_enable\_private\_link) | Enable Private Link connections | `bool` | `false` | no | +| [kms](#input\_kms) | enabled = Toggle,if set use either the specified KMS key\_id or a Domino-generated one.
key\_id = optional(string, null)
additional\_policies = "Allows setting additional KMS key policies when using a Domino-generated key" | 
object({
    enabled             = optional(bool, true)
    key_id              = optional(string, null)
    additional_policies = optional(list(string), [])
  })
| `{}` | no | +| [network](#input\_network) | vpc = {
id = Existing vpc id, it will bypass creation by this module.
subnets = {
private = Existing private subnets.
public = Existing public subnets.
pod = Existing pod subnets.
}), {})
}), {})
network\_bits = {
public = Number of network bits to allocate to the public subnet. i.e /27 -> 32 IPs.
private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs.
}
cidrs = {
vpc = The IPv4 CIDR block for the VPC.
pod = The IPv4 CIDR block for the Pod subnets.
}
use\_pod\_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. | 
object({
    vpc = optional(object({
      id = optional(string, null)
      subnets = optional(object({
        private = optional(list(string), [])
        public  = optional(list(string), [])
        pod     = optional(list(string), [])
      }), {})
    }), {})
    network_bits = optional(object({
      public  = optional(number, 27)
      private = optional(number, 19)
      pod     = optional(number, 19)
      }
    ), {})
    cidrs = optional(object({
      vpc = optional(string, "10.0.0.0/16")
      pod = optional(string, "100.64.0.0/16")
    }), {})
    use_pod_cidr = optional(bool, true)
  })
| `{}` | no | +| [region](#input\_region) | AWS region for the deployment | `string` | n/a | yes | +| [route53\_hosted\_zone\_name](#input\_route53\_hosted\_zone\_name) | Optional hosted zone for External DNS zone. | `string` | `null` | no | +| [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | +| [storage](#input\_storage) | storage = {
efs = {
access\_point\_path = Filesystem path for efs.
backup\_vault = {
create = Create backup vault for EFS toggle.
force\_destroy = Toggle to allow automatic destruction of all backups when destroying.
backup = {
schedule = Cron-style schedule for EFS backup vault (default: once a day at 12pm).
cold\_storage\_after = Move backup data to cold storage after this many days.
delete\_after = Delete backup data after this many days.
}
}
}
s3 = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the s3 buckets. if 'false' terraform will NOT be able to delete non-empty buckets.
}
ecr = {
force\_destroy\_on\_deletion = Toogle to allow recursive deletion of all objects in the ECR repositories. if 'false' terraform will NOT be able to delete non-empty repositories.
}
}
} | 
object({
    efs = optional(object({
      access_point_path = optional(string, "/domino")
      backup_vault = optional(object({
        create        = optional(bool, true)
        force_destroy = optional(bool, true)
        backup = optional(object({
          schedule           = optional(string, "0 12 * * ? *")
          cold_storage_after = optional(number, 35)
          delete_after       = optional(number, 125)
        }), {})
      }), {})
    }), {})
    s3 = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
    ecr = optional(object({
      force_destroy_on_deletion = optional(bool, true)
    }), {})
  })
| `{}` | no | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | `{}` | no | + +## Outputs + +No outputs. + diff --git a/tests/plan/terraform/main.tf b/tests/plan/terraform/main.tf new file mode 100644 index 00000000..74805483 --- /dev/null +++ b/tests/plan/terraform/main.tf @@ -0,0 +1,52 @@ +module "infra" { + source = "./../../../modules/infra/" + + deploy_id = var.deploy_id + additional_node_groups = var.additional_node_groups + bastion = var.bastion + default_node_groups = var.default_node_groups + network = var.network + storage = var.storage + eks = var.eks + kms = var.kms + region = var.region + route53_hosted_zone_name = var.route53_hosted_zone_name + ssh_pvt_key_path = var.ssh_pvt_key_path + tags = var.tags +} + + +module "eks" { + source = "./../../../modules/eks" + deploy_id = module.infra.deploy_id + region = module.infra.region + + ssh_key = module.infra.ssh_key + node_iam_policies = module.infra.node_iam_policies + efs_security_group = module.infra.efs_security_group + eks = module.infra.eks + network_info = module.infra.network + kms_info = module.infra.kms + bastion_info = module.infra.bastion + create_eks_role_arn = module.infra.create_eks_role_arn + tags = module.infra.tags + privatelink = { + enabled = var.enable_private_link + monitoring_bucket = module.infra.monitoring_bucket + route53_hosted_zone_name = var.route53_hosted_zone_name + } +} + + +module "nodes" { + source = "./../../../modules/nodes" + region = module.infra.region + + ssh_key = module.infra.ssh_key + default_node_groups = module.infra.default_node_groups + additional_node_groups = module.infra.additional_node_groups + eks_info = module.eks.info + network_info = module.infra.network + kms_info = module.infra.kms + tags = module.infra.tags +} diff --git a/tests/plan/terraform/outputs.tf b/tests/plan/terraform/outputs.tf new file mode 100644 index 00000000..e69de29b diff --git a/tests/plan/terraform/variables.tf b/tests/plan/terraform/variables.tf new file mode 100644 index 00000000..fbfbb225 --- /dev/null +++ b/tests/plan/terraform/variables.tf @@ -0,0 +1,391 @@ +variable "region" { + type = string + description = "AWS region for the deployment" + nullable = false + validation { + condition = can(regex("(us(-gov)?|ap|ca|cn|eu|sa|me|af|il)-(central|(north|south)?(east|west)?)-[0-9]", var.region)) + error_message = "The provided region must follow the format of AWS region names, e.g., us-west-2, us-gov-west-1." + } +} + +variable "deploy_id" { + type = string + description = "Domino Deployment ID." + default = "domino-eks" + nullable = false + + validation { + condition = length(var.deploy_id) >= 3 && length(var.deploy_id) <= 32 && can(regex("^[a-z]([-a-z0-9]*[a-z0-9])$", var.deploy_id)) + error_message = < 32 IPs. + private = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. + pod = Number of network bits to allocate to the private subnet. i.e /19 -> 8,192 IPs. + } + cidrs = { + vpc = The IPv4 CIDR block for the VPC. + pod = The IPv4 CIDR block for the Pod subnets. + } + use_pod_cidr = Use additional pod CIDR range (ie 100.64.0.0/16) for pod networking. + EOF + + type = object({ + vpc = optional(object({ + id = optional(string, null) + subnets = optional(object({ + private = optional(list(string), []) + public = optional(list(string), []) + pod = optional(list(string), []) + }), {}) + }), {}) + network_bits = optional(object({ + public = optional(number, 27) + private = optional(number, 19) + pod = optional(number, 19) + } + ), {}) + cidrs = optional(object({ + vpc = optional(string, "10.0.0.0/16") + pod = optional(string, "100.64.0.0/16") + }), {}) + use_pod_cidr = optional(bool, true) + }) + + default = {} +} + +variable "bastion" { + description = < 0 : true + error_message = "KMS key ID must be null or set to a non-empty string, when var.kms.enabled is." + } + + validation { + condition = var.kms.key_id != null ? var.kms.enabled : true + error_message = "var.kms.enabled must be true if var.kms.key_id is provided." + } + + default = {} +} + +variable "enable_private_link" { + type = bool + description = "Enable Private Link connections" + default = false +} diff --git a/tests/plan/terraform/versions.tf b/tests/plan/terraform/versions.tf new file mode 100644 index 00000000..94b8dc30 --- /dev/null +++ b/tests/plan/terraform/versions.tf @@ -0,0 +1,13 @@ +provider "aws" { + region = var.region +} + +terraform { + required_version = ">= 1.4.0" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } +} diff --git a/tests/plan/tf-plan-test.sh b/tests/plan/tf-plan-test.sh new file mode 100755 index 00000000..9876ddf5 --- /dev/null +++ b/tests/plan/tf-plan-test.sh @@ -0,0 +1,122 @@ +#! /usr/bin/env bash + +SH_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")" +exclude=("bring-your-vpc.tfvars" "kms-byok.tfvars" "private-link.tfvars") + +failed_vars=() +success_vars=() + +verify_terraform() { + if ! [ -x "$(command -v terraform)" ]; then + printf "\n\033[0;31mError: Terraform is not installed!!!\033[0m\n" + exit 1 + else + terraform_version=$(terraform --version | awk '/Terraform/ {print $2}' | awk -F 'version' '{print $1}' | tr -d '[:space:]') + printf "\033[0;32mTerraform version: ${terraform_version} is installed.\033[0m\n" + fi +} + +verify_aws_creds() { + if ! aws sts get-caller-identity >/dev/null 2>&1; then + printf "\033[0;31mERROR: AWS credentials are wrong or not set.\033[0m\n" + exit 1 + fi +} + +tf_plan() { + local tfvars="$1" + local test_pem="terraform/plan-test.pem" + + if [ ! -f "$test_pem" ]; then + ssh-keygen -q -P '' -t rsa -b 4096 -m PEM -f "$test_pem" && chmod 400 "$test_pem" + fi + + terraform -chdir=terraform init -upgrade + terraform -chdir=terraform plan -var-file="$tfvars" -var "ssh_pvt_key_path=$(basename $test_pem)" + + if [ "$?" != "0" ]; then + printf "\033[0;31mERROR: terraform plan failed for $tfvars\033[0m.\n" + failed_vars+=("$(basename $tfvars)") + else + printf "\033[0;32mSUCCESS: terraform plan succeeded for $tfvars\033[0m.\n" + success_vars+=("$(basename $tfvars)") + fi +} + +run_terraform_plans() { + for tfvars in ../../examples/tfvars/*.tfvars; do + base_tfvars=$(basename "$tfvars") + skip=false + + for excl in "${exclude[@]}"; do + if [[ "$base_tfvars" == *"$excl" ]]; then + skip=true + break + fi + done + + $skip && continue + + printf "\n\033[0;33mRunning terraform plan for ${base_tfvars}\033[0m:\n" + tf_plan "$(realpath $tfvars)" + done + +} + +create_kms_key() { + local dir="create-kms-key" + + printf "\n\033[0;33mCreating KMS key\033[0m\n" + terraform -chdir="$dir" init + if ! terraform -chdir="$dir" apply --auto-approve; then + printf "\n\033[0;31mFailed to create kms key!!!\033[0m\n" + failed_vars+=("kms") + else + printf "\n\033[0;32mKMS key created successfully\033[0m\n" + fi + KMS_KEY_ID="$(terraform -chdir="$dir" output -raw kms_key_id)" + export KMS_KEY_ID +} + +destroy_kms_key() { + local dir="create-kms-key" + + printf "\n\033[0;33mDestroying KMS key\033[0m\n" + terraform -chdir="$dir" destroy --auto-approve || terraform -chdir="$dir" destroy --auto-approve --refresh=false +} + +test_byok_kms() { + create_kms_key + if test -z $KMS_KEY_ID; then + printf "\033[0;31mERROR Obtaining KMS_KEY_ID \033[0m.\n" + exit 1 + fi + + local KMS_VARS_FILE="../../examples/tfvars/kms-byok.tfvars" + local vars_file="$(basename $KMS_VARS_FILE)" + + cat $KMS_VARS_FILE | sed "s/key_id = \".*\"/key_id = \"$KMS_KEY_ID\"/" | tee "$vars_file" + + tf_plan "$(realpath $vars_file)" && rm "$vars_file" +} + +finish() { + destroy_kms_key + + if [ "${#success_vars[@]}" != "0" ]; then + printf "\n\033[0;32mThe following examples ran a terraform plan successfully:\033[0m\n" + printf '\033[0;32m%s\n\033[0m' "${success_vars[@]}" + fi + + if [ "${#failed_vars[@]}" != "0" ]; then + printf "\n\033[0;31mThe following examples failed to run a terraform plan:\033[0m\n" + printf '\033[0;31m%s\n\033[0m' "${failed_vars[@]} " + exit 1 + fi +} + +trap finish EXIT ERR INT TERM +verify_terraform +verify_aws_creds +run_terraform_plans +test_byok_kms diff --git a/versions.tf b/versions.tf deleted file mode 100644 index a00ef96b..00000000 --- a/versions.tf +++ /dev/null @@ -1,43 +0,0 @@ -terraform { - required_version = ">= 1.4.0" - required_providers { - aws = { - source = "hashicorp/aws" - version = ">= 4.0" - } - local = { - source = "hashicorp/local" - version = ">= 2.2.0" - } - tls = { - source = "hashicorp/tls" - version = ">= 3.4.0" - } - time = { - source = "hashicorp/time" - version = ">= 0.9.1" - } - } -} - -provider "aws" { - region = var.region - default_tags { - tags = var.tags - } -} - - -provider "aws" { - alias = "eks" - region = var.region - default_tags { - tags = var.tags - } - - assume_role { - # https://github.com/hashicorp/terraform/issues/30690 - # https://github.com/hashicorp/terraform/issues/2430 - role_arn = "${aws_iam_role.create_eks_role.arn}${time_sleep.create_eks_role_30_seconds.id == "nil" ? "" : ""}" - } -}