diff --git a/.gitignore b/.gitignore index 8a170485..5911cb49 100644 --- a/.gitignore +++ b/.gitignore @@ -35,7 +35,6 @@ domino.pem.pub k8s-functions.sh k8s-pre-setup.sh kubeconfig -mallory.json domino.yml terraform_tflint.txt terraform_tfsec.txt diff --git a/main.tf b/main.tf index 3fdc59cd..4d25630d 100644 --- a/main.tf +++ b/main.tf @@ -155,10 +155,9 @@ data "aws_iam_role" "eks_master_roles" { module "k8s_setup" { count = var.create_bastion ? 1 : 0 source = "./submodules/k8s" - ssh_pvt_key_path = abspath(local.ssh_pvt_key_path) + ssh_pvt_key_path = local.ssh_pvt_key_path bastion_user = local.bastion_user bastion_public_ip = try(module.bastion[0].public_ip, "") - k8s_cluster_endpoint = module.eks.cluster_endpoint eks_node_role_arns = [for r in module.eks.eks_node_roles : r.arn] eks_master_role_arns = [for r in concat(values(data.aws_iam_role.eks_master_roles), module.eks.eks_master_roles) : r.arn] kubeconfig_path = local.kubeconfig_path diff --git a/outputs.tf b/outputs.tf index b3c1ac74..260ad909 100755 --- a/outputs.tf +++ b/outputs.tf @@ -8,11 +8,6 @@ output "bastion_ip" { value = var.create_bastion ? module.bastion[0].public_ip : "" } -output "k8s_tunnel_command" { - description = "Command to run the k8s tunnel mallory." - value = try(module.k8s_setup.k8s_tunnel_command, "") -} - output "hostname" { description = "Domino instance URL." value = "${var.deploy_id}.${var.route53_hosted_zone_name}" diff --git a/submodules/eks/cluster.tf b/submodules/eks/cluster.tf index fae7cc14..984c1475 100755 --- a/submodules/eks/cluster.tf +++ b/submodules/eks/cluster.tf @@ -117,10 +117,25 @@ resource "aws_eks_addon" "this" { resource "null_resource" "kubeconfig" { provisioner "local-exec" { - command = "aws eks update-kubeconfig --kubeconfig ${var.kubeconfig_path} --region ${var.region} --name ${aws_eks_cluster.this.name}" + when = create + command = "aws eks update-kubeconfig --kubeconfig ${self.triggers.kubeconfig_file} --region ${self.triggers.region} --name ${self.triggers.cluster_name}" + } + provisioner "local-exec" { + when = destroy + command = < 0 )); then + kubectl config --kubeconfig ${self.triggers.kubeconfig_file} delete-cluster ${self.triggers.cluster_name} + kubectl config --kubeconfig ${self.triggers.kubeconfig_file} delete-context ${self.triggers.cluster_name} + else + rm -f ${self.triggers.kubeconfig_file} + fi + EOT } triggers = { domino_eks_cluster_ca = aws_eks_cluster.this.certificate_authority[0].data + cluster_name = aws_eks_cluster.this.name + kubeconfig_file = var.kubeconfig_path + region = var.region } depends_on = [aws_eks_cluster.this] } diff --git a/submodules/k8s/README.md b/submodules/k8s/README.md index 4dbaeff2..0a2649c0 100644 --- a/submodules/k8s/README.md +++ b/submodules/k8s/README.md @@ -37,15 +37,11 @@ No modules. | [calico\_version](#input\_calico\_version) | Calico operator version. | `string` | `"v1.11.0"` | no | | [eks\_master\_role\_arns](#input\_eks\_master\_role\_arns) | IAM role arns to be added as masters in eks. | `list(string)` | `[]` | no | | [eks\_node\_role\_arns](#input\_eks\_node\_role\_arns) | Roles arns for EKS nodes to be added to aws-auth for api auth. | `list(string)` | n/a | yes | -| [k8s\_cluster\_endpoint](#input\_k8s\_cluster\_endpoint) | EKS cluster API endpoint. | `string` | n/a | yes | +| [k8s\_tunnel\_port](#input\_k8s\_tunnel\_port) | K8s ssh tunnel port | `string` | `"1080"` | no | | [kubeconfig\_path](#input\_kubeconfig\_path) | Kubeconfig filename. | `string` | `"kubeconfig"` | no | -| [mallory\_local\_normal\_port](#input\_mallory\_local\_normal\_port) | Mallory k8s tunnel normal port. | `string` | `"1315"` | no | -| [mallory\_local\_smart\_port](#input\_mallory\_local\_smart\_port) | Mallory k8s tunnel smart(filters based on the blocked list) port. | `string` | `"1316"` | no | | [ssh\_pvt\_key\_path](#input\_ssh\_pvt\_key\_path) | SSH private key filepath. | `string` | n/a | yes | ## Outputs -| Name | Description | -|------|-------------| -| [k8s\_tunnel\_command](#output\_k8s\_tunnel\_command) | Command to run the k8s tunnel mallory. | +No outputs. diff --git a/submodules/k8s/main.tf b/submodules/k8s/main.tf index 5d7d205d..5128b1ce 100644 --- a/submodules/k8s/main.tf +++ b/submodules/k8s/main.tf @@ -1,21 +1,15 @@ locals { - mallory_config_filename = "mallory.json" - mallory_container_name = "mallory_k8s_tunnel" - mallory_config_path_container = "/root/.config/${local.mallory_config_filename}" - pvt_key_path_container = "/root/${basename(var.ssh_pvt_key_path)}" - k8s_functions_sh_filename = "k8s-functions.sh" - k8s_functions_sh_template = "k8s-functions.sh.tftpl" - k8s_pre_setup_sh_filename = "k8s-pre-setup.sh" - k8s_pre_setup_sh_template = "k8s-pre-setup.sh.tftpl" - aws_auth_filename = "aws-auth.yaml" - aws_auth_template = "aws-auth.yaml.tftpl" + k8s_functions_sh_filename = "k8s-functions.sh" + k8s_functions_sh_template = "k8s-functions.sh.tftpl" + k8s_pre_setup_sh_filename = "k8s-pre-setup.sh" + k8s_pre_setup_sh_template = "k8s-pre-setup.sh.tftpl" + aws_auth_filename = "aws-auth.yaml" + aws_auth_template = "aws-auth.yaml.tftpl" calico = { operator_url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-operator.yaml" custom_resources_url = "https://raw.githubusercontent.com/aws/amazon-vpc-cni-k8s/${var.calico_version}/config/master/calico-crs.yaml" } - k8s_tunnel_command = "docker run --rm --name ${local.mallory_container_name} -d -v $PWD/${local.mallory_config_filename}:${local.mallory_config_path_container} -p ${var.mallory_local_normal_port}:${var.mallory_local_normal_port} -p ${var.mallory_local_smart_port}:${var.mallory_local_smart_port} -v ${var.ssh_pvt_key_path}:${local.pvt_key_path_container} zoobab/mallory" - resources_directory = path.cwd templates_dir = "${path.module}/templates" @@ -24,13 +18,13 @@ locals { filename = local.k8s_functions_sh_filename content = templatefile("${local.templates_dir}/${local.k8s_functions_sh_template}", { kubeconfig_path = var.kubeconfig_path - k8s_tunnel_command = local.k8s_tunnel_command - mallory_port = var.mallory_local_smart_port - mallory_container_name = local.mallory_container_name - mallory_config_file = local.mallory_config_filename + k8s_tunnel_port = var.k8s_tunnel_port aws_auth_yaml = basename(local.aws_auth_filename) calico_operator_url = local.calico.operator_url calico_custom_resources_url = local.calico.custom_resources_url + bastion_user = var.bastion_user + bastion_public_ip = var.bastion_public_ip + ssh_pvt_key_path = var.ssh_pvt_key_path }) } @@ -41,17 +35,6 @@ locals { }) } - mallory_k8s_tunnel = { - filename = local.mallory_config_filename - content = jsonencode( - { - "id_rsa" = local.pvt_key_path_container - "local_smart" = ":${var.mallory_local_smart_port}" - "local_normal" = ":${var.mallory_local_normal_port}" - "remote" = "ssh://${var.bastion_user}@${var.bastion_public_ip}:22" - "blocked" = [var.k8s_cluster_endpoint] - }) } - aws_auth = { filename = local.aws_auth_filename content = templatefile("${local.templates_dir}/${local.aws_auth_template}", @@ -75,7 +58,7 @@ resource "local_file" "templates" { resource "null_resource" "run_k8s_pre_setup" { provisioner "local-exec" { command = basename(local_file.templates["k8s_presetup"].filename) - interpreter = ["bash", "-ex"] + interpreter = ["bash"] working_dir = local.resources_directory } diff --git a/submodules/k8s/outputs.tf b/submodules/k8s/outputs.tf index 4f1aed84..e69de29b 100644 --- a/submodules/k8s/outputs.tf +++ b/submodules/k8s/outputs.tf @@ -1,4 +0,0 @@ -output "k8s_tunnel_command" { - description = "Command to run the k8s tunnel mallory." - value = local.k8s_tunnel_command -} diff --git a/submodules/k8s/templates/docker-repo-creds-secret.yaml.tftpl b/submodules/k8s/templates/docker-repo-creds-secret.yaml.tftpl deleted file mode 100644 index 4f5765a5..00000000 --- a/submodules/k8s/templates/docker-repo-creds-secret.yaml.tftpl +++ /dev/null @@ -1,12 +0,0 @@ -%{ for ns in namespaces ~} ---- -kind: Secret -apiVersion: v1 -metadata: - name: ${name} - namespace: ${ns} -data: - ".dockerconfigjson": ${quay_auth_encoded} -type: kubernetes.io/dockerconfigjson - -%{ endfor ~} diff --git a/submodules/k8s/templates/ebs-storageclass.yaml.tftpl b/submodules/k8s/templates/ebs-storageclass.yaml.tftpl deleted file mode 100644 index 4c718c58..00000000 --- a/submodules/k8s/templates/ebs-storageclass.yaml.tftpl +++ /dev/null @@ -1,12 +0,0 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ${name} -provisioner: ${driver} -parameters: - encrypted: 'true' - type: gp3 -reclaimPolicy: Delete -allowVolumeExpansion: true -volumeBindingMode: WaitForFirstConsumer diff --git a/submodules/k8s/templates/efs-persistent-volume-claim.yaml.tftpl b/submodules/k8s/templates/efs-persistent-volume-claim.yaml.tftpl deleted file mode 100644 index 26348e99..00000000 --- a/submodules/k8s/templates/efs-persistent-volume-claim.yaml.tftpl +++ /dev/null @@ -1,15 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: ${pvc_name} - namespace: ${namespace} -spec: - accessModes: - - ReadWriteMany - resources: - requests: - storage: ${capacity} - volumeName: ${pv_name} - storageClassName: ${storage_class} - volumeMode: Filesystem diff --git a/submodules/k8s/templates/efs-persistent-volume.yaml.tftpl b/submodules/k8s/templates/efs-persistent-volume.yaml.tftpl deleted file mode 100644 index f216e33a..00000000 --- a/submodules/k8s/templates/efs-persistent-volume.yaml.tftpl +++ /dev/null @@ -1,18 +0,0 @@ ---- -kind: PersistentVolume -apiVersion: v1 -metadata: - name: ${pv_name} -spec: - capacity: - storage: ${capacity} - csi: - driver: ${driver} - volumeHandle: ${efs_volume_handle} - volumeAttributes: - encryptInTransit: 'true' - accessModes: - - ReadWriteMany - persistentVolumeReclaimPolicy: Retain - storageClassName: ${storage_class} - volumeMode: Filesystem diff --git a/submodules/k8s/templates/efs-storageclass.yaml.tftpl b/submodules/k8s/templates/efs-storageclass.yaml.tftpl deleted file mode 100644 index 728402aa..00000000 --- a/submodules/k8s/templates/efs-storageclass.yaml.tftpl +++ /dev/null @@ -1,9 +0,0 @@ ---- -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: ${name} -provisioner: ${driver} -reclaimPolicy: Delete -allowVolumeExpansion: true -volumeBindingMode: Immediate diff --git a/submodules/k8s/templates/k8s-functions.sh.tftpl b/submodules/k8s/templates/k8s-functions.sh.tftpl index 5e7a2beb..e5c5a117 100644 --- a/submodules/k8s/templates/k8s-functions.sh.tftpl +++ b/submodules/k8s/templates/k8s-functions.sh.tftpl @@ -5,25 +5,13 @@ GREEN="\e[32m" EC="\e[0m" open_ssh_tunnel_to_k8s_api() { - printf "$GREEN Setting up mallory k8s tunnel... $EC \n" - if [[ -z "$https_proxy" ]]; then - MALLORY_PORT=${mallory_port} - https_proxy=http://localhost:$MALLORY_PORT - export https_proxy - fi - - if test -f "${mallory_config_file}"; then - { docker kill ${mallory_container_name} && docker rm ${mallory_container_name}; } || true - eval ${k8s_tunnel_command} && sleep 3 - else - echo "Mallory config file ${mallory_config_file} does not exist." && exit 1 - fi - - echo + TUNNEL_SOCKET_FILE=$${TUNNEL_SOCKET_FILE:-/tmp/k8s-tunnel-socket} + printf "$GREEN Openning k8s tunnel ... $EC \n" + ssh -q -N -f -M -o "StrictHostKeyChecking no" -o "ExitOnForwardFailure=yes" -i "${ssh_pvt_key_path}" -D ${k8s_tunnel_port} -S "$TUNNEL_SOCKET_FILE" ${bastion_user}@${bastion_public_ip} } check_kubeconfig() { - KUBECONFIG="${kubeconfig_path}" + local KUBECONFIG="${kubeconfig_path}" printf "$GREEN Checking if $KUBECONFIG exists... $EC \n" if test -f "$KUBECONFIG"; then echo "$KUBECONFIG exists." && export KUBECONFIG @@ -34,10 +22,10 @@ check_kubeconfig() { } set_k8s_auth() { - AWS_AUTH_YAML="${aws_auth_yaml}" + local AWS_AUTH_YAML="${aws_auth_yaml}" if test -f "$AWS_AUTH_YAML"; then printf "$GREEN Updating $AWS_AUTH_YAML... $EC \n" - kubectl apply -f "$AWS_AUTH_YAML" + kubectl_apply "$AWS_AUTH_YAML" else printf "$RED $AWS_AUTH_YAML does not exist. $EC \n" && exit 1 fi @@ -45,31 +33,37 @@ set_k8s_auth() { } install_calico() { - CALICO_OPERATOR_YAML_URL=${calico_operator_url} + local CALICO_OPERATOR_YAML_URL=${calico_operator_url} printf "$GREEN Installing Calico Operator $EC \n" - kubectl apply -f $CALICO_OPERATOR_YAML_URL || printf "$RED There was an error installing the calico operator" + kubectl_apply $CALICO_OPERATOR_YAML_URL || printf "$RED There was an error installing the calico operator" echo - CALICO_CRD_YAML_URL=${calico_custom_resources_url} + local CALICO_CRD_YAML_URL=${calico_custom_resources_url} printf "$GREEN Installing Calico Custom resources $EC \n" || printf "$RED There was an error installing the calico CRD" - kubectl apply -f $CALICO_CRD_YAML_URL + kubectl_apply $CALICO_CRD_YAML_URL echo } +validate_url() { + local url="$1" + local log_file="validate-url.log" + if curl --head --fail --max-time 10 --output "$log_file" --stderr "$log_file" "$url"; then + rm "$log_file" && return 0 + else + cat "$log_file" && return 1 + fi +} + kubectl_apply() { - k8s_manifest="$1" - if test -f "$k8s_manifest"; then + local k8s_manifest="$1" + if test -f "$k8s_manifest" || validate_url "$k8s_manifest"; then echo "Applying $k8s_manifest..." - kubectl apply -f $k8s_manifest + HTTPS_PROXY=socks5://127.0.0.1:${k8s_tunnel_port} kubectl --kubeconfig "${kubeconfig_path}" apply -f "$k8s_manifest" else printf "$RED $k8s_manifest does not exist. $EC \n" && exit 1 fi - - kubectl apply -f "$k8s_manifest" } close_ssh_tunnel_to_k8s_api() { - printf "$GREEN Shutting down mallory k8s tunnel ${mallory_container_name} ... $EC" - docker kill "${mallory_container_name}" - docker rm "${mallory_container_name}" || true - echo + printf "$GREEN Shutting down k8s tunnel ... $EC" + ssh -S $TUNNEL_SOCKET_FILE -O exit ${bastion_user}@${bastion_public_ip} } diff --git a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl b/submodules/k8s/templates/k8s-pre-setup.sh.tftpl index 00dc96ec..b5a5aa8f 100644 --- a/submodules/k8s/templates/k8s-pre-setup.sh.tftpl +++ b/submodules/k8s/templates/k8s-pre-setup.sh.tftpl @@ -1,4 +1,4 @@ -#! /usr/bin/env bash +#! /usr/bin/env bash -ex RED="\e[31m" GREEN="\e[32m" diff --git a/submodules/k8s/variables.tf b/submodules/k8s/variables.tf index 3ff646c4..cafeb7ce 100755 --- a/submodules/k8s/variables.tf +++ b/submodules/k8s/variables.tf @@ -21,10 +21,6 @@ variable "bastion_public_ip" { type = string description = "Bastion host public ip." } -variable "k8s_cluster_endpoint" { - type = string - description = "EKS cluster API endpoint." -} variable "eks_node_role_arns" { type = list(string) @@ -37,16 +33,10 @@ variable "eks_master_role_arns" { default = [] } -variable "mallory_local_normal_port" { - type = string - description = "Mallory k8s tunnel normal port." - default = "1315" -} - -variable "mallory_local_smart_port" { +variable "k8s_tunnel_port" { type = string - description = "Mallory k8s tunnel smart(filters based on the blocked list) port." - default = "1316" + description = "K8s ssh tunnel port" + default = "1080" } variable "calico_version" { diff --git a/tests/README.md b/tests/README.md index 7c9b897b..1d4dfe73 100644 --- a/tests/README.md +++ b/tests/README.md @@ -25,10 +25,10 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"domino-eks-test"` | no | +| [deploy\_id](#input\_deploy\_id) | Domino Deployment ID. | `string` | `"mhtest3"` | no | | [k8s\_version](#input\_k8s\_version) | EKS cluster k8s version. | `string` | `"1.23"` | no | | [region](#input\_region) | AWS region for the deployment | `string` | `"us-west-2"` | no | -| [tags](#input\_tags) | Deployment tags. | `map(string)` | 
{
  "deploy_id": "domino-eks-test",
  "deploy_tag": "domino-eks-test",
  "deploy_type": "terraform-aws-eks",
  "domino-deploy-id": "domino-eks-test"
}
| no | +| [tags](#input\_tags) | Deployment tags. | `map(string)` | 
{
  "deploy_id": "mhtest3",
  "deploy_tag": "mhtest3",
  "deploy_type": "terraform-aws-eks",
  "domino-deploy-id": "mhtest3"
}
| no | ## Outputs diff --git a/tests/variables.tf b/tests/variables.tf index 2914ea93..b286f2c4 100644 --- a/tests/variables.tf +++ b/tests/variables.tf @@ -1,7 +1,7 @@ variable "deploy_id" { type = string description = "Domino Deployment ID." - default = "domino-eks-test" + default = "mhtest3" } @@ -16,10 +16,10 @@ variable "tags" { type = map(string) description = "Deployment tags." default = { - deploy_id = "domino-eks-test" - deploy_tag = "domino-eks-test" + deploy_id = "mhtest3" + deploy_tag = "mhtest3" deploy_type = "terraform-aws-eks" - domino-deploy-id = "domino-eks-test" + domino-deploy-id = "mhtest3" } }