Set proxy-url in kubeconfig (temporarily) (#25)

* Set proxy-url in kubeconfig (temporarily)

SOCKS proxy is incompatible with awscli. This normally doesn't
matter the way this module is used, but when a role is specified
then awscli needs to call out and fails at the socks proxy.

By moving the proxy config into the kubeconfig, kubectl uses the
proxy and awscli doesn't, fixing this issue.

* Properly parameterize k8s_tunnel_port

* Use kubectl to add proxy config

* Update readme

submodules/eks/k8s.tf

@@ -9,6 +9,7 @@ module "k8s_setup" {
   ssh_pvt_key_path     = var.ssh_pvt_key_path
   bastion_user         = var.bastion_user
   bastion_public_ip    = try(var.bastion_public_ip, "")
+  eks_cluster_arn      = aws_eks_cluster.this.arn
   eks_node_role_arns   = [aws_iam_role.eks_nodes.arn]
   eks_master_role_arns = [for r in concat(values(data.aws_iam_role.eks_master_roles), [aws_iam_role.eks_cluster]) : r.arn]
   kubeconfig_path      = var.kubeconfig_path

submodules/k8s/README.md

@@ -35,6 +35,7 @@ No modules.
 | <a name="input_bastion_public_ip"></a> [bastion\_public\_ip](#input\_bastion\_public\_ip) | Bastion host public ip. | `string` | n/a | yes |
 | <a name="input_bastion_user"></a> [bastion\_user](#input\_bastion\_user) | ec2 instance user. | `string` | `"ec2-user"` | no |
 | <a name="input_calico_version"></a> [calico\_version](#input\_calico\_version) | Calico operator version. | `string` | `"v1.11.0"` | no |
+| <a name="input_eks_cluster_arn"></a> [eks\_cluster\_arn](#input\_eks\_cluster\_arn) | ARN of the EKS cluster | `string` | n/a | yes |
 | <a name="input_eks_master_role_arns"></a> [eks\_master\_role\_arns](#input\_eks\_master\_role\_arns) | IAM role arns to be added as masters in eks. | `list(string)` | `[]` | no |
 | <a name="input_eks_node_role_arns"></a> [eks\_node\_role\_arns](#input\_eks\_node\_role\_arns) | Roles arns for EKS nodes to be added to aws-auth for api auth. | `list(string)` | n/a | yes |
 | <a name="input_k8s_tunnel_port"></a> [k8s\_tunnel\_port](#input\_k8s\_tunnel\_port) | K8s ssh tunnel port | `string` | `"1080"` | no |

submodules/k8s/main.tf

@@ -28,6 +28,7 @@ locals {
         bastion_user                = var.bastion_user
         bastion_public_ip           = var.bastion_public_ip
         ssh_pvt_key_path            = var.ssh_pvt_key_path
+        eks_cluster_arn             = var.eks_cluster_arn
       })
     }
 

submodules/k8s/templates/k8s-functions.sh.tftpl

@@ -4,17 +4,21 @@ RED="\e[31m"
 GREEN="\e[32m"
 EC="\e[0m"
 
+KUBECONFIG="${kubeconfig_path}"
+KUBECONFIG_PROXY="$KUBECONFIG-proxy"
+
 open_ssh_tunnel_to_k8s_api() {
   TUNNEL_SOCKET_FILE=$${TUNNEL_SOCKET_FILE:-/tmp/k8s-tunnel-socket}
   printf "$GREEN Openning k8s tunnel ... $EC \n"
   ssh -q -N -f -M -o "StrictHostKeyChecking no" -o "ExitOnForwardFailure=yes" -i "${ssh_pvt_key_path}" -D ${k8s_tunnel_port} -S "$TUNNEL_SOCKET_FILE" ${bastion_user}@${bastion_public_ip}
 }
 
 check_kubeconfig() {
-  local KUBECONFIG="${kubeconfig_path}"
   printf "$GREEN Checking if $KUBECONFIG exists... $EC \n"
   if test -f "$KUBECONFIG"; then
-    echo "$KUBECONFIG exists." && export KUBECONFIG
+    echo "$KUBECONFIG exists, creating $KUBECONFIG_PROXY for proxy use."
+    cp $KUBECONFIG $KUBECONFIG_PROXY
+    kubectl --kubeconfig $KUBECONFIG_PROXY config set "clusters.${eks_cluster_arn}.proxy-url" "socks5://127.0.0.1:${k8s_tunnel_port}"
   else
     echo "$KUBECONFIG does not exist." && exit 1
   fi
@@ -72,7 +76,7 @@ kubectl_apply() {
   local k8s_manifest="$1"
   if test -f "$k8s_manifest" || validate_url "$k8s_manifest"; then
     echo "Applying $k8s_manifest..."
-    HTTPS_PROXY=socks5://127.0.0.1:${k8s_tunnel_port} kubectl --kubeconfig "${kubeconfig_path}" apply -f "$k8s_manifest"
+    kubectl --kubeconfig "$KUBECONFIG_PROXY" apply -f "$k8s_manifest"
     if [ $? -ne 0 ]; then
       printf "$RED Error applying $k8s_manifest \n"
       exit 1
@@ -85,7 +89,7 @@ kubectl_apply() {
 
 kubectl_cmd() {
   echo "Running kubectl $@..."
-  HTTPS_PROXY=socks5://127.0.0.1:${k8s_tunnel_port} kubectl --kubeconfig "${kubeconfig_path}" $@
+  kubectl --kubeconfig "$KUBECONFIG_PROXY" $@
   if [ $? -ne 0 ]; then
     printf "$RED Error running kubectl $@ \n"
     exit 1

submodules/k8s/variables.tf

@@ -22,6 +22,11 @@ variable "bastion_public_ip" {
   description = "Bastion host public ip."
 }
 
+variable "eks_cluster_arn" {
+  type        = string
+  description = "ARN of the EKS cluster"
+}
+
 variable "eks_node_role_arns" {
   type        = list(string)
   description = "Roles arns for EKS nodes to be added to aws-auth for api auth."