Add High Availability Support

main.tf

@@ -33,6 +33,7 @@ module "masters" {
   ssh_public_key    = "${var.ssh_public_key}"
   region            = "${var.region}"
   linode_group      = "${var.cluster_name}"
+  lb_ip             = "${module.loadbalancer.loadbalancer_ip}"
 
   //todo variable instead of workspace?
   cluster_name = "${var.cluster_name == "" ? terraform.workspace : var.cluster_name}"
@@ -55,6 +56,27 @@ module "nodes" {
   kubeadm_join_command = "${module.masters.kubeadm_join_command}"
 }
 
+module "loadbalancer" {
+  source = "./modules/loadbalancer"
+  addl_master_count = "${var.addl-masters > 5 ? "5" : var.addl-masters}"
+  cluster_name = "${var.cluster_name == "" ? terraform.workspace : var.cluster_name}"
+  master_ip = "${module.masters.k8s_master_private_ip}"
+  master_label = "${module.masters.label[0]}"
+  label_prefix = "${var.cluster_name == "" ? terraform.workspace : var.cluster_name}"
+  node_class   = "master"
+  node_count   = "${var.addl-masters}"
+  node_type    = "${var.server_type_master}"
+  kubeadm_join_command = "${module.masters.kubeadm_join_command}"
+  kubeadm_cert_key  = "${module.masters.kubeadm_cert_key}"
+  k8s_version       = "${var.k8s_version}"
+  crictl_version    = "${var.crictl_version}"
+  k8s_feature_gates = "${var.k8s_feature_gates}"
+  cni_version       = "${var.cni_version}"
+  ssh_public_key    = "${var.ssh_public_key}"
+  region            = "${var.region}"
+  linode_group      = "${var.cluster_name}"
+}
+
 resource "null_resource" "local_kubectl" {
   // todo
   depends_on = ["module.masters"]

modules/addl-master/README.md

@@ -0,0 +1,3 @@
+## CoreOS Kubernetes Additional Master Instances
+
+This module provisions a Kubernetes Additional Master Linode Instance using CoreOS.

modules/addl-master/main.tf

@@ -0,0 +1,38 @@
+module "addl-master" {
+  source         = "../instances"
+  label_prefix   = "${var.label_prefix}"
+  node_type      = "${var.node_type}"
+  node_count     = "${var.node_count}"
+  master_type    = "addl-master"
+  node_class     = "master"
+  private_ip     = "true"
+  ssh_public_key = "${var.ssh_public_key}"
+  region         = "${var.region}"
+
+  linode_group = "${var.linode_group}"
+
+  k8s_version       = "${var.k8s_version}"
+  k8s_feature_gates = "${var.k8s_feature_gates}"
+  cni_version       = "${var.cni_version}"
+  crictl_version    = "${var.crictl_version}"
+}
+
+resource "null_resource" "kubeadm_join" {
+  count = "${var.node_count}"
+
+  provisioner "remote-exec" {
+    inline = [
+      "set -e",
+      "export PATH=$${PATH}:/opt/bin",
+      "sudo ${var.kubeadm_join_command} --experimental-control-plane --certificate-key ${var.kubeadm_cert_key}",
+      "sudo KUBECONFIG=/etc/kubernetes/kubelet.conf kubectl annotate node $${HOSTNAME} --overwrite container-linux-update.v1.coreos.com/reboot-paused=true",
+      "chmod +x /home/core/init/end.sh && sudo /home/core/init/end.sh",
+    ]
+
+    connection {
+      host    = "${element(module.addl-master.instances_public_ip, count.index)}"
+      user    = "core"
+      timeout = "300s"
+    }
+  }
+}

modules/addl-master/outputs.tf

@@ -0,0 +1,7 @@
+output "addl_masters_private_ip" {
+  value = "${module.addl-master.instances_private_ip}"
+}
+
+output "label" {
+  value   = "${module.addl-master.label}"
+}

modules/addl-master/variables.tf

@@ -0,0 +1,66 @@
+variable "node_count" {
+  default     = "0"
+  description = "Number of Kubernetes Nodes to provision"
+}
+
+variable "master_type" {
+  default     = "addl-master"
+  description = "Determines the type of the master (primary master or additional master)"
+}
+
+variable "node_class" {
+  default     = "master"
+  description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)"
+}
+
+variable "node_type" {
+  default     = "g6-standard-4"
+  description = "Linode Instance type for nodes"
+}
+
+variable "private_ip" {
+  default     = true
+  description = "Enables Linode Instance Private IP addresses"
+}
+
+variable "label_prefix" {
+  default     = ""
+  description = "Linode label prefix"
+}
+
+variable "linode_group" {
+  default     = ""
+  description = "Linode display group for provisioned instances"
+}
+
+variable "kubeadm_join_command" {
+  description = "Kubernetes 'kubeadm join' command to join this node to the cluster"
+}
+
+variable "kubeadm_cert_key" {
+  description = "Kubernetes kubeadm cert key to join the additional controlplane"
+}
+
+variable "region" {
+  description = "Linode region for instances"
+}
+
+variable "ssh_public_key" {
+  description = "SSH keys authorized for the Linux user account (core on Container Linux, root otherwise)"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes version to install"
+}
+
+variable "cni_version" {
+  description = "CNI version to install"
+}
+
+variable "crictl_version" {
+  description = "Container Runtime Interface version to install"
+}
+
+variable "k8s_feature_gates" {
+  description = "Kubernetes Feature gates to enable in the Kubelet"
+}

modules/instances/main.tf

@@ -5,7 +5,7 @@ data "linode_instance_type" "type" {
 resource "linode_instance" "instance" {
   count      = "${var.node_count}"
   region     = "${var.region}"
-  label      = "${var.label_prefix == "" ? "" : "${var.label_prefix}-"}${var.node_class}-${count.index + 1}"
+  label      = "${var.label_prefix == "" ? "" : "${var.label_prefix}-"}${var.node_class}-${var.node_class == "node" || var.master_type == "master" ? count.index + 1 : count.index + 2}"
   group      = "${var.linode_group}"
   type       = "${var.node_type}"
   private_ip = "${var.private_ip}"

modules/instances/outputs.tf

@@ -1,19 +1,23 @@
-// todo: ha, return nb address
-output "public_ip_address" {
-  depends_on  = ["linode_instance.instance.0"]
-  description = "Public IP Address of the first instance in the group"
-  value       = "${element(concat(linode_instance.instance.*.ip_address, list("")), 0)}"
+output "master_public_ip" {
+  depends_on = ["linode_instance.instance.0"]
+  value      = "${element(concat(linode_instance.instance.*.ip_address, list("")), 0)}"
 }
 
-// todo: this doesnt make sense in ha  -- return all?
-output "private_ip_address" {
-  description = "Private IP Address of the first instance in the group"
-  depends_on  = ["linode_instance.instance.0"]
-  value       = "${element(concat(linode_instance.instance.*.private_ip_address, list("")), 0)}"
+output "master_private_ip" {
+  depends_on = ["linode_instance.instance.0"]
+  value      = "${element(concat(linode_instance.instance.*.private_ip_address, list("")), 0)}"
 }
 
-output "nodes_public_ip" {
-  depends_on  = ["linode_instance.instance.*"]
-  description = "Public IP Address of the instance(s)"
-  value       = "${concat(linode_instance.instance.*.ip_address)}"
+output "instances_private_ip" {
+  depends_on = ["linode_instance.instance.*"]
+  value      = "${concat(linode_instance.instance.*.private_ip_address, list(""))}"
 }
+
+output "instances_public_ip" {
+  depends_on = ["linode_instance.instance.*"]
+  value      = "${concat(linode_instance.instance.*.ip_address)}"
+}
+
+output "label" {
+  value   = "${linode_instance.instance.*.label}"
+}

modules/instances/scripts/kubeadm-init.sh

@@ -3,9 +3,8 @@ set -e
 
 K8S_CLUSTERNAME="$1"
 K8S_VERSION="$2"
-NODE_PRIVATE_IP="$3"
-NODE_PUBLIC_IP="$4"
-K8S_FEATURE_GATES="$5"
+K8S_FEATURE_GATES="$3"
+LOAD_BALANCER_IP="$4"
 POD_NETWORK="10.244.0.0/16"
 
 # Generated with kubeadm config print-default
@@ -21,7 +20,6 @@ apiVersion: kubeadm.k8s.io/v1beta1
 # - authentication
 kind: InitConfiguration
 localAPIEndpoint:
-  advertiseAddress: ${NODE_PUBLIC_IP}
   bindPort: 6443
 nodeRegistration:
   criSocket: /var/run/dockershim.sock
@@ -34,15 +32,15 @@ nodeRegistration:
 ---
 apiServer:
   certSANs:
-  - ${NODE_PRIVATE_IP}
+  - ${LOAD_BALANCER_IP}
   extraArgs:
     cloud-provider: external
     feature-gates: ${K8S_FEATURE_GATES}
   timeoutForControlPlane: 4m0s
 apiVersion: kubeadm.k8s.io/v1beta1
 certificatesDir: /etc/kubernetes/pki
 clusterName: ${K8S_CLUSTERNAME}
-controlPlaneEndpoint: ""
+controlPlaneEndpoint: "${LOAD_BALANCER_IP}:6443"
 controllerManager:
   extraArgs:
     cloud-provider: external
@@ -100,4 +98,4 @@ resourceContainer: /kube-proxy
 udpIdleTimeout: 250ms
 EOF
 
-kubeadm init --config $HOME/kubeadm-config.yml
+kubeadm init --config $HOME/kubeadm-config.yml --experimental-upload-certs

modules/instances/variables.tf

@@ -3,6 +3,11 @@ variable "node_count" {
   description = "Number of Kubernetes Nodes to provision"
 }
 
+variable "master_type" {
+  default     = "master"
+  description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)"
+}
+
 variable "node_class" {
   default     = "node"
   description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)"

modules/loadbalancer/README.md

@@ -0,0 +1,3 @@
+## Linode Nodebalancer
+
+This module provisions a Nodebalancer which is used as Loadbalancer of the k8s apiserver

modules/loadbalancer/main.tf

@@ -0,0 +1,47 @@
+module "addl-masters" {
+  source       = "../addl-master"
+  label_prefix = "${var.cluster_name == "" ? terraform.workspace : var.cluster_name}"
+  node_class   = "master"
+  node_count   = "${var.addl_master_count}"
+  node_type    = "${var.node_type}"
+  kubeadm_join_command = "${var.kubeadm_join_command}"
+  kubeadm_cert_key  = "${var.kubeadm_cert_key}"
+  k8s_version       = "${var.k8s_version}"
+  crictl_version    = "${var.crictl_version}"
+  k8s_feature_gates = "${var.k8s_feature_gates}"
+  cni_version       = "${var.cni_version}"
+  ssh_public_key    = "${var.ssh_public_key}"
+  region            = "${var.region}"
+  linode_group      = "${var.cluster_name}"
+}
+
+resource "linode_nodebalancer" "apiserver-nodebalancer" {
+  label = "${var.cluster_name}"
+  region = "${var.region}"
+}
+
+resource "linode_nodebalancer_config" "apiserver-nodebalancer-config" {
+  nodebalancer_id = "${linode_nodebalancer.apiserver-nodebalancer.id}"
+  port = 6443
+  protocol = "tcp"
+}
+
+resource "linode_nodebalancer_node" "nodebalancer-node-master" {
+  nodebalancer_id = "${linode_nodebalancer.apiserver-nodebalancer.id}"
+  config_id = "${linode_nodebalancer_config.apiserver-nodebalancer-config.id}"
+  label = "${var.master_label}"
+  address = "${var.master_ip}:6443"
+  mode = "accept"
+}
+
+resource "linode_nodebalancer_node" "nodebalancer-node-addl-master" {
+  count = "${var.addl_master_count}"
+  nodebalancer_id = "${linode_nodebalancer.apiserver-nodebalancer.id}"
+  config_id = "${linode_nodebalancer_config.apiserver-nodebalancer-config.id}"
+  label = "${element(module.addl-masters.label, count.index)}"
+  address = "${element(module.addl-masters.addl_masters_private_ip, count.index)}:6443"
+  mode = "accept"
+
+  # The additional masters should be added to the loadbalancer after all of them are ready
+  depends_on = ["module.addl-masters"]
+}

modules/loadbalancer/outputs.tf

@@ -0,0 +1,3 @@
+output "loadbalancer_ip" {
+  value = "${linode_nodebalancer.apiserver-nodebalancer.ipv4}"
+}

modules/loadbalancer/variables.tf

@@ -0,0 +1,82 @@
+variable "cluster_name" {
+    description = "Name of the cluster"
+}
+
+variable "region" {
+  description = "Linode region"
+}
+
+variable "master_label" {
+  description = "Label of the master node"
+}
+
+variable "addl_master_count" {
+  description = "Number of Kubernetes additional master to provision"
+}
+
+variable "master_ip" {
+  description = "IP of the k8s master"
+}
+
+variable "node_count" {
+  default     = "0"
+  description = "Number of Kubernetes Nodes to provision"
+}
+
+variable "master_type" {
+  default     = "addl-master"
+  description = "Determines the type of the master (primary master or additional master)"
+}
+
+variable "node_class" {
+  default     = "master"
+  description = "Node class is determines Kubernetes provisioning behavior (also used as a Linode label prefix)"
+}
+
+variable "node_type" {
+  default     = "g6-standard-4"
+  description = "Linode Instance type for nodes"
+}
+
+variable "private_ip" {
+  default     = true
+  description = "Enables Linode Instance Private IP addresses"
+}
+
+variable "label_prefix" {
+  default     = ""
+  description = "Linode label prefix"
+}
+
+variable "linode_group" {
+  default     = ""
+  description = "Linode display group for provisioned instances"
+}
+
+variable "kubeadm_join_command" {
+  description = "Kubernetes 'kubeadm join' command to join this node to the cluster"
+}
+
+variable "kubeadm_cert_key" {
+  description = "Kubernetes kubeadm cert key to join the additional controlplane"
+}
+
+variable "ssh_public_key" {
+  description = "SSH keys authorized for the Linux user account (core on Container Linux, root otherwise)"
+}
+
+variable "k8s_version" {
+  description = "Kubernetes version to install"
+}
+
+variable "cni_version" {
+  description = "CNI version to install"
+}
+
+variable "crictl_version" {
+  description = "Container Runtime Interface version to install"
+}
+
+variable "k8s_feature_gates" {
+  description = "Kubernetes Feature gates to enable in the Kubelet"
+}