kubernetes.yaml

#cloud-config
package_upgrade: true

write_files:

- path: /etc/modules-load.d/containerd.conf
  content: |
    overlay
    br_netfilter

- path: /etc/sysctl.d/no-ipv6.conf
  content: |
    net.ipv6.conf.all.disable_ipv6     = 1
    net.ipv6.conf.default.disable_ipv6 = 1

- path: /etc/sysctl.d/k8s.conf
  content: |
    net.bridge.bridge-nf-call-iptables  = 1
    net.bridge.bridge-nf-call-ip6tables = 1
    net.ipv4.ip_forward                 = 1

# https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#dpkg-k8s-package-repo
- path: /usr/local/bin/kubernetes-install.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    DEBIAN_FRONTEND=noninteractive
    VERSION=v1.31
    curl -fsSL https://pkgs.k8s.io/core:/stable:/${VERSION}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
    echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/${VERSION}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
    apt-get update
    apt-get install -y kubelet kubeadm kubectl
    apt-mark hold kubelet kubeadm kubectl

- path: /usr/local/bin/kubernetes-setup-kubectl.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    kubectl completion bash > /etc/bash_completion.d/kubectl
    user=ubuntu
    home=/home/$user
    kube=$home/.kube
    mkdir -p $kube
    cp /etc/kubernetes/admin.conf $kube/config
    chown -R $user:$user $kube
    cat <<EOF >> $home/.bashrc
    alias k=kubectl
    complete -F __start_kubectl k
    EOF

# https://docs.cilium.io/en/stable/installation/k8s-install-kubeadm/
- path: /usr/local/bin/kubernetes-setup-cni.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    id=${1-1}
    version=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)
    curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${version}/cilium-linux-amd64.tar.gz
    tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin
    export KUBECONFIG=/etc/kubernetes/admin.conf
    if [ -e /tmp/ca.crt ] && [ -e /tmp/ca.key ]; then
      kubectl create secret generic cilium-ca -n kube-system --from-file=/tmp/ca.crt --from-file=/tmp/ca.key
      kubectl label secret -n kube-system cilium-ca app.kubernetes.io/managed-by=Helm
      kubectl annotate secret -n kube-system cilium-ca meta.helm.sh/release-name=cilium
      kubectl annotate secret -n kube-system cilium-ca meta.helm.sh/release-namespace=kube-system
    fi
    cilium install --wait \
      --set cluster.id=${id} \
      --set ipam.mode=kubernetes \
      --set envoy.enabled=false \
      --set encryption.enabled=true \
      --set encryption.type=wireguard \
      --set devices=ens+ \
      --set l2announcements.enabled=true \
      --set externalIPs.enabled=true \
      --set socketLB.enabled=true \
      --set socketLB.hostNamespaceOnly=true \
      --set k8sClientRateLimit.qps=50 \
      --set k8sClientRateLimit.burst=100

- path: /usr/local/bin/kubernetes-create-config.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    ipaddr=$(ifconfig | grep 'inet[^6]' | awk '{print $2}' | grep -v '127.0.0.1')
    hostname=$(hostname)
    clusterName=${hostname%-*}
    podSubnet=${1-10.244.0.0/16}
    serviceSubnet=${2-10.96.0.0/12}
    ctrlEndpoint=${3-${ipaddr}}
    cat <<EOF > /etc/kubernetes/kubeadm-config.yaml
    ---
    apiVersion: kubeadm.k8s.io/v1beta4
    kind: InitConfiguration
    localAPIEndpoint:
      advertiseAddress: "${ipaddr}"
    skipPhases:
    - addon/kube-proxy
    ---
    apiVersion: kubeadm.k8s.io/v1beta4
    kind: ClusterConfiguration
    clusterName: "${clusterName}"
    controlPlaneEndpoint: "${ctrlEndpoint}"
    networking:
      podSubnet: "${podSubnet}"
      serviceSubnet: "${serviceSubnet}"
    ---
    kind: KubeletConfiguration
    apiVersion: kubelet.config.k8s.io/v1beta1
    cgroupDriver: systemd
    EOF

- path: /usr/local/bin/kubernetes-setup-single-master.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    id=${1-1}
    kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml
    kubeadm token create --print-join-command > /tmp/setup_worker.sh
    /usr/local/bin/kubernetes-setup-cni.sh ${id}
    /usr/local/bin/kubernetes-setup-kubectl.sh

- path: /usr/local/bin/kubernetes-setup-primary-master.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    id=${1-1}
    kubeadm init --config=/etc/kubernetes/kubeadm-config.yaml --upload-certs
    cert_key=$(sudo kubeadm init phase upload-certs --upload-certs | tail -n 1)
    kubeadm token create --print-join-command --certificate-key $cert_key > /tmp/setup_secondary_master.sh
    kubeadm token create --print-join-command > /tmp/setup_worker.sh
    /usr/local/bin/kubernetes-setup-cni.sh ${id}
    /usr/local/bin/kubernetes-setup-kubectl.sh

- path: /usr/local/bin/kubernetes-setup-secondary-master.sh
  permissions: '0755'
  content: |
    #!/bin/bash
    set -eu
    if ! [ -f "/tmp/setup_secondary_master.sh" ]; then
      echo "Cannot find join script..."
      exit
    fi
    ipaddr=$(ifconfig | grep 'inet[^6]' | awk '{print $2}' | grep -v '127.0.0.1')
    $(cat /tmp/setup_secondary_master.sh) --apiserver-advertise-address=${ipaddr}

packages:
- net-tools
- apt-transport-https
- ca-certificates
- curl
- software-properties-common
- gnupg2
- containerd

runcmd:
- systemctl restart systemd-sysctl
- ufw disable
- timedatectl set-timezone America/New_York
- timedatectl set-ntp on
- systemctl stop apparmor
- systemctl disable apparmor
- modprobe overlay
- modprobe br_netfilter
- swapoff -a
- sed -i '/ swap / s/^\(.*\)$/#\1/g' /etc/fstab
- mkdir -p /etc/containerd
- containerd config default | sed '/SystemdCgroup/s/false/true/' > /etc/containerd/config.toml
- systemctl restart containerd
- systemctl enable containerd
- kubernetes-install.sh