feat: tailscale correct node destroy

loic-roux-404 · Jul 13, 2024 · 9e32967 · 9e32967
1 parent c53aad0
commit 9e32967
Show file tree

Hide file tree

Showing 26 changed files with 448 additions and 218 deletions.
diff --git a/.github/workflows/release-nixos.yml b/.github/workflows/release-nixos.yml
@@ -6,11 +6,8 @@ on:
     paths:
       - 'nixos/**.nix'
       - 'nixos-options/**.nix'
-      - 'nixos-generators/**.nix'
       - flake.nix
       - flake.lock
-      - '!**.md'
-      - 'packer/**'
       - .github/workflows/release-nixos.yml
 
 permissions:

diff --git a/nixos/configuration.nix b/nixos/configuration.nix
@@ -14,12 +14,6 @@ let
     url = "https://github.com/cert-manager/cert-manager/releases/download/v1.14.4/cert-manager.crds.yaml";
     sha256 = "060bn3gvrr5jphaig1g195prip5rn0x1s7qrp09q47719fgc6636";
   };
-  manifests = [{
-    file = certManagerCrds;
-    toWait = "crd/certificates.cert-manager.io";
-    namespace = "";
-    condition = "condition=established";
-  }];
 in {
 
   fileSystems."/" = {
@@ -65,7 +59,7 @@ in {
     tailscale = {
       enable = true;
       openFirewall = true;
-      extraUpFlags = ["--ssh"];
+      extraUpFlags = ["--ssh" "--accept-dns"];
       extraDaemonFlags = tailscale.baseDaemonExtraArgs;
       permitCertUid = user.name;
     };
@@ -99,13 +93,20 @@ in {
   };
 
   system.activationScripts.k3sCerts.text = (pkgs.callPackage ./install-k3s-manifest.nix { 
-    inherit lib pkgs manifests;
+    inherit pkgs;
+    manifest = {
+      file = certManagerCrds;
+      toWait = "crd/certificates.cert-manager.io";
+      namespace = "";
+      condition = "condition=established";
+    };
   }).script;
 
   environment = {
     shells = [ pkgs.bashInteractive ];
     variables = {
       EDITOR = "vim";
+      SYSTEMD_EDITOR = "vim";
       PAGER = "less -FirSwX";
     };
     systemPackages = with pkgs; [

diff --git a/nixos/contabo.nix b/nixos/contabo.nix
@@ -1,4 +1,4 @@
-{ ... }:
+{ pkgs, ... }:
 {
 
   boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "virtio_pci" "virtio_scsi" "sd_mod" ];
@@ -8,6 +8,42 @@
 
   swapDevices = [ ];
 
+  security.sudo.wheelNeedsPassword = true;
+  security.sudo = {
+    enable = true;
+    extraRules = [{
+      commands = map (cmd: {
+        command = cmd;
+        options = [ "NOPASSWD" ];
+      }) [
+        "${pkgs.systemd}/bin/systemctl status"
+        "${pkgs.systemd}/bin/systemctl show"
+        "${pkgs.systemd}/bin/systemctl list-units"
+        "${pkgs.systemd}/bin/systemctl list-machines"
+        "${pkgs.systemd}/bin/systemctl list-jobs"
+        "${pkgs.systemd}/bin/systemctl is-system-running"
+        "${pkgs.systemd}/bin/journalctl"
+        "${pkgs.k3s}/bin/kubectl get"
+        "${pkgs.k3s}/bin/kubectl describe"
+        "${pkgs.k3s}/bin/kubectl explain"
+        "${pkgs.k3s}/bin/kubectl logs"
+        "${pkgs.k3s}/bin/kubectl diff"
+        "${pkgs.k3s}/bin/kubectl events"
+        "${pkgs.k3s}/bin/kubectl wait"
+        "${pkgs.k3s}/bin/kubectl api-resources"
+        "${pkgs.k3s}/bin/kubectl version"
+        "${pkgs.vim}/bin/vim"
+        "${pkgs.less}/bin/less"
+        "${pkgs.coreutils}/bin/tail"
+        "${pkgs.coreutils}/bin/grep"
+        "${pkgs.nettools}/bin/ifconfig"
+        "${pkgs.iproute2}/bin/ip"
+        "${pkgs.iptables}/bin/iptables"
+      ];
+      groups = [ "wheel" ];
+    }];
+  };
+
   k3s-paas.dns.name = "404-tools.xyz";
   k3s-paas.certs = [];
 }
diff --git a/nixos/install-k3s-manifest.nix b/nixos/install-k3s-manifest.nix
@@ -1,16 +1,15 @@
 {
-  lib,
   pkgs,
-  manifests ? []
+  manifest,
+  ...
 } : 
-{ 
-  script = "mkdir -p /var/lib/rancher/k3s/server/manifests;" +
-    lib.strings.concatMapStrings (manifest: 
-      with manifest; 
-      let namespaceExpr = if namespace != "" then "-n ${namespace}" else ""; in 
-      ''
-        cp -fp ${file} /var/lib/rancher/k3s/server/manifests;
-        ${pkgs.k3s}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
-      ''
-    ) manifests;
+with manifest;
+let namespaceExpr = if namespace != "" then "-n ${namespace}" else ""; 
+in { 
+  script = ''
+    mkdir -p /var/lib/rancher/k3s/server/manifests;
+    cp -fp ${file} /var/lib/rancher/k3s/server/manifests;
+    sleep 30;
+    ${pkgs.k3s}/bin/kubectl wait --for='${condition}' ${toWait} ${namespaceExpr} --timeout=2m;
+  '';
 }
diff --git a/nixos/tailscale-deploy.nix b/nixos/tailscale-deploy.nix
@@ -1,36 +1,13 @@
-{ config, pkgs, lib, ... }: 
+{ config, pkgs, ... }: 
 
-let manifests = [
-  {
-    file = config.sops.templates."tailscale.yaml".path;
-    toWait = "deployment.apps/operator";
-    namespace = "tailscale";
-    condition = "condition=Available";
-  } 
-  {
-    file = pkgs.writeText "tailscale-namespace.yaml" ''
-      apiVersion: v1
-      kind: Namespace
-      metadata:
-        name: tailscale
-    '';
-    condition = "jsonpath={.status.phase}=Active";
-    toWait = "namespace/tailscale";
-    namespace = "";
-  }
-];
-in {
-  services.tailscale.authKeyFile = config.sops.secrets.tailscale.path;
-  services.tailscale.extraUpFlags = ["--ssh" "--hostname=${config.networking.hostName}"];
+{
+  services.tailscale.authKeyFile = config.sops.secrets.tailscaleNodeKey.path;
+  services.tailscale.extraUpFlags = ["--ssh" "--accept-dns" ];
 
-  system.activationScripts.tailscaleOperator.deps = [ "renderSecrets" ];
-  system.activationScripts.tailscaleOperator.text = (pkgs.callPackage ./install-k3s-manifest.nix { 
-    inherit lib pkgs manifests;
-  }).script;
-
-  sops.secrets.tailscale = {};
-  sops.secrets.tailscale_oauth_client_id = {};
-  sops.secrets.tailscale_oauth_client_secret = {};
+  sops.secrets.tailscaleNodeKey = {};
+  sops.secrets.tailscaleNodeHostname = {};
+  sops.secrets.tailscaleOauthClientId = {};
+  sops.secrets.tailscaleOauthClientSecret = {};
 
   sops.templates."tailscale.yaml".content = ''
     apiVersion: helm.cattle.io/v1
@@ -43,12 +20,39 @@ in {
       chart: tailscale-operator
       targetNamespace: tailscale
       valuesContent: |
+        operatorConfig:
+          hostname: "k8s-operator-${config.sops.placeholder.tailscaleNodeHostname}"
         oauth:
-          clientId: ${config.sops.placeholder.tailscale_oauth_client_id}
-          clientSecret: ${config.sops.placeholder.tailscale_oauth_client_secret}
+          clientId: ${config.sops.placeholder.tailscaleOauthClientId}
+          clientSecret: ${config.sops.placeholder.tailscaleOauthClientSecret}
         apiServerProxyConfig:
           mode: "true"
       waitForJobs: true
       waitForHelm: true
   '';
+
+  system.activationScripts.tailscaleNamespace.text = (pkgs.callPackage ./install-k3s-manifest.nix { 
+    inherit pkgs;
+    manifest = {
+      file = pkgs.writeText "tailscale-namespace.yaml" ''
+        apiVersion: v1
+        kind: Namespace
+        metadata:
+          name: tailscale
+      '';
+      condition = "jsonpath={.status.phase}=Active";
+      toWait = "namespace/tailscale";
+      namespace = "";
+    };
+  }).script;
+  system.activationScripts.tailscaleOperator.deps = [ "renderSecrets" "tailscaleNamespace" ];
+  system.activationScripts.tailscaleOperator.text = (pkgs.callPackage ./install-k3s-manifest.nix { 
+    inherit pkgs;
+    manifest = {
+      file = config.sops.templates."tailscale.yaml".path;
+      toWait = "deployment.apps/operator";
+      namespace = "tailscale";
+      condition = "condition=Available";
+    };
+  }).script;
 }
diff --git a/tf-modules-cloud/contabo/main.tf b/tf-modules-cloud/contabo/main.tf
@@ -1,7 +1,7 @@
 resource "contabo_secret" "k3s_paas_master_trusted_key" {
   name  = "k3s_paas_master_trusted_key"
   type  = "ssh"
-  value = var.ssh_connection.public_key
+  value = file(var.ssh_connection.public_key)
 }
 
 resource "contabo_image" "k3s_paas_master_image" {
@@ -23,15 +23,15 @@ resource "contabo_instance" "k3s_paas_master" {
   ssh_keys             = [contabo_secret.k3s_paas_master_trusted_key.id]
 }
 
-output "name" {
+output "node_hostname" {
   depends_on = [ contabo_instance.k3s_paas_master ]
   value = contabo_instance.k3s_paas_master.name
 }
 
-output "ip" {
+output "node_ip" {
   value = data.contabo_instance.k3s_paas_master.ip_config[0].v4[0].ip
 }
 
-output "id" {
+output "node_id" {
   value = contabo_instance.k3s_paas_master.id
 }
diff --git a/tf-modules-cloud/contabo/variables.tf b/tf-modules-cloud/contabo/variables.tf
@@ -14,7 +14,6 @@ variable "image_url_format" {
 variable "ssh_connection" {
   type = object({
     user        = string
-    password    = string
     public_key  = string
     private_key = string
   })

diff --git a/tf-modules-cloud/tailscale/delete-node-devices.sh b/tf-modules-cloud/tailscale/delete-node-devices.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+ENDPOINT=${ENDPOINT:-https://api.tailscale.com}
+API_KEY=$(curl -s -d "client_id=$OAUTH_CLIENT_ID" -d "client_secret=$OAUTH_CLIENT_SECRET" \
+  "${ENDPOINT}/api/v2/oauth/token" | jq -r '.access_token')
+NODE_HOSTNAMES=${NODE_HOSTNAMES:-}
+
+IFS=',' read -ra ADDR <<< "$NODE_HOSTNAMES"
+for NODE_HOSTNAME in "${ADDR[@]}"; do
+
+    curl -s "${ENDPOINT}/api/v2/tailnet/$TAILNET/devices" -u "$API_KEY:" | jq -r '.devices[] |  "\(.id) \(.name)"' |
+    while read -r id name; do
+        if [[ $name = *"$NODE_HOSTNAME.$TAILNET"* ]]
+        then
+        echo "$name matching $NODE_HOSTNAME.$TAILNET - getting rid of $id"
+        curl -s -X DELETE "${ENDPOINT}/api/v2/device/$id" -u "$API_KEY:"
+        else
+        echo "$name not matching $NODE_HOSTNAME.$TAILNET, keeping it"
+        fi
+    done
+done
diff --git a/tf-modules-cloud/tailscale/main.tf b/tf-modules-cloud/tailscale/main.tf
@@ -34,9 +34,9 @@ resource "tailscale_acl" "as_json" {
         users : ["autogroup:nonroot"]
       },
       {
-        action: "accept",
-        src: ["autogroup:member"],
-        dst: ["tag:k8s-operator"],
+        action : "accept",
+        src : ["autogroup:member"],
+        dst : ["tag:k8s-operator"],
         users : ["autogroup:nonroot"]
       },
     ],
@@ -69,17 +69,53 @@ resource "tailscale_dns_preferences" "sample_preferences" {
   magic_dns = true
 }
 
+resource "terraform_data" "node_changed" {
+  triggers_replace = [var.node_id]
+}
+
 resource "tailscale_tailnet_key" "k3s_paas_node" {
   depends_on          = [tailscale_acl.as_json]
   reusable            = true
-  ephemeral           = true
+  ephemeral           = false
   expiry              = 3600
   recreate_if_invalid = "always"
   preauthorized       = true
   description         = "VM instance key"
   tags                = ["tag:all"]
 }
 
-output "key" {
-  value = tailscale_tailnet_key.k3s_paas_node.key
+resource "terraform_data" "destroy_node" {
+  input = {
+    TAILNET             = var.tailscale_tailnet
+    OAUTH_CLIENT_ID     = var.tailscale_oauth_client.id
+    OAUTH_CLIENT_SECRET = var.tailscale_oauth_client.secret
+    NODE_HOSTNAMES      = join(",", [
+      var.node_hostname, 
+      "k8s-operator-${var.node_hostname}"
+    ])
+  }
+
+  provisioner "local-exec" {
+    when        = destroy
+    environment = self.input
+    on_failure  = fail
+    command     = "${path.module}/delete-node-devices.sh"
+  }
+}
+
+output "node_id" {
+  value = var.node_id
+}
+
+output "node_ip" {
+  value = var.node_ip
+}
+
+output "config" {
+  depends_on = [tailscale_tailnet_key.k3s_paas_node]
+  value = {
+    node_hostname = var.node_hostname
+    node_fqdn     = "${var.node_hostname}.${var.tailscale_tailnet}"
+    node_key      = tailscale_tailnet_key.k3s_paas_node.key
+  }
 }
diff --git a/tf-modules-cloud/tailscale/variables.tf b/tf-modules-cloud/tailscale/variables.tf
@@ -9,3 +9,23 @@ variable "trusted_ssh_user" {
 variable "tailscale_tailnet" {
   type = string
 }
+
+variable "node_hostname" {
+  type = string
+}
+
+variable "node_ip" {
+  type = string
+}
+
+variable "node_id" {
+  type = string
+}
+
+variable "tailscale_oauth_client" {
+  sensitive = true
+  type = object({
+    id     = string
+    secret = string
+  })
+}
diff --git a/tf-modules-k8s/waypoint-config/main.tf b/tf-modules-k8s/waypoint-config/main.tf
@@ -32,8 +32,8 @@ locals {
   ])
 }
 
-resource "null_resource" "setup_oidc" {
-  triggers = {
+resource "terraform_data" "setup_oidc" {
+  triggers_replace = {
     login_cmd      = local.login_cmd
     oidc_setup_cmd = local.oidc_setup_cmd
   }

diff --git a/tf-modules-nix/deploy/key-to-age.sh b/tf-modules-nix/deploy/key-to-age.sh
@@ -2,6 +2,6 @@
 
 eval "$(jq -r '@sh "key=\(.key) args=\(.args)"')"
 
-OUTPUT=$(echo "$key" | ssh-to-age "${args:-}")
+OUTPUT=$(ssh-to-age "${args:-}" < "$key")
 
 jq -n --arg output "$OUTPUT" '{"key": $output}'