fix: revert to k3s

loic-roux-404 · Sep 2, 2024 · 3f6c007 · 3f6c007
1 parent 1b90e30
commit 3f6c007
Show file tree

Hide file tree

Showing 19 changed files with 96 additions and 175 deletions.
diff --git a/Makefile b/Makefile
@@ -18,20 +18,19 @@ bootstrap-aarch64-linux:
 
 bootstrap-x86_64-linux:
 	@VARIANT=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment"
-	@echo "Waiting builder to"
+	@echo "Waiting builder to start..."
 	@sleep 15
 
 bootstrap: bootstrap-$(SYSTEM)
 
 nixos-local: bootstrap build
 
 build:
-	@nix build .#nixosConfigurations.default --system $(SYSTEM)
+	@nix build .#nixosConfigurations.initial.config.formats.qcow --system $(SYSTEM)
 
 TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;)
 
 $(TERRAGRUNT_FILES):
-	@sudo chmod -fR 755 $@/.terragrunt-cache/ && sudo chmod -fR 755 result
 	@cd $@ && terragrunt $(TF_CMD)
 
 release-stable:

diff --git a/flake.lock b/flake.lock
diff --git a/flake.nix b/flake.nix
@@ -24,11 +24,6 @@
       inputs.nixpkgs.follows = "srvos/nixpkgs";
     };
 
-    rke2 = {
-      url = "github:numtide/nixos-rke2";
-      inputs.nixpkgs.follows = "nixpkgs";
-    };
-
     # Flake utilities
     flake-compat = { url = "github:edolstra/flake-compat"; flake = false; };
     flake-utils.url = "github:numtide/flake-utils";
@@ -90,13 +85,14 @@
       });
 
       nixosModules = {
-        rke2 = inputs.rke2.nixosModules.default;
         sops = inputs.sops-nix.nixosModules.sops;
         common = srvos.nixosModules.common;
         server = srvos.nixosModules.server;
         home-manager = inputs.home-manager.nixosModules.home-manager;
         os = ./nixos/configuration.nix;
         config = ./nixos-options/default.nix;
+        qcowCompressed = ./nixos/qcow-compressed.nix;
+        allFormats = nixos-generators.nixosModules.all-formats;
       };
 
       nixosAllModules = rec {
@@ -142,26 +138,19 @@
     // flake-utils.lib.eachDefaultSystem (baseSystem:
     {
       packages.nixosConfigurations = let
-        system = builtins.replaceStrings ["darwin"] ["linux"] baseSystem;
+        rebuildSystem = (builtins.getEnv "NIXOS_REBUILD_SYSTEM");
+        system = if rebuildSystem != "" then rebuildSystem else "x86_64-linux";
         oldLegacyPackages = import inputs.nixpkgs-legacy (nixpkgsDefaults // { inherit system; });
         specialArgs = {
           inherit oldLegacyPackages;
         };
-        qcowSystemFormat = [
-          ({ ... }: { 
-            imports = [ 
-              nixos-generators.nixosModules.all-formats
-              ./nixos/qcow-compressed.nix
-            ];
-            nixpkgs.hostPlatform = system;
-          })
-        ];
+
       in {
         ## Libvirt configurations
 
         initial = nixosSystem {
           inherit system specialArgs;
-          modules = qcowSystemFormat ++ self.nixosAllModules.default;
+          modules = self.nixosAllModules.default;
         };
 
         deploy = nixosSystem {
@@ -173,7 +162,7 @@
 
         initial-contabo = nixosSystem {
           inherit system specialArgs;
-          modules = qcowSystemFormat ++ self.nixosAllModules.contabo;
+          modules = self.nixosAllModules.contabo;
         };
 
         deploy-contabo = nixosSystem {

diff --git a/nixos-darwin/configuration-x86.nix b/nixos-darwin/configuration-x86.nix
@@ -1,6 +1,7 @@
 {
   pkgs, lib, ...
 }: {
+  nix.settings.extra-platforms = [ "x86_64-linux" ];
   nix.linux-builder = {
     package = pkgs.darwin.linux-builder-x86_64;
     ephemeral = lib.mkDefault true;

diff --git a/nixos-darwin/configuration.nix b/nixos-darwin/configuration.nix
@@ -100,6 +100,7 @@ with config.k3s-paas;
     keep-outputs = false;
     # https://github.com/NixOS/nix/issues/7273
     auto-optimise-store = false;
+    extra-platforms = [ "x86_64-linux" ];
   };
   nix.gc = {
     automatic = true;

diff --git a/nixos-options/default.nix b/nixos-options/default.nix
@@ -19,9 +19,8 @@
 
     dns.dest-ips = lib.mkOption {
       default = [
-        "127.0.0.1" "192.168.205.2" "192.168.205.3" 
-        "192.168.205.4" "192.168.205.5" "192.168.205.6"
-        "192.168.205.7" "192.168.205.8" "192.168.205.9"
+        "127.0.0.1" "192.168.205.2" "192.168.205.3" "192.168.205.4" "192.168.205.5" 
+        "192.168.205.6" "192.168.205.7" "192.168.205.8" "192.168.205.9"
       ];
       type = lib.types.listOf lib.types.str;
       description = "Target IP address for dns.name (only in local dev)";
@@ -46,7 +45,7 @@
     };
 
     k3s.disableServices = lib.mkOption {
-      default = ["traefik" "rke2-ingress-nginx" "servicelb" ];
+      default = ["traefik" "servicelb" ];
       type = lib.types.listOf lib.types.str;
       description = "Disable k8s services eg: traefik,servicelb";
     };

diff --git a/nixos/configuration.nix b/nixos/configuration.nix
@@ -101,16 +101,40 @@ in {
     ''; 
   };
 
-  systemd.services.numtide-rke2.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin:${pkgs.coreutils}/bin";
-  services.numtide-rke2 = {
+  systemd.services.k3s.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin:${pkgs.coreutils}/bin";
+  services.k3s = {
     enable = lib.mkDefault false;
     role = "server";
-    extraFlags = (
-      builtins.concatMap (service: ["--disable" service]) k3s.disableServices
+    package = k3sPkg;
+    extraFlags = lib.strings.concatStringsSep " " (
+      map (service: "--disable=${service}") k3s.disableServices
       ++ k3s.serverExtraArgs
+      ++ [
+        "--flannel-backend=none"
+        "--disable-kube-proxy"
+        "--disable-network-policy"
+        "--egress-selector-mode=disabled"
+      ]
     );
-  };
+    # manifests = {
+    #   certManager = {
+    #     name = "cert-manager";
+    #     namespace = certManagerNamespace;
+    #     createNamespace = true;
+    #     repository = "https://charts.jetstack.io";
+    #     chart = "cert-manager";
+    #     version = "1.15.2";
+    #     waitForJobs = true;
+    #     atomic = true;
+    #     timeout = 120;
 
+    #     values = ''
+    #       crds:
+    #         enabled = true
+    #     '';
+    #   };
+    # };
+  };
   services.fail2ban.enable = true;
 
   security.pki.certificateFiles = certs;
@@ -121,7 +145,7 @@ in {
     xdg.enable = true;
     home.stateVersion = "24.05";
     home.sessionVariables = {
-      KUBECONFIG = "/etc/rancher/rke2/rke2.yaml";
+      KUBECONFIG = "/etc/rancher/k3s/k3s.yaml";
     };
     home.shellAliases = {
       kubectl = "sudo -E kubectl";