diff --git a/Makefile b/Makefile index ccd9a38..54bb42e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ bootstrap-aarch64-linux: bootstrap-x86_64-linux: @VARIANT=builder-x86 $(BUILDER_EXEC) echo "Started x86 environment" - @echo "Waiting builder to" + @echo "Waiting builder to start..." @sleep 15 bootstrap: bootstrap-$(SYSTEM) @@ -26,12 +26,11 @@ bootstrap: bootstrap-$(SYSTEM) nixos-local: bootstrap build build: - @nix build .#nixosConfigurations.default --system $(SYSTEM) + @nix build .#nixosConfigurations.initial.config.formats.qcow --system $(SYSTEM) TERRAGRUNT_FILES:=$(shell find terragrunt -type d -name '.*' -prune -o -name 'terragrunt.hcl' -exec dirname {} \;) $(TERRAGRUNT_FILES): - @sudo chmod -fR 755 $@/.terragrunt-cache/ && sudo chmod -fR 755 result @cd $@ && terragrunt $(TF_CMD) release-stable: diff --git a/flake.lock b/flake.lock index 43fcade..ba713fa 100644 --- a/flake.lock +++ b/flake.lock @@ -7,11 +7,11 @@ ] }, "locked": { - "lastModified": 1724994893, - "narHash": "sha256-yutISDGg6HUaZqCaa54EcsfTwew3vhNtt/FNXBBo44g=", + "lastModified": 1725189302, + "narHash": "sha256-IhXok/kwQqtusPsoguQLCHA+h6gKvgdCrkhIaN+kByA=", "owner": "LnL7", "repo": "nix-darwin", - "rev": "c8d3157d1f768e382de5526bb38e74d2245cad04", + "rev": "7c4b53a7d9f3a3df902b3fddf2ae245ef20ebcda", "type": "github" }, "original": { @@ -62,11 +62,11 @@ ] }, "locked": { - "lastModified": 1724435763, - "narHash": "sha256-UNky3lJNGQtUEXT2OY8gMxejakSWPTfWKvpFkpFlAfM=", + "lastModified": 1725180166, + "narHash": "sha256-fzssXuGR/mCeGbzM1ExaTqDz7QDGta3WA4jJsZyRruo=", "owner": "nix-community", "repo": "home-manager", - "rev": "c2cd2a52e02f1dfa1c88f95abeb89298d46023be", + "rev": "471e3eb0a114265bcd62d11d58ba8d3421ee68eb", "type": "github" }, "original": { @@ -78,11 +78,11 @@ }, "nixlib": { "locked": { - "lastModified": 1724547350, - "narHash": "sha256-WKkGeNpenNMKD1gOF0Xuqi3VsKX/QCAiwz9qe5PDvzA=", + "lastModified": 1725152544, + "narHash": "sha256-Tm344cnFM9f2YZsgWtJduvhIrvLr3Bi8J4Xc+UZDKYE=", "owner": "nix-community", "repo": "nixpkgs.lib", - "rev": "b741d900fecd2f0c32d90f853b24be9f5f098b7d", + "rev": "7f0b9e4fbd91826cb9ce6babbc11c87903191051", "type": "github" }, "original": { @@ -100,11 +100,11 @@ ] }, "locked": { - "lastModified": 1724893087, - "narHash": "sha256-M3+Z8SSpzKPQ+/vw9a99G9HfqKWbVGzhFz4p3KAX0NI=", + "lastModified": 1725238763, + "narHash": "sha256-HYsDDHdjqpZf8XUZum0d+EjMSd88oDSvyunDKxzDJro=", "owner": "nix-community", "repo": "nixos-generators", - "rev": "0dd0205bc3f6d602ddb62aaece5f62a8715a9e85", + "rev": "89db25d85b0324cbec7c944c157c9550e5c576db", "type": "github" }, "original": { @@ -163,11 +163,11 @@ }, "nixpkgs-stable-darwin": { "locked": { - "lastModified": 1725062077, - "narHash": "sha256-ARdb2SNoV+zAN80CXeweNm3FZ8NWLmVna7mGKWVONeE=", + "lastModified": 1725140114, + "narHash": "sha256-tlRqsd84YFI7dL8Lz/Sm+M9Bm+Mh7kUs+5ArJbZsuy8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "be55bcada114b8e0385544b15cc4bc2148046aee", + "rev": "4927f77b7a68615ce99678086cd3dcd0eda34fdd", "type": "github" }, "original": { @@ -179,11 +179,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1724819573, - "narHash": "sha256-GnR7/ibgIH1vhoy8cYdmXE6iyZqKqFxQSVkFgosBh6w=", + "lastModified": 1725103162, + "narHash": "sha256-Ym04C5+qovuQDYL/rKWSR+WESseQBbNAe5DsXNx5trY=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "71e91c409d1e654808b2621f28a327acfdad8dc2", + "rev": "12228ff1752d7b7624a54e9c1af4b222b3c1073b", "type": "github" }, "original": { @@ -211,11 +211,11 @@ }, "nixpkgs_3": { "locked": { - "lastModified": 1724870369, - "narHash": "sha256-dGHUOi1tBiVOsVdT9QNEuk+FuSMtQxkyx+9CN/34kkk=", + "lastModified": 1725183711, + "narHash": "sha256-gkjg8FfjL92azt3gzZUm1+v+U4y+wbQE630uIf4Aybo=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "215ea7473ff80eb6cb157ee07223920cc53f4b09", + "rev": "a2c345850e5e1d96c62e7fa8ca6c9d77ebad1c37", "type": "github" }, "original": { @@ -225,26 +225,6 @@ "type": "github" } }, - "rke2": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1718609143, - "narHash": "sha256-HWDmtyLzohQb9kHI2AVKzb91EJTBi5YPnh+lKrjSOCY=", - "owner": "numtide", - "repo": "nixos-rke2", - "rev": "c28d68bac74a55e6dc5c32147b00e2c4620278a3", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "nixos-rke2", - "type": "github" - } - }, "root": { "inputs": { "darwin": "darwin", @@ -260,7 +240,6 @@ ], "nixpkgs-stable-darwin": "nixpkgs-stable-darwin", "nixpkgs-unstable": "nixpkgs-unstable", - "rke2": "rke2", "sops-nix": "sops-nix", "srvos": "srvos" } @@ -271,11 +250,11 @@ "nixpkgs-stable": "nixpkgs-stable" }, "locked": { - "lastModified": 1723501126, - "narHash": "sha256-N9IcHgj/p1+2Pvk8P4Zc1bfrMwld5PcosVA0nL6IGdE=", + "lastModified": 1725201042, + "narHash": "sha256-lj5pxOwidP0W//E7IvyhbhXrnEUW99I07+QpERnzTS4=", "owner": "Mic92", "repo": "sops-nix", - "rev": "be0eec2d27563590194a9206f551a6f73d52fa34", + "rev": "5db5921e40ae382d6716dce591ea23b0a39d96f7", "type": "github" }, "original": { @@ -289,11 +268,11 @@ "nixpkgs": "nixpkgs_3" }, "locked": { - "lastModified": 1725040185, - "narHash": "sha256-hOv19L8aRprqdm1Jz7T4kT8h/ckdj8BgLtLSNOOj+RE=", + "lastModified": 1725238235, + "narHash": "sha256-T6K8odVAi3l41REzrVcEblEfQS5gY1aB+0mnwjjdVpg=", "owner": "numtide", "repo": "srvos", - "rev": "b8e10788e84670049b30dc11d4c5893aedf7b32b", + "rev": "a6ee5a7167866cc1a72609e1782d86be1de4acce", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index e6328ab..ceec92b 100644 --- a/flake.nix +++ b/flake.nix @@ -24,11 +24,6 @@ inputs.nixpkgs.follows = "srvos/nixpkgs"; }; - rke2 = { - url = "github:numtide/nixos-rke2"; - inputs.nixpkgs.follows = "nixpkgs"; - }; - # Flake utilities flake-compat = { url = "github:edolstra/flake-compat"; flake = false; }; flake-utils.url = "github:numtide/flake-utils"; @@ -90,13 +85,14 @@ }); nixosModules = { - rke2 = inputs.rke2.nixosModules.default; sops = inputs.sops-nix.nixosModules.sops; common = srvos.nixosModules.common; server = srvos.nixosModules.server; home-manager = inputs.home-manager.nixosModules.home-manager; os = ./nixos/configuration.nix; config = ./nixos-options/default.nix; + qcowCompressed = ./nixos/qcow-compressed.nix; + allFormats = nixos-generators.nixosModules.all-formats; }; nixosAllModules = rec { @@ -142,26 +138,19 @@ // flake-utils.lib.eachDefaultSystem (baseSystem: { packages.nixosConfigurations = let - system = builtins.replaceStrings ["darwin"] ["linux"] baseSystem; + rebuildSystem = (builtins.getEnv "NIXOS_REBUILD_SYSTEM"); + system = if rebuildSystem != "" then rebuildSystem else "x86_64-linux"; oldLegacyPackages = import inputs.nixpkgs-legacy (nixpkgsDefaults // { inherit system; }); specialArgs = { inherit oldLegacyPackages; }; - qcowSystemFormat = [ - ({ ... }: { - imports = [ - nixos-generators.nixosModules.all-formats - ./nixos/qcow-compressed.nix - ]; - nixpkgs.hostPlatform = system; - }) - ]; + in { ## Libvirt configurations initial = nixosSystem { inherit system specialArgs; - modules = qcowSystemFormat ++ self.nixosAllModules.default; + modules = self.nixosAllModules.default; }; deploy = nixosSystem { @@ -173,7 +162,7 @@ initial-contabo = nixosSystem { inherit system specialArgs; - modules = qcowSystemFormat ++ self.nixosAllModules.contabo; + modules = self.nixosAllModules.contabo; }; deploy-contabo = nixosSystem { diff --git a/nixos-darwin/configuration-x86.nix b/nixos-darwin/configuration-x86.nix index a54bf47..0881d7d 100644 --- a/nixos-darwin/configuration-x86.nix +++ b/nixos-darwin/configuration-x86.nix @@ -1,6 +1,7 @@ { pkgs, lib, ... }: { + nix.settings.extra-platforms = [ "x86_64-linux" ]; nix.linux-builder = { package = pkgs.darwin.linux-builder-x86_64; ephemeral = lib.mkDefault true; diff --git a/nixos-darwin/configuration.nix b/nixos-darwin/configuration.nix index ce4bc76..494dfb2 100644 --- a/nixos-darwin/configuration.nix +++ b/nixos-darwin/configuration.nix @@ -100,6 +100,7 @@ with config.k3s-paas; keep-outputs = false; # https://github.com/NixOS/nix/issues/7273 auto-optimise-store = false; + extra-platforms = [ "x86_64-linux" ]; }; nix.gc = { automatic = true; diff --git a/nixos-options/default.nix b/nixos-options/default.nix index 0433d00..ebefe35 100644 --- a/nixos-options/default.nix +++ b/nixos-options/default.nix @@ -19,9 +19,8 @@ dns.dest-ips = lib.mkOption { default = [ - "127.0.0.1" "192.168.205.2" "192.168.205.3" - "192.168.205.4" "192.168.205.5" "192.168.205.6" - "192.168.205.7" "192.168.205.8" "192.168.205.9" + "127.0.0.1" "192.168.205.2" "192.168.205.3" "192.168.205.4" "192.168.205.5" + "192.168.205.6" "192.168.205.7" "192.168.205.8" "192.168.205.9" ]; type = lib.types.listOf lib.types.str; description = "Target IP address for dns.name (only in local dev)"; @@ -46,7 +45,7 @@ }; k3s.disableServices = lib.mkOption { - default = ["traefik" "rke2-ingress-nginx" "servicelb" ]; + default = ["traefik" "servicelb" ]; type = lib.types.listOf lib.types.str; description = "Disable k8s services eg: traefik,servicelb"; }; diff --git a/nixos/configuration.nix b/nixos/configuration.nix index a80d3a6..848851b 100644 --- a/nixos/configuration.nix +++ b/nixos/configuration.nix @@ -101,16 +101,40 @@ in { ''; }; - systemd.services.numtide-rke2.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin:${pkgs.coreutils}/bin"; - services.numtide-rke2 = { + systemd.services.k3s.serviceConfig.Environment = "PATH=${pkgs.tailscale}/bin:${pkgs.coreutils}/bin"; + services.k3s = { enable = lib.mkDefault false; role = "server"; - extraFlags = ( - builtins.concatMap (service: ["--disable" service]) k3s.disableServices + package = k3sPkg; + extraFlags = lib.strings.concatStringsSep " " ( + map (service: "--disable=${service}") k3s.disableServices ++ k3s.serverExtraArgs + ++ [ + "--flannel-backend=none" + "--disable-kube-proxy" + "--disable-network-policy" + "--egress-selector-mode=disabled" + ] ); - }; + # manifests = { + # certManager = { + # name = "cert-manager"; + # namespace = certManagerNamespace; + # createNamespace = true; + # repository = "https://charts.jetstack.io"; + # chart = "cert-manager"; + # version = "1.15.2"; + # waitForJobs = true; + # atomic = true; + # timeout = 120; + # values = '' + # crds: + # enabled = true + # ''; + # }; + # }; + }; services.fail2ban.enable = true; security.pki.certificateFiles = certs; @@ -121,7 +145,7 @@ in { xdg.enable = true; home.stateVersion = "24.05"; home.sessionVariables = { - KUBECONFIG = "/etc/rancher/rke2/rke2.yaml"; + KUBECONFIG = "/etc/rancher/k3s/k3s.yaml"; }; home.shellAliases = { kubectl = "sudo -E kubectl"; diff --git a/nixos/deploy.nix b/nixos/deploy.nix index e3e48d1..c60603d 100644 --- a/nixos/deploy.nix +++ b/nixos/deploy.nix @@ -28,71 +28,16 @@ with config.k3s-paas; sops.secrets.nodeIp = {}; sops.secrets.internalNodeIp = {}; - sops.secrets.nodePrivateKey = {}; + sops.secrets.nodePrivateKey = { + neededForUsers = true; + }; sops.secrets.tailscaleNodeKey = {}; sops.secrets.paasDomain = {}; sops.secrets.tailscaleDomain = {}; sops.secrets.password = { neededForUsers = true; }; - services.numtide-rke2.enable = true; - services.numtide-rke2.configFile = config.sops.templates."config.yaml".path; - services.numtide-rke2.manifests = { - "cilium-config.yaml" = config.sops.templates."cilium-config.yaml".path; - }; - - sops.templates."cilium-config.yaml".content = '' - apiVersion: helm.cattle.io/v1 - kind: HelmChartConfig - metadata: - name: rke2-cilium - namespace: kube-system - spec: - valuesContent: |- - ipam: - operator: - clusterPoolIPv4PodCIDRList: ["10.100.0.0/16"] - k8sServiceHost: ${config.sops.placeholder.internalNodeIp} - k8sServicePort: 6443 - l2announcements: - enabled: true - kubeProxyReplacement: true - bpf: - masquerade: true - lbExternalClusterIP: false - gatewayAPI: - enabled: false - routingMode: "tunnel" - tunnelProtocol: "vxlan" - ingressController: - enabled: true - default: true - loadbalancerMode: "dedicated" - service: - name: "cilium-ingress-external" - labels: - "k3s-paas/internal": "true" - prometheus: - enabled: true - serviceMonitor: - enabled: true - operator: - replicas: 1 - prometheus: - enabled: true - hubble: - relay: - enabled: true - metrics: - enabled: - - "dns" - - "drop" - - "tcp" - - "flow" - - "port-distribution" - - "icmp" - - "httpV2:exemplars=true;labelsContext=source_ip,source_namespace,source_workload,destination_ip,destination_namespace,destination_workload,traffic_direction" - enableOpenMetrics: true - ''; + services.k3s.enable = true; + services.k3s.configPath = config.sops.templates."config.yaml".path; sops.templates."config.yaml".content = '' advertise-address: ${config.sops.placeholder.internalNodeIp} @@ -110,28 +55,11 @@ with config.k3s-paas; - "${config.sops.placeholder.tailscaleDomain}" - "${config.sops.placeholder.nodeIp}" - "${config.sops.placeholder.internalNodeIp}" - - cni: cilium - protect-kernel-defaults: true - - kube-apiserver-arg: - - '--authorization-mode=Node,RBAC' - - '--oidc-issuer-url=https://dex.${config.sops.placeholder.paasDomain}' - - '--oidc-client-id=${dex.dexClientId}' - - '--oidc-username-claim=email' - - '--oidc-groups-claim=groups' - - '--default-not-ready-toleration-seconds=30' - - '--default-unreachable-toleration-seconds=30' - - kube-controller-manager-arg: - - '--node-monitor-period=4s' - kubelet-arg: - - '--node-status-update-frequency=4s' - - '--max-pods=100' - - etcd-arg: "--quota-backend-bytes 2048000000" - etcd-snapshot-schedule-cron: "0 3 * * *" - etcd-snapshot-retention: 10 + kube-apiserver-arg=authorization-mode: Node,RBAC + kube-apiserver-arg=oidc-issuer-url: https://dex.${config.sops.placeholder.paasDomain} + kube-apiserver-arg=oidc-client-id: ${dex.dexClientId} + kube-apiserver-arg=oidc-username-claim: email + kube-apiserver-arg=oidc-groups-claim: groups ''; users.users.reader.hashedPasswordFile = config.sops.secrets.password.path; diff --git a/nixos/qcow-compressed.nix b/nixos/qcow-compressed.nix index e1d2c8e..5d8d366 100644 --- a/nixos/qcow-compressed.nix +++ b/nixos/qcow-compressed.nix @@ -4,7 +4,7 @@ ]; system.build.qcow = lib.mkForce (import "${toString modulesPath}/../lib/make-disk-image.nix" { inherit lib config pkgs; - diskSize = "auto"; + diskSize = "16"; format = "qcow2-compressed"; partitionTableType = "hybrid"; }); diff --git a/terragrunt/network/local/.terraform.lock.hcl b/terragrunt/network/local/.terraform.lock.hcl old mode 100644 new mode 100755 diff --git a/terragrunt/network/local/env.hcl b/terragrunt/network/local/env.hcl old mode 100644 new mode 100755 diff --git a/terragrunt/network/local/terragrunt.hcl b/terragrunt/network/local/terragrunt.hcl old mode 100644 new mode 100755 diff --git a/tf-modules-cloud/k3s-get-config/variables.tf b/tf-modules-cloud/k3s-get-config/variables.tf index f09cd15..c359ff2 100644 --- a/tf-modules-cloud/k3s-get-config/variables.tf +++ b/tf-modules-cloud/k3s-get-config/variables.tf @@ -17,7 +17,7 @@ variable "ssh_connection" { } variable "remote_k3s_config_location" { - default = "/etc/rancher/rke2/rke2.yaml" + default = "/etc/rancher/k3s/k3s.yaml" } variable "context_cluster_name" { diff --git a/tf-modules-cloud/libvirt/get-ip.sh b/tf-modules-cloud/libvirt/get-ip.sh index 7a7db83..fad9dfe 100755 --- a/tf-modules-cloud/libvirt/get-ip.sh +++ b/tf-modules-cloud/libvirt/get-ip.sh @@ -12,7 +12,8 @@ while [ -z "$ip_address" ] && [ $elapsed -lt ${timeout:-90} ]; do break fi - elapsed=$((elapsed + 10)) + sleep 1 + elapsed=$((elapsed + 1)) done diff --git a/tf-modules-cloud/libvirt/main.tf b/tf-modules-cloud/libvirt/main.tf index d7250c7..cb31752 100644 --- a/tf-modules-cloud/libvirt/main.tf +++ b/tf-modules-cloud/libvirt/main.tf @@ -23,15 +23,15 @@ resource "libvirt_volume" "nixos_worker" { name = "nixos-worker.qcow2" base_volume_id = libvirt_volume.nixos.id pool = libvirt_pool.volumetmp.name - size = 16384 * 1024 * 1024 } resource "libvirt_domain" "machine" { name = var.node_hostname - vcpu = 2 + vcpu = 4 memory = 4096 type = "hvf" autostart = true + arch = var.arch disk { volume_id = libvirt_volume.nixos_worker.id @@ -53,10 +53,6 @@ resource "libvirt_domain" "machine" { type = "vga" } - cpu { - mode = "host-passthrough" - } - xml { xslt = templatefile("${path.module}/nixos.xslt.tmpl", { args = local.darwin_cmdline @@ -68,7 +64,7 @@ data "external" "get_ip" { depends_on = [ libvirt_domain.machine ] program = ["bash", "${path.module}/get-ip.sh"] query = { - timeout = 60 + timeout = 90 mac = var.mac } } diff --git a/tf-modules-cloud/libvirt/nixos.xslt.tmpl b/tf-modules-cloud/libvirt/nixos.xslt.tmpl index 3711ace..74b09e7 100644 --- a/tf-modules-cloud/libvirt/nixos.xslt.tmpl +++ b/tf-modules-cloud/libvirt/nixos.xslt.tmpl @@ -13,7 +13,7 @@ - + diff --git a/tf-modules-cloud/libvirt/variables.tf b/tf-modules-cloud/libvirt/variables.tf index 8a8bd55..3324396 100644 --- a/tf-modules-cloud/libvirt/variables.tf +++ b/tf-modules-cloud/libvirt/variables.tf @@ -5,7 +5,7 @@ variable "mac" { variable "vm_size" { description = "vm size in MB" - default = 8092 + default = "16G" } variable "darwin" { @@ -34,6 +34,10 @@ variable "node_hostname" { default = "localhost-0" } +variable "arch" { + default = "x86_64" +} + variable "libvirt_qcow_source" { default = "result/nixos.qcow2" } diff --git a/tf-root-network/main.tf b/tf-root-network/main.tf index 8c2fd32..790e5c7 100644 --- a/tf-root-network/main.tf +++ b/tf-root-network/main.tf @@ -66,7 +66,7 @@ module "k3s_get_config" { source = "../tf-modules-cloud/k3s-get-config" ssh_connection = var.ssh_connection node_hostname = module.deploy.config.node_address - remote_k3s_config_location = "/etc/rancher/rke2/rke2.yaml" + remote_k3s_config_location = "/etc/rancher/k3s/k3s.yaml" } output "password" { diff --git a/tf-root-network/variables.tf b/tf-root-network/variables.tf index a24b1d3..0c7c6bc 100644 --- a/tf-root-network/variables.tf +++ b/tf-root-network/variables.tf @@ -72,5 +72,5 @@ variable "nix_flake_reset" { } variable "remote_k3s_config_location" { - default = "/etc/rancher/rke2/rke2.yaml" + default = "/etc/rancher/k3s/k3s.yaml" }