+ prometheus

oluceps · Mar 9, 2024 · 06fd59c · 06fd59c
1 parent cf7b796
commit 06fd59c
Show file tree

Hide file tree

Showing 23 changed files with 121 additions and 73 deletions.
diff --git a/age.nix b/age.nix
@@ -20,7 +20,7 @@
         genGlobalR = i: gen i "root" "root" "444";
       in
       (genProxys [ "rat" "ss" "sing" "hyst-us" "tuic" "naive" "dae.sub" "jc-do" "juic-san" "tuic-san" "caddy-lsa" "ss-az" ]) //
-      (genMaterial [ "minisign.key" "ssh-cfg" "gh-eu" "riro.u2f" "elen.u2f" "gh-token" "age" "pub" "id" "id_sk" "minio" "prism" "aws-s3-cred" "vault" "restic-repo" "restic-envs" ]) //
+      (genMaterial [ "minisign.key" "ssh-cfg" "gh-eu" "riro.u2f" "elen.u2f" "gh-token" "age" "pub" "id" "id_sk" "minio" "prism" "aws-s3-cred" "vault" "restic-repo" "restic-envs" "metric" ]) //
       (genBoot [ "db.key" "db.pem" ]) //
       (genWg [ "wg" "wgk" "wgy" "wga" "wgc-warp" "wge" ]) //
       (genGlobalR [ "ntfy-token" ]) //

diff --git a/f b/f
@@ -34,7 +34,7 @@ export def b [
   let machine_spec = "x86_64-linux - - - big-parallel"
 
   $nodes | par-each {|| 
-    (nix build $'.#nixosConfigurations.($in).config.system.build.toplevel'
+    (nom build $'.#nixosConfigurations.($in).config.system.build.toplevel'
       --builders $"($target_addr) ($machine_spec)"
       ...($job) -vvv) 
   }

diff --git a/hosts/colour/caddy.nix b/hosts/colour/caddy.nix
@@ -4,16 +4,53 @@
     "nyaw.cert"
     "nyaw.key"
   ];
+  systemd.services.caddy.serviceConfig.EnvironmentFile = [ config.age.secrets.restic-envs.path ];
   services.caddy = {
     enable = true;
-    package = pkgs.caddy-naive;
+    # package = pkgs.caddy-naive;
     settings = {
+      admin = {
+        listen = "unix//tmp/caddy.sock";
+        config.persist = false;
+      };
       apps = {
+        http.grace_period = "1s";
         http = {
           servers = {
             srv0 = {
               listen = [ ":443" ];
+              strict_sni_host = false;
+              metrics = { };
               routes = [
+                {
+                  match = [{
+                    host = [ config.networking.fqdn ];
+                    path = [ "/prom" "/prom/*" ];
+                  }];
+                  handle = [{
+                    handler = "reverse_proxy";
+                    upstreams = [{ dial = "10.0.2.2:9090"; }];
+                  }];
+                }
+
+                {
+                  match = [{
+                    host = [ config.networking.fqdn ];
+                    path = [ "/caddy" ];
+                  }];
+                  handle = [
+                    {
+                      handler = "authentication";
+                      providers.http_basic.accounts = [{
+                        username = "prometheus";
+                        password = "{env.AWS_ACCESS_KEY_ID}";
+                      }];
+                    }
+                    {
+                      handler = "metrics";
+                    }
+                  ];
+                }
                 {
                   handle = [{
                     handler = "subroute";
@@ -110,6 +147,7 @@
                 "api.heartrate.nyaw.xyz"
                 "api.atuin.nyaw.xyz"
                 "chat.nyaw.xyz"
+                config.networking.fqdn
               ];
             }];
           };

diff --git a/hosts/colour/network.nix b/hosts/colour/network.nix
@@ -3,6 +3,7 @@
 , ...
 }: {
   networking = {
+    domain = "nyaw.xyz";
     resolvconf.useLocalResolver = true;
     firewall = {
       checkReversePath = false;

diff --git a/hosts/hastur/default.nix b/hosts/hastur/default.nix
@@ -62,6 +62,7 @@
 
               ./vaultwarden.nix
               inputs.niri.nixosModules.niri
+              ./prometheus.nix
 
             ] ++ lib.sharedModules
             ++

diff --git a/hosts/hastur/network.nix b/hosts/hastur/network.nix
@@ -8,6 +8,7 @@
     # useHostResolvConf = true;
 
     hostName = "hastur"; # Define your hostname.
+    domain = "nyaw.xyz";
     # replicates the default behaviour.
     enableIPv6 = true;
     interfaces.eth0.wakeOnLan.enable = true;

diff --git a/hosts/hastur/prometheus.nix b/hosts/hastur/prometheus.nix
@@ -1,52 +1,34 @@
 { config, pkgs, lib, data, ... }:
 let
   # cfg = config.services.prometheus;
-  targets = map (n: "${n}.nyaw.xyz") data.withoutHeads;
+  targets = map (n: "${n}.nyaw.xyz") [ "nodens" "colour" ];
 in
 {
   services.prometheus = {
     enable = true;
-    webExternalUrl = "https://${config.networking.fqdn}/prom";
-    listenAddress = "127.0.0.1";
+    webExternalUrl = "https://colour.nyaw.xyz/prom";
+    listenAddress = "10.0.2.2";
     port = 9090;
     retentionTime = "7d";
     globalConfig = {
       scrape_interval = "1m";
       evaluation_interval = "1m";
     };
     scrapeConfigs = [
-      # {
-      #   job_name = "metrics";
-      #   scheme = "https";
-      #   basic_auth = {
-      #     username = "prometheus";
-      #     password_file = config.sops.secrets.prometheus.path;
-      #   };
-      #   static_configs = [{ inherit targets; }];
-      # }
       {
         job_name = "caddy";
         scheme = "https";
         basic_auth = {
           username = "prometheus";
-          password_file = config.age.secrets.wg.path;
+          password_file = config.age.secrets.metric.path;
         };
         metrics_path = "/caddy";
         static_configs = [{ inherit targets; }];
       }
       {
         job_name = "dns";
-        scheme = "http";
-        # basic_auth = {
-        #   username = "prometheus";
-        #   password_file = config.sops.secrets.prometheus.path;
-        # };
-        metrics_path = "/metric";
-        static_configs = [{ targets = [ "hastur.nyaw.xyz:9092" ]; }];
-        # relabel_configs = [{
-        #   source_labels = [ "__param_target" ];
-        #   target_label = "target";
-        # }];
+        metrics_path = "/metrics";
+        static_configs = [{ targets = [ "localhost:9092" ]; }];
       }
 
     ];

diff --git a/hosts/hastur/spec.nix b/hosts/hastur/spec.nix
@@ -122,7 +122,7 @@
 
     tailscale = { enable = true; openFirewall = true; };
 
-    sing-box.enable = true;
+    sing-box.enable = false;
     # beesd.filesystems = {
     #   os = {
     #     spec = "LABEL=nixos";

diff --git a/hosts/kaambl/network.nix b/hosts/kaambl/network.nix
@@ -19,6 +19,7 @@
     useDHCP = false;
 
     hostName = "kaambl"; # Define your hostname.
+    domain = "nyaw.xyz";
     # wireless.enable = true;  # Enables wireless support via wpa_supplicant.
     # The global useDHCP flag is deprecated, therefore explicitly set to false here.
     # Per-interface useDHCP will be mandatory in the future, so this generated config

diff --git a/hosts/nodens/caddy.nix b/hosts/nodens/caddy.nix
@@ -1,14 +1,40 @@
-{ pkgs, ... }: {
+{ pkgs, config, ... }: {
+
+  systemd.services.caddy.serviceConfig.EnvironmentFile = [ config.age.secrets.restic-envs.path ];
   services.caddy = {
     enable = true;
-    package = pkgs.caddy-naive;
+    # package = pkgs.caddy-naive;
     settings = {
+      admin = {
+        listen = "unix//tmp/caddy.sock";
+        config.persist = false;
+      };
       apps = {
         http = {
           servers = {
             srv0 = {
               listen = [ ":443" ];
+              strict_sni_host = false;
+              metrics = { };
               routes = [
+                {
+                  match = [{
+                    host = [ config.networking.fqdn ];
+                    path = [ "/caddy" ];
+                  }];
+                  handle = [
+                    {
+                      handler = "authentication";
+                      providers.http_basic.accounts = [{
+                        username = "prometheus";
+                        password = "{env.AWS_ACCESS_KEY_ID}";
+                      }];
+                    }
+                    {
+                      handler = "metrics";
+                    }
+                  ];
+                }
                 {
                   handle = [{
                     handler = "subroute";
@@ -270,6 +296,7 @@
                 "pb.nyaw.xyz"
                 "nyaw.xyz"
                 "api.heartrate.nyaw.xyz"
+                config.networking.fqdn
               ];
             }
               {

diff --git a/hosts/nodens/network.nix b/hosts/nodens/network.nix
@@ -3,6 +3,7 @@
 , ...
 }: {
   networking = {
+    domain = "nyaw.xyz";
     resolvconf.useLocalResolver = true;
     firewall = {
       checkReversePath = false;

diff --git a/sec/metric.age b/sec/metric.age
@@ -1,11 +1,10 @@
 age-encryption.org/v1
--> X25519 VYknc7h4HfiLcFbJYnuMHDyf2X9yzOFcn1c8m9EmkHM
-PwWZFNPANz4HyUYeJT6KbXbBEaQ0SqxV9dQsLT0M6+k
--> piv-p256 fV1VQA A9gtnflSQ/YFQhMJKzBtMQ/vy4/igTm5OylXbJq19sJO
-dtxLsOMqLZPEzM/t7oCDrzuUpWKFIUPMbtRinvpcVgM
--> "-grease g _q| w7y/<[[ 6M
-scD/5Hn0k95bc5dGwnR2/bs8f1FKHgRFd2lD1HEJOuQE/3v7PjdVLTXaOcfFxTzy
-Yi/QYXiWftUNt53nSRjc4mI
---- LFCT5b8UehCsebde4jjp5J2uyDqfhRvPnPxnGbka6+w
-��cau%��%6:1'Y<B==PQw
-z5<��0�`�EU�S��w���c��G�K��7L��( ��mt�
+-> X25519 kED1nmYH3QrS1qyHffBjVGyNtkWQosY5lGke0PFboFk
+9Ux3CGQ/nQydLRbLBQXACDB+wT/HW1rHRhIlPcQtWOM
+-> piv-p256 fV1VQA A9GAc2HSoVj15QKhXavQJJv32Da6EdGzEZBAnbBkrUyW
+YBeUPVn/7dybpZIyMUjjNcFplFoqRQUNfQE4JwCo3NM
+-> 4xjJrk@-grease $rtRH A}z/2>
+Xp8JTA5klR1imoXTyb2E4YPzIpJD3xEjnmrwrUWU0N3lwfV39QM112gpwJJ37y/q
+uJJ5CM9jhQwBtK+M2cJS/A1Mk8YFA8oN/w
+--- 3aSp82plRkz1b5McnhxLfEoaoDFj/pDGiTdNZY0fKb8
+K�YWW@����Js8�ު�Fv��ے7ط��^sN�JbOu�nQP!�FK(Ȥ
diff --git a/sec/rekeyed/colour/6e584e61f19b16753abba2fd3d038fda-metric.age b/sec/rekeyed/colour/6e584e61f19b16753abba2fd3d038fda-metric.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 vL61NA rKPecqgSo57SguEv9QAqPuK94jYYhYGS5FPchLcqYxQ
+l0AnucKg2gEW2Z+/y3y3h7Bf2Wtp3ckRqvbvjRbqxAg
+-> ^wvI/-grease p9
+
+--- PGQA4SNl6G9e8ytG5BcOoh1+ZFStBy+rgkWCZoEWodc
+3w�EM9��u�Ӏ�|k�rN.;V�POɺ����#2���%%8/;:�A|"b@
diff --git a/sec/rekeyed/colour/a1e0f81d0a5117da5add01bef725edb5-metric.age b/sec/rekeyed/colour/a1e0f81d0a5117da5add01bef725edb5-metric.age
diff --git a/sec/rekeyed/eihort/26364ed5718e377cf6a73de78489fb41-metric.age b/sec/rekeyed/eihort/26364ed5718e377cf6a73de78489fb41-metric.age
diff --git a/sec/rekeyed/eihort/677ba3456965d87cfb395ef13ba011cb-metric.age b/sec/rekeyed/eihort/677ba3456965d87cfb395ef13ba011cb-metric.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 /dedbw BF0hU3knrcoj2Pr1JgoPr+PsNRDZYbuRMn76n/4DAlo
+d0vR3ZmPIMMj1DT2HQ+An8ngWufHa33wbLEDs7u77fg
+-> =hD|i#qT-grease jHC ]T` dt^0[9 Z)yM.n|=
+svVF5KDnrVO5MNh8l4I/4WtPc76qWYHsh+nlSAE1Qond0iRNh3/9npiV+bWZTGk
+--- EDBrXnjx9tbBIJQz2f9XI3VnEPccTwDXvPlTGh/W2iY
+������o�ᤑ$�i�.��4Ɗ/�qn�	w��B����E��)Ċ�^r���
diff --git a/sec/rekeyed/hastur/111c06a823aaaf52d54b9d8165d0ba5f-metric.age b/sec/rekeyed/hastur/111c06a823aaaf52d54b9d8165d0ba5f-metric.age
diff --git a/sec/rekeyed/hastur/f8a2aef35a3d7423c6b58440516e0424-metric.age b/sec/rekeyed/hastur/f8a2aef35a3d7423c6b58440516e0424-metric.age
@@ -0,0 +1,8 @@
+age-encryption.org/v1
+-> ssh-ed25519 9tTW2g 0gzGzaCIjujnU6rKuKbn+7d0hK+MmOS0dG9gBggSfFw
+90P71GvYD7rWTtQjc+5YfR9WZvDgWToVurTV07VmFcc
+-> p.-grease o( xjd
+qgo4b63VHLfGAToRjtRjFaiBWNsDOVfpmt4zUYponO53JeIpNmEKTOeRPF1PWfvR
+Hntj9yEKlbZQy/ACgTuXAxxl/1JR2WE2RO2ojNQmplJzvyg/ZMaQTpkj0Te9
+--- CTu2KsYAbatetxFrmQgANSK1xQa3sp7Ljgi4RC/ODGA
+;��-¬��"~�҈��:T�C	R��[��7�WT�.��l�{�]��nNS
diff --git a/sec/rekeyed/kaambl/347b9885e18ede9df75372e94d50a14c-metric.age b/sec/rekeyed/kaambl/347b9885e18ede9df75372e94d50a14c-metric.age
@@ -0,0 +1,7 @@
+age-encryption.org/v1
+-> ssh-ed25519 rUtFuA QbOFDwqhojvV7aUVRtGosO3SWcdG/XScIGim2bnfFz8
+PyQsU+MP91ZeqaRw1TqvxBmxrj+mXMXDxpsLoIYg8UU
+-> Odxt-grease d
+q6GyuAsB
+--- DgF3yLQKPEpW2sO+AuxowpJR/X5W+FqJRsMfvAn34V4
+(�t�����u"e�,��$�����Q��>w��IQz��K��=������v��
diff --git a/sec/rekeyed/kaambl/dd788a3428bec6c88c5ac4e143ec0b11-metric.age b/sec/rekeyed/kaambl/dd788a3428bec6c88c5ac4e143ec0b11-metric.age
diff --git a/sec/rekeyed/nodens/06a8485aaa473a141854234f5e2ba9d1-metric.age b/sec/rekeyed/nodens/06a8485aaa473a141854234f5e2ba9d1-metric.age
diff --git a/sec/rekeyed/nodens/72e723baf1756271c0d13849beb99918-metric.age b/sec/rekeyed/nodens/72e723baf1756271c0d13849beb99918-metric.age
diff --git a/services/mosdns.nix b/services/mosdns.nix
@@ -2,6 +2,7 @@
   enable = true;
   config = {
     log = { level = "debug"; production = false; };
+    api.http = "localhost:9092";
     plugins = let src = "${inputs.nixyDomains}/assets"; in [
       {
         args = {