Monitor awsarm through SSH

docs/monitoring-server.md

@@ -0,0 +1,141 @@
+<!--
+SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# Monitoring server
+
+Monitoring server is running Grafana on port 80 and Prometheus on port 9090.
+
+It's configuration is located in `hosts/monitoring`.
+
+## Adding metrics endpoint
+
+### ubuntu host
+
+```sh
+sudo apt-get install prometheus-node-exporter
+sudo systemctl enable prometheus-node-exporter.service
+sudo systemctl start prometheus-node-exporter.service
+```
+
+Metrics should now be available at `127.0.0.1:9100/metrics`
+
+```sh
+curl 127.0.0.1:9100/metrics
+```
+
+If metrics are not accessible from another machine in ficolo network, check firewall:
+
+```sh
+sudo ufw status
+sudo ufw allow 9100/tcp
+```
+### nixos host
+
+For a host managed by this repo, simply import `service-node-exporter`
+
+## Authentication
+
+### ubuntu host
+
+For http basic auth, it is easiest to use nginx.
+
+Create password file
+
+```sh
+sudo apt-get install apache2-utils
+sudo htpasswd -c /etc/nginx/.htpasswd scraper
+```
+
+And add this to the nginx config of your server:
+
+```
+location /metrics {
+    proxy_pass http://127.0.0.1:9100/metrics;
+
+    auth_basic           "Verify yourself!";
+    auth_basic_user_file /etc/nginx/.htpasswd;
+}
+```
+
+### nixos host
+
+- Add metrics htpasswd to secrets.yaml
+- Add authenticated nginx location with htpasswd file from secrets
+
+```nix
+locations."/metrics" = {
+  proxyPass = "http://127.0.0.1:${toString config.services.prometheus.exporters.node.port}/metrics";
+  basicAuthFile = config.sops.secrets.metrics-htpasswd.path;
+};
+```
+
+### prometheus side
+
+For prometheus to use this username and password when scraping, an authenticated job must be added:
+
+```nix
+sops.secrets.metrics-password.owner = "prometheus";
+...
+{
+  job_name = "authenticated";
+  scheme = "https";
+  basic_auth = {
+    username = "scraper";
+    password_file = config.sops.secrets.metrics-password.path;
+  };
+  static_configs = [
+    {
+      targets = [
+        "mytarget"
+      ];
+    }
+  ];
+}
+```
+
+### Using ssh proxy
+
+When webserver is not desired, or ports 80 and 443 are not available, metrics can be scraped through ssh.
+This is not natively supported by prometheus, but a proxy server like [sshified](https://github.com/hoffie/sshified) can be used.
+
+```sh
+./sshified --proxy.listen-addr 127.0.0.1:8888 \
+  --ssh.user sshified \
+  --ssh.key-file ~/.ssh/id_ed25519 \
+  --ssh.known-hosts-file ~/.ssh/known_hosts \
+  --ssh.port 22 -v
+```
+
+This has been set up in monitoring server as a systemd service.
+
+Remote server has to be set up to allow ssh access for user `sshified` with the given ssh key.
+
+Prometheus can then be set up to scrape the remote server with `127.0.0.1:8888` as the proxy.
+
+The ssh proxy will redirect the request to remote server's `127.0.0.1:9100`.
+
+## Reading metrics
+
+In the monitoring server, there is prometheus instance that is scraping all targets.
+Adding a metrics target is simple.
+
+```diff
+services.prometheus.scrapeConfigs = [
+  {
+    ...
+    static_configs = [
+      {
+        targets = [
+          "ganymede.vedenemo.dev"
++         "mytarget.com"
+        ];
+      }
+    ];
+  }
+];
+```
+
+By default prometheus will search for metrics in the given target at `/metrics`

hosts/binarycache/configuration.nix

@@ -20,6 +20,7 @@
     (with self.nixosModules; [
       common
       qemu-common
+      ficolo-hosts
       service-openssh
       service-binary-cache
       service-nginx

hosts/default.nix

@@ -11,6 +11,7 @@
     # shared modules
     azure-common = import ./azure-common.nix;
     qemu-common = import ./qemu-common.nix;
+    ficolo-hosts = import ./ficolo-hosts.nix;
     common = import ./common.nix;
     generic-disk-config = import ./generic-disk-config.nix;
   };

hosts/ficolo-hosts.nix

@@ -0,0 +1,10 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  networking.extraHosts = ''
+    172.18.20.102 vedenemo.dev
+    172.18.20.105 ganymede.vedenemo.dev
+    172.18.20.109 cache.vedenemo.dev
+  '';
+}

hosts/monitoring/configuration.nix

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 {
+  pkgs,
   self,
   inputs,
   lib,
@@ -10,15 +11,21 @@
 }: let
   # "public" but really only available with ficolo vpn
   public-ip = "172.18.20.108";
+  sshified = pkgs.callPackage ../../pkgs/sshified/default.nix {};
 in {
+  sops.defaultSopsFile = ./secrets.yaml;
+  sops.secrets.sshified_private_key.owner = "sshified";
+
   imports = lib.flatten [
     (with inputs; [
       nix-serve-ng.nixosModules.default
+      sops-nix.nixosModules.sops
       disko.nixosModules.disko
     ])
     (with self.nixosModules; [
       common
       qemu-common
+      ficolo-hosts
       service-openssh
       service-nginx
       service-node-exporter
@@ -38,6 +45,36 @@ in {
     };
   };
 
+  environment.systemPackages = [
+    sshified
+  ];
+
+  users.users."sshified" = {
+    isNormalUser = true;
+  };
+
+  services.openssh.knownHosts = {
+    "[awsarm.vedenemo.dev]:20220".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3f7tAAO3Fc+8BqemsBQc/Yl/NmRfyhzr5SFOSKqrv0";
+  };
+
+  systemd.services."sshified" = {
+    wantedBy = ["multi-user.target"];
+    after = ["network.target"];
+    description = "Run the sshified http-to-ssh proxy";
+    serviceConfig = {
+      User = "sshified";
+      ExecStart = ''
+        ${sshified}/bin/sshified \
+        --proxy.listen-addr 127.0.0.1:8888 \
+        --ssh.user sshified \
+        --ssh.key-file ${config.sops.secrets.sshified_private_key.path} \
+        --ssh.known-hosts-file /etc/ssh/ssh_known_hosts \
+        --ssh.port 20220 \
+        -v
+      '';
+    };
+  };
+
   services.grafana = {
     enable = true;
 
@@ -54,6 +91,7 @@ in {
       };
 
       # allow read-only access to dashboards without login
+      # this is fine because the page is only accessible with vpn
       "auth.anonymous".enabled = true;
     };
 
@@ -77,12 +115,27 @@ in {
 
     scrapeConfigs = [
       {
-        job_name = "ficolo-node-exporter";
+        job_name = "ficolo-internal-node-exporter";
+        static_configs = [
+          {
+            targets = [
+              "172.18.20.102:9100" # build1
+              "172.18.20.105:9100" # build4
+              "172.18.20.106:9100" # old cache
+              "172.18.20.107:9100" # gerrit
+              "172.18.20.109:9100" # binarycache
+            ];
+          }
+        ];
+      }
+      {
+        job_name = "awsarm-ssh-node-exporter";
+        scheme = "http";
+        proxy_url = "http://127.0.0.1:8888";
         static_configs = [
           {
             targets = [
-              "172.18.20.109:9002" # binarycache
-              "172.18.20.105:9999" # build4
+              "awsarm.vedenemo.dev:9100"
             ];
           }
         ];

hosts/monitoring/secrets.yaml

@@ -1,4 +1,5 @@
 ssh_host_ed25519_key: ENC[AES256_GCM,data:3syc79zImKOJ85pvTNTFN1HWC4cX2aguP8OtlbxiElZWVC6nanJ6boAhjaO/sdIh9tppmrUdBID6QJsLnFABwtEwUj2O0oGYUddtZxerXnyEeP4EncSubJQDtmJxdVWMT7lnu0nseZT54CozbhQ+7EdX6kEOGwLR2Hm0nbaGNnHQLZKaxN37gKycG2csLs1Z2pI03dutFk4E7z1bD0QMnp2V6fzl1Gig4F4eip47nuWC76YDcVDLn8lrOsn5Hsbdbf63OfGqfw4HzEZrXrtf2TWkUgsek6zGJGZeAOMvEZHOFrzj9ojYpKuBtTcG6Dw2XgKIdld96ujwhiHzm8sWjBSp0XQs2lU0W7LndNypOA5PwZwVrQguh1fVTJlcCLJPqSi0DlIbdjpUaZAdAWwxsHkB2Mj1z9sh3K9CA/Ia045VbVixXZUobxERIkvqkhCmoF5WubaMwZFLBE2pF2VaRwm9pHXg9bbDh9PPoACsTgyFYBtmOJKgPk5uik7uj3+KyxeNhp09Lc34XzrATx6z,iv:M9WhCLuUDyo4w44R1AQovxNRSYaAQWtlYQK2EHkRubY=,tag:GyceOfzblQnH6WPCXlsgQg==,type:str]
+sshified_private_key: ENC[AES256_GCM,data:Z2u1AruKYtHTjoXrYJNqwWQjnISGkm3+C2CMYcXF7s+h2Zw9uREqJaYUcxOUtSh4D3u5uiKrPyBObkEPQT+mHPG3W2m1kl5SezTowlW9Pno5ABK4FKtT//45oFt5UcN+eH1cnslkNlA0xRuvpCMVDGamfKt6NWmkKWH4aDJHas0KVGGbwVcttHW0LJtFvmu/jbt6W37X0yvg/XrkAEWFH1x5INox0ksI/EQhbKl3GxHGlHz2JoyuoviIPF6I+wMQ1AzQ50mqYEnnPYhaUa/phEVCH4S39/77vN/AlIT7JYchBalls5XnPJJq5zAJoiSEbLZw/Zyz2YO9R3vBEo8OzRkSz767flNhNndlIVHRCO4gux+vUb/26lGdFeDx5OsGTB4dPhP3GD9myvbD3cfBSixfkKNLuxrR6yRH3Tc4eizwxKP3nxAEPl1oEPDSf2pCbYGQ0UIyw1cABVn+kbpr5p+8tA+aj7o7R/GchZo5TnF22WfiXwiZgWZX6E15JHEEoCHoCtleb6h82MXesp7j,iv:6bUMuH0y+Jw7EN+dwnbYgXFzPtXCWLIM2uXW3ROkLEc=,tag:pRO8rtR3gNnkF5u9U1XxRg==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -23,8 +24,8 @@ sops:
             Sk1sQU9iUVBJRG1jRFcyTWg2ODA4UUUKCO2FYq3r6RZhtEMrLzs+hl+LJNTaH/M4
             hyzYEGTzQahj0JzbRLeQPFmgV9x49N6nUpgY4fdkI81RM4Q/M827JQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2023-11-23T10:29:16Z"
-    mac: ENC[AES256_GCM,data:ZDofz3Sj4oGeuntdpUSqTIG2AMUmzNi3E/hYzE79wbmQNz4Fjb7nU26Ko7wR1V7p1ilYFCykL5vFiJzHTwXP2nFR6DsycADquKXGcM6sKTSJrxPcKdym/pA3OW5UoSanBBSQrDLkAldyZdjqSQpCrovwyfUkt1ikR2WT0YTcW94=,iv:+mZl+Z35Zu75BOACAlXWCcp7fHAcHOqu5QQomNIHWko=,tag:en9ihk127dmSm/jfnMIpXQ==,type:str]
+    lastmodified: "2023-12-12T14:02:00Z"
+    mac: ENC[AES256_GCM,data:cwGky75oCZ3HrXv8joXFp3gEM9c+pJtxU9STDwOjKyquQpRrqJ4ttaw35S/x9Ag6VKdiOXQeowH5Ci+8zNK3Z8FU+UbTEi8IG5tx18aOlqDfyXi3w66VYtPY0aYiN8OqaebOGcom6BEznfrZ/zU3HXELLqXuVpf25TNfn9WPBIA=,iv:7jUopaqyjeEFc86YpOhi9/KdT7kfaDsK2YluoG9TxEk=,tag:WO3ixikWrKe9isktLdNkFQ==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.7.3

hosts/qemu-common.nix

@@ -17,8 +17,4 @@
       efiSupport = false;
     };
   };
-
-  networking.extraHosts = ''
-    172.18.20.109 cache.vedenemo.dev
-  '';
 }

pkgs/sshified/default.nix

@@ -0,0 +1,28 @@
+# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII)
+#
+# SPDX-License-Identifier: Apache-2.0
+{
+  buildGoModule,
+  fetchFromGitHub,
+}:
+buildGoModule rec {
+  pname = "sshified";
+  version = "1.1.15";
+
+  src = fetchFromGitHub {
+    owner = "hoffie";
+    repo = pname;
+    rev = "v${version}";
+    sha256 = "sha256-zbgwCWs+DNJ1ZmAl9h0PuJvLO3yMhE/t6T1aqpwYOgk=";
+  };
+
+  vendorHash = null;
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X main.Version=${version}"
+  ];
+
+  subPackages = ["."];
+}

services/node-exporter/default.nix

@@ -11,7 +11,7 @@
     node = {
       enable = true;
       enabledCollectors = ["systemd"];
-      port = 9002;
+      port = 9100;
     };
   };
 }