diff --git a/docs/monitoring-server.md b/docs/monitoring-server.md new file mode 100644 index 00000000..e18f1b54 --- /dev/null +++ b/docs/monitoring-server.md @@ -0,0 +1,141 @@ + + +# Monitoring server + +Monitoring server is running Grafana on port 80 and Prometheus on port 9090. + +It's configuration is located in `hosts/monitoring`. + +## Adding metrics endpoint + +### ubuntu host + +```sh +sudo apt-get install prometheus-node-exporter +sudo systemctl enable prometheus-node-exporter.service +sudo systemctl start prometheus-node-exporter.service +``` + +Metrics should now be available at `127.0.0.1:9100/metrics` + +```sh +curl 127.0.0.1:9100/metrics +``` + +If metrics are not accessible from another machine in ficolo network, check firewall: + +```sh +sudo ufw status +sudo ufw allow 9100/tcp +``` +### nixos host + +For a host managed by this repo, simply import `service-node-exporter` + +## Authentication + +### ubuntu host + +For http basic auth, it is easiest to use nginx. + +Create password file + +```sh +sudo apt-get install apache2-utils +sudo htpasswd -c /etc/nginx/.htpasswd scraper +``` + +And add this to the nginx config of your server: + +``` +location /metrics { + proxy_pass http://127.0.0.1:9100/metrics; + + auth_basic "Verify yourself!"; + auth_basic_user_file /etc/nginx/.htpasswd; +} +``` + +### nixos host + +- Add metrics htpasswd to secrets.yaml +- Add authenticated nginx location with htpasswd file from secrets + +```nix +locations."/metrics" = { + proxyPass = "http://127.0.0.1:${toString config.services.prometheus.exporters.node.port}/metrics"; + basicAuthFile = config.sops.secrets.metrics-htpasswd.path; +}; +``` + +### prometheus side + +For prometheus to use this username and password when scraping, an authenticated job must be added: + +```nix +sops.secrets.metrics-password.owner = "prometheus"; +... +{ + job_name = "authenticated"; + scheme = "https"; + basic_auth = { + username = "scraper"; + password_file = config.sops.secrets.metrics-password.path; + }; + static_configs = [ + { + targets = [ + "mytarget" + ]; + } + ]; +} +``` + +### Using ssh proxy + +When webserver is not desired, or ports 80 and 443 are not available, metrics can be scraped through ssh. +This is not natively supported by prometheus, but a proxy server like [sshified](https://github.com/hoffie/sshified) can be used. + +```sh +./sshified --proxy.listen-addr 127.0.0.1:8888 \ + --ssh.user sshified \ + --ssh.key-file ~/.ssh/id_ed25519 \ + --ssh.known-hosts-file ~/.ssh/known_hosts \ + --ssh.port 22 -v +``` + +This has been set up in monitoring server as a systemd service. + +Remote server has to be set up to allow ssh access for user `sshified` with the given ssh key. + +Prometheus can then be set up to scrape the remote server with `127.0.0.1:8888` as the proxy. + +The ssh proxy will redirect the request to remote server's `127.0.0.1:9100`. + +## Reading metrics + +In the monitoring server, there is prometheus instance that is scraping all targets. +Adding a metrics target is simple. + +```diff +services.prometheus.scrapeConfigs = [ + { + ... + static_configs = [ + { + targets = [ + "ganymede.vedenemo.dev" ++ "mytarget.com" + ]; + } + ]; + } +]; +``` + +By default prometheus will search for metrics in the given target at `/metrics` diff --git a/hosts/binarycache/configuration.nix b/hosts/binarycache/configuration.nix index 6f69db3e..46df0757 100644 --- a/hosts/binarycache/configuration.nix +++ b/hosts/binarycache/configuration.nix @@ -20,6 +20,7 @@ (with self.nixosModules; [ common qemu-common + ficolo-hosts service-openssh service-binary-cache service-nginx diff --git a/hosts/default.nix b/hosts/default.nix index ca5f85b4..549bd2c8 100644 --- a/hosts/default.nix +++ b/hosts/default.nix @@ -11,6 +11,7 @@ # shared modules azure-common = import ./azure-common.nix; qemu-common = import ./qemu-common.nix; + ficolo-hosts = import ./ficolo-hosts.nix; common = import ./common.nix; generic-disk-config = import ./generic-disk-config.nix; }; diff --git a/hosts/ficolo-hosts.nix b/hosts/ficolo-hosts.nix new file mode 100644 index 00000000..3cad5331 --- /dev/null +++ b/hosts/ficolo-hosts.nix @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{ + networking.extraHosts = '' + 172.18.20.102 vedenemo.dev + 172.18.20.105 ganymede.vedenemo.dev + 172.18.20.109 cache.vedenemo.dev + ''; +} diff --git a/hosts/monitoring/configuration.nix b/hosts/monitoring/configuration.nix index 915aa4a4..569af5ac 100644 --- a/hosts/monitoring/configuration.nix +++ b/hosts/monitoring/configuration.nix @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 { + pkgs, self, inputs, lib, @@ -10,15 +11,21 @@ }: let # "public" but really only available with ficolo vpn public-ip = "172.18.20.108"; + sshified = pkgs.callPackage ../../pkgs/sshified/default.nix {}; in { + sops.defaultSopsFile = ./secrets.yaml; + sops.secrets.sshified_private_key.owner = "sshified"; + imports = lib.flatten [ (with inputs; [ nix-serve-ng.nixosModules.default + sops-nix.nixosModules.sops disko.nixosModules.disko ]) (with self.nixosModules; [ common qemu-common + ficolo-hosts service-openssh service-nginx service-node-exporter @@ -38,6 +45,36 @@ in { }; }; + environment.systemPackages = [ + sshified + ]; + + users.users."sshified" = { + isNormalUser = true; + }; + + services.openssh.knownHosts = { + "[awsarm.vedenemo.dev]:20220".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIL3f7tAAO3Fc+8BqemsBQc/Yl/NmRfyhzr5SFOSKqrv0"; + }; + + systemd.services."sshified" = { + wantedBy = ["multi-user.target"]; + after = ["network.target"]; + description = "Run the sshified http-to-ssh proxy"; + serviceConfig = { + User = "sshified"; + ExecStart = '' + ${sshified}/bin/sshified \ + --proxy.listen-addr 127.0.0.1:8888 \ + --ssh.user sshified \ + --ssh.key-file ${config.sops.secrets.sshified_private_key.path} \ + --ssh.known-hosts-file /etc/ssh/ssh_known_hosts \ + --ssh.port 20220 \ + -v + ''; + }; + }; + services.grafana = { enable = true; @@ -54,6 +91,7 @@ in { }; # allow read-only access to dashboards without login + # this is fine because the page is only accessible with vpn "auth.anonymous".enabled = true; }; @@ -77,12 +115,27 @@ in { scrapeConfigs = [ { - job_name = "ficolo-node-exporter"; + job_name = "ficolo-internal-node-exporter"; + static_configs = [ + { + targets = [ + "172.18.20.102:9100" # build1 + "172.18.20.105:9100" # build4 + "172.18.20.106:9100" # old cache + "172.18.20.107:9100" # gerrit + "172.18.20.109:9100" # binarycache + ]; + } + ]; + } + { + job_name = "awsarm-ssh-node-exporter"; + scheme = "http"; + proxy_url = "http://127.0.0.1:8888"; static_configs = [ { targets = [ - "172.18.20.109:9002" # binarycache - "172.18.20.105:9999" # build4 + "awsarm.vedenemo.dev:9100" ]; } ]; diff --git a/hosts/monitoring/secrets.yaml b/hosts/monitoring/secrets.yaml index 8680c50e..cbb5e528 100644 --- a/hosts/monitoring/secrets.yaml +++ b/hosts/monitoring/secrets.yaml @@ -1,4 +1,5 @@ ssh_host_ed25519_key: ENC[AES256_GCM,data:3syc79zImKOJ85pvTNTFN1HWC4cX2aguP8OtlbxiElZWVC6nanJ6boAhjaO/sdIh9tppmrUdBID6QJsLnFABwtEwUj2O0oGYUddtZxerXnyEeP4EncSubJQDtmJxdVWMT7lnu0nseZT54CozbhQ+7EdX6kEOGwLR2Hm0nbaGNnHQLZKaxN37gKycG2csLs1Z2pI03dutFk4E7z1bD0QMnp2V6fzl1Gig4F4eip47nuWC76YDcVDLn8lrOsn5Hsbdbf63OfGqfw4HzEZrXrtf2TWkUgsek6zGJGZeAOMvEZHOFrzj9ojYpKuBtTcG6Dw2XgKIdld96ujwhiHzm8sWjBSp0XQs2lU0W7LndNypOA5PwZwVrQguh1fVTJlcCLJPqSi0DlIbdjpUaZAdAWwxsHkB2Mj1z9sh3K9CA/Ia045VbVixXZUobxERIkvqkhCmoF5WubaMwZFLBE2pF2VaRwm9pHXg9bbDh9PPoACsTgyFYBtmOJKgPk5uik7uj3+KyxeNhp09Lc34XzrATx6z,iv:M9WhCLuUDyo4w44R1AQovxNRSYaAQWtlYQK2EHkRubY=,tag:GyceOfzblQnH6WPCXlsgQg==,type:str] +sshified_private_key: ENC[AES256_GCM,data:Z2u1AruKYtHTjoXrYJNqwWQjnISGkm3+C2CMYcXF7s+h2Zw9uREqJaYUcxOUtSh4D3u5uiKrPyBObkEPQT+mHPG3W2m1kl5SezTowlW9Pno5ABK4FKtT//45oFt5UcN+eH1cnslkNlA0xRuvpCMVDGamfKt6NWmkKWH4aDJHas0KVGGbwVcttHW0LJtFvmu/jbt6W37X0yvg/XrkAEWFH1x5INox0ksI/EQhbKl3GxHGlHz2JoyuoviIPF6I+wMQ1AzQ50mqYEnnPYhaUa/phEVCH4S39/77vN/AlIT7JYchBalls5XnPJJq5zAJoiSEbLZw/Zyz2YO9R3vBEo8OzRkSz767flNhNndlIVHRCO4gux+vUb/26lGdFeDx5OsGTB4dPhP3GD9myvbD3cfBSixfkKNLuxrR6yRH3Tc4eizwxKP3nxAEPl1oEPDSf2pCbYGQ0UIyw1cABVn+kbpr5p+8tA+aj7o7R/GchZo5TnF22WfiXwiZgWZX6E15JHEEoCHoCtleb6h82MXesp7j,iv:6bUMuH0y+Jw7EN+dwnbYgXFzPtXCWLIM2uXW3ROkLEc=,tag:pRO8rtR3gNnkF5u9U1XxRg==,type:str] sops: kms: [] gcp_kms: [] @@ -23,8 +24,8 @@ sops: Sk1sQU9iUVBJRG1jRFcyTWg2ODA4UUUKCO2FYq3r6RZhtEMrLzs+hl+LJNTaH/M4 hyzYEGTzQahj0JzbRLeQPFmgV9x49N6nUpgY4fdkI81RM4Q/M827JQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2023-11-23T10:29:16Z" - mac: ENC[AES256_GCM,data:ZDofz3Sj4oGeuntdpUSqTIG2AMUmzNi3E/hYzE79wbmQNz4Fjb7nU26Ko7wR1V7p1ilYFCykL5vFiJzHTwXP2nFR6DsycADquKXGcM6sKTSJrxPcKdym/pA3OW5UoSanBBSQrDLkAldyZdjqSQpCrovwyfUkt1ikR2WT0YTcW94=,iv:+mZl+Z35Zu75BOACAlXWCcp7fHAcHOqu5QQomNIHWko=,tag:en9ihk127dmSm/jfnMIpXQ==,type:str] + lastmodified: "2023-12-12T14:02:00Z" + mac: ENC[AES256_GCM,data:cwGky75oCZ3HrXv8joXFp3gEM9c+pJtxU9STDwOjKyquQpRrqJ4ttaw35S/x9Ag6VKdiOXQeowH5Ci+8zNK3Z8FU+UbTEi8IG5tx18aOlqDfyXi3w66VYtPY0aYiN8OqaebOGcom6BEznfrZ/zU3HXELLqXuVpf25TNfn9WPBIA=,iv:7jUopaqyjeEFc86YpOhi9/KdT7kfaDsK2YluoG9TxEk=,tag:WO3ixikWrKe9isktLdNkFQ==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.7.3 diff --git a/hosts/qemu-common.nix b/hosts/qemu-common.nix index 97925537..742dcb73 100644 --- a/hosts/qemu-common.nix +++ b/hosts/qemu-common.nix @@ -17,8 +17,4 @@ efiSupport = false; }; }; - - networking.extraHosts = '' - 172.18.20.109 cache.vedenemo.dev - ''; } diff --git a/pkgs/sshified/default.nix b/pkgs/sshified/default.nix new file mode 100644 index 00000000..5cfb56f3 --- /dev/null +++ b/pkgs/sshified/default.nix @@ -0,0 +1,28 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{ + buildGoModule, + fetchFromGitHub, +}: +buildGoModule rec { + pname = "sshified"; + version = "1.1.15"; + + src = fetchFromGitHub { + owner = "hoffie"; + repo = pname; + rev = "v${version}"; + sha256 = "sha256-zbgwCWs+DNJ1ZmAl9h0PuJvLO3yMhE/t6T1aqpwYOgk="; + }; + + vendorHash = null; + + ldflags = [ + "-s" + "-w" + "-X main.Version=${version}" + ]; + + subPackages = ["."]; +} diff --git a/services/node-exporter/default.nix b/services/node-exporter/default.nix index ca9dec9c..3f7216b4 100644 --- a/services/node-exporter/default.nix +++ b/services/node-exporter/default.nix @@ -11,7 +11,7 @@ node = { enable = true; enabledCollectors = ["systemd"]; - port = 9002; + port = 9100; }; }; }