From 30ba7b43b67a37524579c6accaebec6aafb882db Mon Sep 17 00:00:00 2001 From: Joonas Rautiola Date: Wed, 15 Nov 2023 20:53:45 +0200 Subject: [PATCH] Add monitoring server configuration Signed-off-by: Joonas Rautiola --- .sops.yaml | 6 ++ hosts/binarycache/default.nix | 8 +- hosts/default.nix | 5 ++ hosts/monitoring/default.nix | 104 ++++++++++++++++++++++++++ hosts/monitoring/disk-config.nix | 31 ++++++++ hosts/monitoring/secrets.yaml | 30 ++++++++ hosts/monitoring/secrets.yaml.license | 3 + hosts/qemu-common.nix | 23 +++++- services/default.nix | 1 + services/node-exporter/default.nix | 17 +++++ tasks.py | 3 + 11 files changed, 220 insertions(+), 11 deletions(-) create mode 100644 hosts/monitoring/default.nix create mode 100644 hosts/monitoring/disk-config.nix create mode 100644 hosts/monitoring/secrets.yaml create mode 100644 hosts/monitoring/secrets.yaml.license create mode 100644 services/node-exporter/default.nix diff --git a/.sops.yaml b/.sops.yaml index 0779a9ed..193ee81d 100644 --- a/.sops.yaml +++ b/.sops.yaml @@ -10,6 +10,7 @@ keys: - &karim age122lvqyrdqz30fkfututykl0yle9u63u2em6e4aut7e5draws83ns3npt3a - &jrautiola age15jq5gjjd7ypsdlqfjtqy4red57v8ggqq9na6u3xffznu678nydpsuuwjg0 - &binarycache age1s47a3y44j695gemcl0kqgjlxxvaa50de9s69jy2l6vc8xtmk5pcskhpknl + - &monitoring age17s9sc2cgt9t30cyl65zya8p4zmwnndrx2r896e7gzgl08sjn0qmq3t6shs creation_rules: - path_regex: terraform/secrets.yaml$ key_groups: @@ -34,3 +35,8 @@ creation_rules: - age: - *jrautiola - *binarycache + - path_regex: hosts/monitoring/secrets.yaml$ + key_groups: + - age: + - *jrautiola + - *monitoring diff --git a/hosts/binarycache/default.nix b/hosts/binarycache/default.nix index 723940af..9f58b4b8 100644 --- a/hosts/binarycache/default.nix +++ b/hosts/binarycache/default.nix @@ -23,6 +23,7 @@ service-openssh service-binary-cache service-nginx + service-node-exporter user-jrautiola user-cazfi user-hydra @@ -37,13 +38,6 @@ }; nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; - services.openssh.enable = true; - - boot.loader.grub = { - enable = true; - # qemu vms are using SeaBIOS which is not UEFI - efiSupport = false; - }; networking = { hostName = "binarycache"; diff --git a/hosts/default.nix b/hosts/default.nix index 80cf0fd0..f15a2303 100644 --- a/hosts/default.nix +++ b/hosts/default.nix @@ -17,6 +17,7 @@ host-build01 = import ./build01; host-ghafhydra = import ./ghafhydra; host-binarycache = import ./binarycache; + host-monitoring = import ./monitoring; }; flake.nixosConfigurations = let @@ -36,5 +37,9 @@ inherit specialArgs; modules = [self.nixosModules.host-binarycache]; }; + monitoring = lib.nixosSystem { + inherit specialArgs; + modules = [self.nixosModules.host-monitoring]; + }; }; } diff --git a/hosts/monitoring/default.nix b/hosts/monitoring/default.nix new file mode 100644 index 00000000..915aa4a4 --- /dev/null +++ b/hosts/monitoring/default.nix @@ -0,0 +1,104 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{ + self, + inputs, + lib, + config, + ... +}: let + # "public" but really only available with ficolo vpn + public-ip = "172.18.20.108"; +in { + imports = lib.flatten [ + (with inputs; [ + nix-serve-ng.nixosModules.default + disko.nixosModules.disko + ]) + (with self.nixosModules; [ + common + qemu-common + service-openssh + service-nginx + service-node-exporter + user-jrautiola + ]) + ./disk-config.nix + ]; + + nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux"; + + networking = { + hostName = "monitoring"; + nameservers = ["1.1.1.1" "8.8.8.8"]; + firewall = { + allowedTCPPorts = [config.services.prometheus.port config.services.grafana.settings.server.http_port]; + allowedUDPPorts = [config.services.prometheus.port config.services.grafana.settings.server.http_port]; + }; + }; + + services.grafana = { + enable = true; + + settings = { + server = { + http_port = 3000; + http_addr = "127.0.0.1"; + }; + + # disable telemetry + analytics = { + reporting_enabled = false; + feedback_links_enabled = false; + }; + + # allow read-only access to dashboards without login + "auth.anonymous".enabled = true; + }; + + provision.datasources.settings.datasources = [ + { + name = "prometheus"; + type = "prometheus"; + isDefault = true; + url = "http://${config.services.prometheus.listenAddress}:${toString config.services.prometheus.port}"; + } + ]; + }; + + services.prometheus = { + enable = true; + + port = 9090; + listenAddress = "0.0.0.0"; + webExternalUrl = "http://${public-ip}:${toString config.services.prometheus.port}"; + checkConfig = true; + + scrapeConfigs = [ + { + job_name = "ficolo-node-exporter"; + static_configs = [ + { + targets = [ + "172.18.20.109:9002" # binarycache + "172.18.20.105:9999" # build4 + ]; + } + ]; + } + ]; + }; + + services.nginx = { + virtualHosts = { + "_" = { + default = true; + locations."/" = { + proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}"; + proxyWebsockets = true; + }; + }; + }; + }; +} diff --git a/hosts/monitoring/disk-config.nix b/hosts/monitoring/disk-config.nix new file mode 100644 index 00000000..8ddb3878 --- /dev/null +++ b/hosts/monitoring/disk-config.nix @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +# BIOS compatible gpt partition +{ + disko.devices = { + disk = { + vda = { + device = "/dev/vda"; + type = "disk"; + content = { + type = "gpt"; + partitions = { + boot = { + size = "1M"; + type = "EF02"; + }; + root = { + size = "100%"; + content = { + type = "filesystem"; + format = "ext4"; + mountpoint = "/"; + }; + }; + }; + }; + }; + }; + }; +} diff --git a/hosts/monitoring/secrets.yaml b/hosts/monitoring/secrets.yaml new file mode 100644 index 00000000..8680c50e --- /dev/null +++ b/hosts/monitoring/secrets.yaml @@ -0,0 +1,30 @@ +ssh_host_ed25519_key: ENC[AES256_GCM,data:3syc79zImKOJ85pvTNTFN1HWC4cX2aguP8OtlbxiElZWVC6nanJ6boAhjaO/sdIh9tppmrUdBID6QJsLnFABwtEwUj2O0oGYUddtZxerXnyEeP4EncSubJQDtmJxdVWMT7lnu0nseZT54CozbhQ+7EdX6kEOGwLR2Hm0nbaGNnHQLZKaxN37gKycG2csLs1Z2pI03dutFk4E7z1bD0QMnp2V6fzl1Gig4F4eip47nuWC76YDcVDLn8lrOsn5Hsbdbf63OfGqfw4HzEZrXrtf2TWkUgsek6zGJGZeAOMvEZHOFrzj9ojYpKuBtTcG6Dw2XgKIdld96ujwhiHzm8sWjBSp0XQs2lU0W7LndNypOA5PwZwVrQguh1fVTJlcCLJPqSi0DlIbdjpUaZAdAWwxsHkB2Mj1z9sh3K9CA/Ia045VbVixXZUobxERIkvqkhCmoF5WubaMwZFLBE2pF2VaRwm9pHXg9bbDh9PPoACsTgyFYBtmOJKgPk5uik7uj3+KyxeNhp09Lc34XzrATx6z,iv:M9WhCLuUDyo4w44R1AQovxNRSYaAQWtlYQK2EHkRubY=,tag:GyceOfzblQnH6WPCXlsgQg==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age15jq5gjjd7ypsdlqfjtqy4red57v8ggqq9na6u3xffznu678nydpsuuwjg0 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSByUzBsYVg0SlgxK1ZjWEZz + Q092VHdvYnExN09TK3p0U3k2NmVKK0FHdVJrCk1jYlBKYjh6NU1vVHRzbmFmSllq + bzBDdzhZRWV6MDFBRGRXTHNpMk9HQ28KLS0tIEtNZUVMWndKOFVnZkhXWGVwb1dS + dzJEN1Y5d01vanRtV2J1aFhRbnM1bjAK5N+aLBhNtHRsz2l+rjTU+Szj/bHDgq1m + xc6uZKNIqwcXP+t308mKDDJdFxZRp/AkJqgV1IYFRs8U9CLZi11ccw== + -----END AGE ENCRYPTED FILE----- + - recipient: age17s9sc2cgt9t30cyl65zya8p4zmwnndrx2r896e7gzgl08sjn0qmq3t6shs + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1ejRkQVFxYzBiYVEzMXlz + T2VqTk5YSVRtcmJ6QnFLanZlS1FqeFhZeDNNCnNEK25tTzNNT0tQMDBFSTcxVjN3 + cHpBTXpINEEvVUJWMExLVm5vTUVWUkkKLS0tIHorZUVYZ2RTMmVJdWRXa3h5MmdJ + Sk1sQU9iUVBJRG1jRFcyTWg2ODA4UUUKCO2FYq3r6RZhtEMrLzs+hl+LJNTaH/M4 + hyzYEGTzQahj0JzbRLeQPFmgV9x49N6nUpgY4fdkI81RM4Q/M827JQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2023-11-23T10:29:16Z" + mac: ENC[AES256_GCM,data:ZDofz3Sj4oGeuntdpUSqTIG2AMUmzNi3E/hYzE79wbmQNz4Fjb7nU26Ko7wR1V7p1ilYFCykL5vFiJzHTwXP2nFR6DsycADquKXGcM6sKTSJrxPcKdym/pA3OW5UoSanBBSQrDLkAldyZdjqSQpCrovwyfUkt1ikR2WT0YTcW94=,iv:+mZl+Z35Zu75BOACAlXWCcp7fHAcHOqu5QQomNIHWko=,tag:en9ihk127dmSm/jfnMIpXQ==,type:str] + pgp: [] + unencrypted_suffix: _unencrypted + version: 3.7.3 diff --git a/hosts/monitoring/secrets.yaml.license b/hosts/monitoring/secrets.yaml.license new file mode 100644 index 00000000..856ce263 --- /dev/null +++ b/hosts/monitoring/secrets.yaml.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) + +SPDX-License-Identifier: Apache-2.0 diff --git a/hosts/qemu-common.nix b/hosts/qemu-common.nix index a9ea5f87..97925537 100644 --- a/hosts/qemu-common.nix +++ b/hosts/qemu-common.nix @@ -1,9 +1,24 @@ # SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) # # SPDX-License-Identifier: Apache-2.0 -_: { +{ services.qemuGuest.enable = true; - boot.kernelParams = ["console=ttyS0" "earlyprintk=ttyS0" "rootdelay=300" "panic=1" "boot.panic_on_fail"]; - boot.initrd.availableKernelModules = ["ahci" "xhci_pci" "virtio_pci" "sr_mod" "virtio_blk" "uhci_hcd" "ehci_pci" "virtio_scsi"]; - boot.initrd.kernelModules = ["kvm-intel" "dm-snapshot"]; + + boot = { + kernelParams = ["console=ttyS0" "earlyprintk=ttyS0" "rootdelay=300" "panic=1" "boot.panic_on_fail"]; + initrd = { + availableKernelModules = ["ahci" "xhci_pci" "virtio_pci" "sr_mod" "virtio_blk" "uhci_hcd" "ehci_pci" "virtio_scsi"]; + kernelModules = ["kvm-intel" "dm-snapshot"]; + }; + + loader.grub = { + enable = true; + # qemu vms are using SeaBIOS which is not UEFI + efiSupport = false; + }; + }; + + networking.extraHosts = '' + 172.18.20.109 cache.vedenemo.dev + ''; } diff --git a/services/default.nix b/services/default.nix index 519d5f90..d5003410 100644 --- a/services/default.nix +++ b/services/default.nix @@ -7,5 +7,6 @@ service-hydra = import ./hydra; service-nginx = import ./nginx; service-openssh = import ./openssh; + service-node-exporter = import ./node-exporter; }; } diff --git a/services/node-exporter/default.nix b/services/node-exporter/default.nix new file mode 100644 index 00000000..ca9dec9c --- /dev/null +++ b/services/node-exporter/default.nix @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2023 Technology Innovation Institute (TII) +# +# SPDX-License-Identifier: Apache-2.0 +{config, ...}: { + networking.firewall = { + allowedTCPPorts = [config.services.prometheus.exporters.node.port]; + allowedUDPPorts = [config.services.prometheus.exporters.node.port]; + }; + + services.prometheus.exporters = { + node = { + enable = true; + enabledCollectors = ["systemd"]; + port = 9002; + }; + }; +} diff --git a/tasks.py b/tasks.py index d782e2e2..e0532f66 100644 --- a/tasks.py +++ b/tasks.py @@ -79,6 +79,9 @@ class TargetHost: "binarycache-ficolo": TargetHost( hostname="172.18.20.109", nixosconfig="binarycache" ), + "monitoring-ficolo": TargetHost( + hostname="172.18.20.108", nixosconfig="monitoring" + ), } )