From 6cabba54e0d93f23c1497f00e6f3296ffdc71e6d Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Mon, 3 Apr 2023 15:28:44 +0100 Subject: [PATCH 1/7] feat: Use new ocp20 IPs --- docs/develop/update/firewall.rst | 4 ++-- pillar/prometheus_client.sls | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/develop/update/firewall.rst b/docs/develop/update/firewall.rst index 81d9d73c..909c9a0b 100644 --- a/docs/develop/update/firewall.rst +++ b/docs/develop/update/firewall.rst @@ -117,7 +117,7 @@ You can configure a Hetzner firewall as follows: - - Accept * - Allow Prometheus - - 213.138.113.219/32 + - 139.162.253.17/32 - 0.0.0.0/0 - 0-65535 - 7231 @@ -176,7 +176,7 @@ You can configure a Linode Cloud Firewall as follows: * - Allow-Prometheus - TCP - 7231 - - 213.138.113.219/32, 2001:41c8:51:7db::219/128 + - 139.162.253.17/32, 2a01:7e00::f03c:93ff:fe13:a12c/128 - Accept Most servers will also have: diff --git a/pillar/prometheus_client.sls b/pillar/prometheus_client.sls index 808a08fa..1007622c 100644 --- a/pillar/prometheus_client.sls +++ b/pillar/prometheus_client.sls @@ -1,6 +1,6 @@ firewall: - prometheus_ipv4: 213.138.113.219 - prometheus_ipv6: 2001:41c8:51:7db::219 + prometheus_ipv4: 139.162.253.17 + prometheus_ipv6: 2a01:7e00::f03c:93ff:fe13:a12c prometheus: node_exporter: From 70787adbadd243fd581aae08c5e7bdf86fad346c Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Mon, 3 Apr 2023 15:29:04 +0100 Subject: [PATCH 2/7] docs: fix broken Linode link --- docs/develop/update/network.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/develop/update/network.rst b/docs/develop/update/network.rst index 66469236..37d5580e 100644 --- a/docs/develop/update/network.rst +++ b/docs/develop/update/network.rst @@ -21,7 +21,7 @@ Linux networking systemd-networkd ~~~~~~~~~~~~~~~~ -`systemd-networkd `__ is a system daemon to configure networking, and is our preferred solution for Linode instances. Configurations are available for `Linode`_ and other hosts. The configuration is written to ``/etc/systemd/network/05-eth0.network``. +`systemd-networkd `__ is a system daemon to configure networking, and is our preferred solution for Linode instances. Configurations are available for Linode and other hosts. The configuration is written to ``/etc/systemd/network/05-eth0.network``. Linode template ^^^^^^^^^^^^^^^ From 76aff60e33954228a135a44a40711c80b96dec1a Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Mon, 3 Apr 2023 15:29:39 +0100 Subject: [PATCH 3/7] feat: update prometheus to ubuntu 22 --- pillar/prometheus_server.sls | 12 ++++++++++-- salt-config/roster | 2 +- salt/prometheus/files/conf-prometheus.yml | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/pillar/prometheus_server.sls b/pillar/prometheus_server.sls index 7768b869..371b9792 100644 --- a/pillar/prometheus_server.sls +++ b/pillar/prometheus_server.sls @@ -1,9 +1,17 @@ +network: + host_id: ocp20 + ipv4: 139.162.253.17 + #ipv6: 2a01:7e00::f03c:93ff:fe13:a12c + networkd: + template: linode + gateway4: 139.162.253.1 + prometheus: prometheus: service: prometheus-server user: prometheus-server basename: prometheus - version: 2.36.2 + version: 2.37.6 local_storage_retention: 120d config: conf-prometheus.yml: salt://prometheus/files/conf-prometheus.yml @@ -13,7 +21,7 @@ prometheus: service: prometheus-alertmanager user: prometheus-alertmanager basename: alertmanager - version: 0.24.0 + version: 0.25.0 config: conf-alertmanager.yml: salt://prometheus/files/conf-alertmanager.yml diff --git a/salt-config/roster b/salt-config/roster index 741e45da..968227d5 100644 --- a/salt-config/roster +++ b/salt-config/roster @@ -5,7 +5,7 @@ cove-ocds: ocp18.open-contracting.org docs: ocp07.open-contracting.org kingfisher-process: ocp04.open-contracting.org kingfisher-replica: ocp05.open-contracting.org -prometheus: ocp03.open-contracting.org +prometheus: ocp20.open-contracting.org redash: ocp14.open-contracting.org redmine: ocp16.open-contracting.org registry: diff --git a/salt/prometheus/files/conf-prometheus.yml b/salt/prometheus/files/conf-prometheus.yml index a6c21c82..c1837d46 100644 --- a/salt/prometheus/files/conf-prometheus.yml +++ b/salt/prometheus/files/conf-prometheus.yml @@ -16,7 +16,7 @@ scrape_configs: 'ocds-kingfisher-replica': 'ocp05.open-contracting.org', 'ocds-kingfisher2': 'ocp04.open-contracting.org', 'ocds-live.docs': 'ocp07.open-contracting.org', - 'prometheus-server-node': 'ocp03.open-contracting.org', + 'prometheus-server-node': 'ocp20.open-contracting.org', 'data-registry': 'ocp13.open-contracting.org', 'redash': 'ocp14.open-contracting.org', 'redmine': 'ocp16.open-contracting.org', From 417c722431049a9b962feb9a9a18edbd352bba9f Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Mon, 3 Apr 2023 15:29:57 +0100 Subject: [PATCH 4/7] fix: set required alertmanager flag --- salt/core/systemd/files/prometheus-alertmanager.service | 1 + 1 file changed, 1 insertion(+) diff --git a/salt/core/systemd/files/prometheus-alertmanager.service b/salt/core/systemd/files/prometheus-alertmanager.service index 73f4ba5a..51d67d2c 100644 --- a/salt/core/systemd/files/prometheus-alertmanager.service +++ b/salt/core/systemd/files/prometheus-alertmanager.service @@ -8,6 +8,7 @@ Group={{ user }} ExecReload=/bin/kill -HUP $MAINPID ExecStart=/home/{{ user }}/{{ entry.basename }}-{{ entry.version }}.{{ grains.kernel|lower }}-{{ grains.osarch }}/{{ entry.basename }} \ --web.listen-address 127.0.0.1:9095 \ + --cluster.listen-address ""\ --config.file /home/{{ user }}/conf-alertmanager.yml \ {%- if salt['pillar.get']('apache:sites:prometheus-alertmanager:servername') %} --web.external-url https://{{ pillar.apache.sites['prometheus-alertmanager'].servername }}/ \ From cdbd147d6b88b008b4f5192ba9f3dabd4d874e58 Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Tue, 4 Apr 2023 09:17:36 +0100 Subject: [PATCH 5/7] refactor: remove maintenance.sls --- pillar/maintenance.sls | 3 --- pillar/top.sls | 2 -- 2 files changed, 5 deletions(-) delete mode 100644 pillar/maintenance.sls diff --git a/pillar/maintenance.sls b/pillar/maintenance.sls deleted file mode 100644 index 03e0fc7b..00000000 --- a/pillar/maintenance.sls +++ /dev/null @@ -1,3 +0,0 @@ -maintenance: - enabled: True - patching: manual diff --git a/pillar/top.sls b/pillar/top.sls index b2aa46ee..089a6ad6 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -10,14 +10,12 @@ base: - cove_oc4ids - cove_oc4ids_maintenance - private.cove_oc4ids - - maintenance 'cove-ocds': - cove - cove_ocds - cove_ocds_maintenance - private.cove_ocds - - maintenance 'docs': - docs From 17dd0a4ce6bddb1b11f0724a2ab10594f379adbf Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Tue, 4 Apr 2023 09:18:09 +0100 Subject: [PATCH 6/7] feat: add prometheus maintenance config --- pillar/prometheus_server_maintenance.sls | 7 +++++++ pillar/top.sls | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 pillar/prometheus_server_maintenance.sls diff --git a/pillar/prometheus_server_maintenance.sls b/pillar/prometheus_server_maintenance.sls new file mode 100644 index 00000000..60e66eee --- /dev/null +++ b/pillar/prometheus_server_maintenance.sls @@ -0,0 +1,7 @@ +maintenance: + enabled: True + patching: manual + rkhunter_customisation: | + ALLOW_SSH_ROOT_USER=yes + RTKT_FILE_WHITELIST=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9 + USER_FILEPROP_FILES_DIRS=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9 diff --git a/pillar/top.sls b/pillar/top.sls index 089a6ad6..96270747 100644 --- a/pillar/top.sls +++ b/pillar/top.sls @@ -41,7 +41,7 @@ base: - prometheus_server - private.smtp - private.prometheus_server - - maintenance + - prometheus_server_maintenance 'redash': - redash From e8ea729a843e5ab82cddc86a05e2faf2eb2a6462 Mon Sep 17 00:00:00 2001 From: Rob Hooper Date: Thu, 6 Apr 2023 09:38:42 +0100 Subject: [PATCH 7/7] docs: add prometheus migration steps --- docs/deploy/create_server.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/deploy/create_server.rst b/docs/deploy/create_server.rst index 4be9ec75..1e6b457e 100644 --- a/docs/deploy/create_server.rst +++ b/docs/deploy/create_server.rst @@ -300,6 +300,11 @@ For Kingfisher servers (instructions are incomplete): For Redash servers, see :doc:`redash`. +For Prometheus: + +#. Copy the ``/home/prometheus-server/data`` directory +#. Update the IP addresses in the ``pillar/prometheus_client.sls`` file, and deploy to all services + 6. Update external services ---------------------------