feat: add ansible config to make backend-hel as the primary test server

ansible/deploy-clickhouse.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy oonidata clickhouse hosts
+  hosts:
+    - notebook.ooni.org
+    - data1.htz-fsn.prod.ooni.nu
+    #- data2.htz-fsn.prod.ooni.nu
+    - data3.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - clickhouse
+  roles:
+    - prometheus_node_exporter
+    - oonidata_clickhouse

ansible/deploy-monitoring.yml

@@ -0,0 +1,12 @@
+---
+- name: Update monitoring config
+  hosts: monitoring.ooni.org
+  become: true
+  tags:
+    - monitoring
+  roles:
+    - prometheus
+    - prometheus_blackbox_exporter
+    - prometheus_alertmanager
+
+

ansible/deploy-ooni-backend.yml

@@ -0,0 +1,20 @@
+---
+- hosts: backend-hel.ooni.org
+  roles:
+    - role: bootstrap
+    - role: nftables
+    - role: nginx
+      tags: nginx
+      vars:
+        nginx_user: "www-data"
+    - role: dehydrated
+      tags: dehydrated
+      expand: yes
+      vars: 
+        ssl_domains:
+          # with dehydrated the first entry is the cert FQDN
+          # and the other ones are alternative names
+          - "backend-hel.ooni.org"
+    - role: ooni-backend
+      vars: 
+        ssl_domain: backend-hel.ooni.org

ansible/deploy-tier0.yml

@@ -0,0 +1,22 @@
+---
+- name: Include monitoring playbook
+  ansible.builtin.import_playbook: deploy-monitoring.yml
+
+- name: Include ooni-backend playbook
+  ansible.builtin.import_playbook: deploy-ooni-backend.yml
+
+- name: Include clickhouse playbook
+  ansible.builtin.import_playbook: deploy-clickhouse.yml
+
+- name: Deploy oonidata worker nodes
+  hosts:
+    - data1.htz-fsn.prod.ooni.nu
+  become: true
+  tags:
+    - oonidata_worker
+  roles:
+    - oonidata
+  vars:
+    enable_jupyterhub: false
+    enable_oonipipeline_worker: true
+    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"

ansible/deploy-tier2.yml

@@ -0,0 +1,25 @@
+---
+- name: Setup OpenVPN server
+  hosts: openvpn-server1.ooni.io
+  become: true
+  remote_user: root
+  roles:
+    - ssh_users
+
+- name: Deploy notebook host
+  hosts: notebook.ooni.org
+  become: true
+  tags:
+    - notebook
+  vars:
+    enable_oonipipeline_worker: false
+  roles:
+    - oonidata
+
+# commented out due to the fact it requires manual config of ~/.ssh/config
+#- name: Setup codesign box
+#  hosts: codesign-box
+#  become: true
+#  remote_user: ubuntu
+#  roles:
+#    - codesign_box

ansible/group_vars/all/vars.yml

@@ -2,11 +2,17 @@ ssh_users:
   agrabeli:
     login: agrabeli
     comment: Maria Xynou
-    keys: ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD0JSwM+t3Uz9lS3Mjoz9oo4vOToWyzboZhYQbP8JY5HvFtAvWanWHnUBO91t6hkgKIMiUqhdCJn26fqkhSGe/bRBaFUocOmuyfcmZoRdi0qzAskmycJsj/w6vWR4x6MYkmJvSeI/MGxjEFt4s2MfOG1tP8CBLUYft9qUleeJa7Jln8c+xbnqB7YngaI190icQHE9NuIB2CXvzbmo3tLtHNMagEwI7VoBDj6mxzTxBd9JhuhF4w5uGxxm0Gp1hzk+15obNnaBS+Anr7jXz8FPwwxCH+XhBZxB1PPpcIayKrf9iLyGtwmhkdDoWCqYAr1mue3LxFso+TZF4bwE4Cjt1 agrabelh@agrabelh"]
+    keys:
+      [
+        "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD0JSwM+t3Uz9lS3Mjoz9oo4vOToWyzboZhYQbP8JY5HvFtAvWanWHnUBO91t6hkgKIMiUqhdCJn26fqkhSGe/bRBaFUocOmuyfcmZoRdi0qzAskmycJsj/w6vWR4x6MYkmJvSeI/MGxjEFt4s2MfOG1tP8CBLUYft9qUleeJa7Jln8c+xbnqB7YngaI190icQHE9NuIB2CXvzbmo3tLtHNMagEwI7VoBDj6mxzTxBd9JhuhF4w5uGxxm0Gp1hzk+15obNnaBS+Anr7jXz8FPwwxCH+XhBZxB1PPpcIayKrf9iLyGtwmhkdDoWCqYAr1mue3LxFso+TZF4bwE4Cjt1 agrabelh@agrabelh",
+      ]
   art:
     login: art
     comment: Arturo Filasto
-    keys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsibU0nsQFFIdolD1POzXOws4VetV0ZNByINRzY8Hx0 arturo@ooni.org"]
+    keys:
+      [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsibU0nsQFFIdolD1POzXOws4VetV0ZNByINRzY8Hx0 arturo@ooni.org",
+      ]
   majakomel:
     login: majakomel
     comment: Maja Komel
@@ -23,7 +29,9 @@ ssh_users:
     keys:
       - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBXprrutdT6AhrV9hWBKjyzq6RqGmCBWpWxi3qwJyRcBJfkiEYKV9QWl3H0g/Sg9JzLd9lWG2yfAai7cyBAT4Ih0+OhwQ0V7wkhBn4YkNjs7d4BGPHjuLIywS9VtmiyH7VafikMjmqPLL/uPBIbRrx9RuSfLkAuN9XFZpVmqzWY8ePpcRCvnG6ucPxEY8o+4j5nfTrgxSaIT31kH16/PFJe07tn1SZjxZE4sZTz/p9xKt6s8HXmlP3RdnXSpXWmH8ZwYDrNhkcH8m6mC3giiqSKThFdwvQVflRRvn9pAlUOhy6KIBtAt1KobVJtOCPrrkcLhQ1C+2P9wKhfYspCGrScFGnrUqumLxPpwlqILxJvmgqGAtkm8Ela9f2D9sEv8CUv5x9XptZKlyRhtOLixvLYoJlwfXXnmXa8T1pg8+4063BhHUOu/bg0InpSp3hdscOfk0R8FtDlXnn6COwbPXynIt4PxzIxD/WQhP0ymgH3ky6ClB5wRBVhOqYvxQw32n2QFS9A5ocga+nATiOE7BTOufgmDCA/OIXfJ/GukXRaMCBsvlx7tObHS1LOMt0I+WdoOEjI0ARUrFzwoiTrs9QYmd922e7S35EnheT3JjnCTjebJrCNtwritUy8vjsN/M27wJs7MAXleT7drwXXnm+3xYrH+4KQ+ru0dxMe1zfBw== aanorbel@gmail.com"
 
-admin_usernames: [ art, mehul ]
-root_usernames: [ art, mehul ]
-non_admin_usernames: [ ]
-deactivated_usernames: [ sbs, federico, sarath ]
+admin_usernames: [art, mehul]
+root_usernames: [art, mehul]
+non_admin_usernames: []
+deactivated_usernames: [sbs, federico, sarath]
+
+prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}"

ansible/group_vars/clickhouse/vars.yml

@@ -7,6 +7,8 @@ nftables_clickhouse_allow:
     ip: 168.119.7.188
   - fqdn: notebook.ooni.org
     ip: 138.201.19.39
+  - fqdn: clickhouseproxy.dev.ooni.io
+    ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}"
 
 nftables_zookeeper_allow:
   - fqdn: data1.htz-fsn.prod.ooni.nu
@@ -24,7 +26,7 @@ clickhouse_config:
   max_connections: 4096
   keep_alive_timeout: 3
   max_concurrent_queries: 100
-  max_server_memory_usage: 0
+  max_server_memory_usage: 21001001000
   max_thread_pool_size: 10000
   max_server_memory_usage_to_ram_ratio: 0.9
   total_memory_profiler_step: 4194304
@@ -154,6 +156,10 @@ clickhouse_distributed_ddl:
 clickhouse_default_profiles:
   default:
     readonly: 2
+    max_memory_usage: 11001001000
+    use_uncompressed_cache: 0
+    load_balancing: random
+    max_partitions_per_insert_block: 100
   readonly:
     readonly: 1
   write:
@@ -194,3 +200,17 @@ clickhouse_default_quotas:
     result_rows: 0
     read_rows: 0
     execution_time: 0
+
+clickhouse_prometheus:
+  endpoint: "/metrics"
+  port: 9363
+  metrics: true
+  events: true
+  asynchronous_metrics: true
+  status_info: true
+
+prometheus_nginx_proxy_config:
+  - location: /metrics/node_exporter
+    proxy_pass: http://127.0.0.1:8100/metrics
+  - location: /metrics/clickhouse
+    proxy_pass: http://127.0.0.1:9363/metrics

ansible/inventory

@@ -1,22 +1,24 @@
-[all]
-# This requires manual setup of ~/.ssh/config
-#codesign-box
+[all:children]
+htz-fsn
+ghs-ams
 
-[prod]
-data.ooni.org
-oonidata.ooni.org
-monitoring.ooni.org
-openvpn-server1.ooni.io
+## Role tags
+
+[clickhouse]
 notebook.ooni.org
 data1.htz-fsn.prod.ooni.nu
 data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
 
-[dev]
-oonidatatest.ooni.nu
+## Location tags
 
-[clickhouse]
+[htz-fsn]
+data.ooni.org
+monitoring.ooni.org
 notebook.ooni.org
 data1.htz-fsn.prod.ooni.nu
 data2.htz-fsn.prod.ooni.nu
 data3.htz-fsn.prod.ooni.nu
+
+[ghs-ams]
+openvpn-server1.ooni.io

ansible/playbook.yml

@@ -7,63 +7,8 @@
   tags:
     - bootstrap
 
-- name: Update monitoring config
-  hosts: monitoring.ooni.org
-  become: true
-  tags:
-    - monitoring
-  roles:
-    - prometheus
-    - prometheus_blackbox_exporter
-    - prometheus_alertmanager
-
-- name: Setup OpenVPN server
-  hosts: openvpn-server1.ooni.io
-  become: true
-  remote_user: root
-  roles:
-    - ssh_users
-
-- name: Deploy oonidata clickhouse hosts
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-    #- data2.htz-fsn.prod.ooni.nu
-    - data3.htz-fsn.prod.ooni.nu
-    - notebook.ooni.org
-  become: true
-  tags:
-    - clickhouse
-  roles:
-    #- tailnet
-    - oonidata_clickhouse
-
-- name: Deploy oonidata worker nodes
-  hosts:
-    - data1.htz-fsn.prod.ooni.nu
-  become: true
-  tags:
-    - oonidata_worker
-  roles:
-    - oonidata
-  vars:
-    enable_jupyterhub: false
-    enable_oonipipeline_worker: true
-    clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni"
-
-- name: Deploy notebook host
-  hosts: notebook.ooni.org
-  become: true
-  tags:
-    - notebook
-  vars:
-    enable_oonipipeline_worker: false
-  roles:
-    - oonidata
+- name: Include tier0 playbook
+  ansible.builtin.import_playbook: deploy-tier0.yml
 
-# commented out due to the fact it requires manual config of ~/.ssh/config
-#- name: Setup codesign box
-#  hosts: codesign-box
-#  become: true
-#  remote_user: ubuntu
-#  roles:
-#    - codesign_box
+- name: Include tier2 playbook
+  ansible.builtin.import_playbook: deploy-tier2.yml

ansible/requirements.yml

@@ -1,7 +1,6 @@
 - src: willshersystems.sshd
 - src: nginxinc.nginx
 - src: geerlingguy.certbot
-- src: geerlingguy.node_exporter
 - src: artis3n.tailscale
 - src: https://github.com/idealista/clickhouse_role
   scm: git

ansible/roles/base-backend/README.adoc

@@ -0,0 +1 @@
+Configure base host based on backend hosts

ansible/roles/base-backend/handlers/main.yml

@@ -0,0 +1,15 @@
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded
+
+- name: restart chrony
+  ansible.builtin.systemd:
+    name: chrony.service
+    state: restarted
+
+- name: restart netdata
+  ansible.builtin.systemd:
+    name: netdata.service
+    state: restarted

ansible/roles/base-backend/meta/main.yml

@@ -0,0 +1,6 @@
+---
+dependencies:
+  - role: adm
+    become: false
+    remote_user: root
+    gather_facts: false