diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml new file mode 100644 index 00000000..c2d34cc7 --- /dev/null +++ b/ansible/deploy-clickhouse.yml @@ -0,0 +1,13 @@ +--- +- name: Deploy oonidata clickhouse hosts + hosts: + - notebook.ooni.org + - data1.htz-fsn.prod.ooni.nu + #- data2.htz-fsn.prod.ooni.nu + - data3.htz-fsn.prod.ooni.nu + become: true + tags: + - clickhouse + roles: + - prometheus_node_exporter + - oonidata_clickhouse diff --git a/ansible/deploy-monitoring.yml b/ansible/deploy-monitoring.yml new file mode 100644 index 00000000..a1eadee9 --- /dev/null +++ b/ansible/deploy-monitoring.yml @@ -0,0 +1,12 @@ +--- +- name: Update monitoring config + hosts: monitoring.ooni.org + become: true + tags: + - monitoring + roles: + - prometheus + - prometheus_blackbox_exporter + - prometheus_alertmanager + + diff --git a/ansible/deploy-ooni-backend.yml b/ansible/deploy-ooni-backend.yml new file mode 100644 index 00000000..24c70aac --- /dev/null +++ b/ansible/deploy-ooni-backend.yml @@ -0,0 +1,21 @@ +--- +- hosts: backend-hel.ooni.org + roles: + - role: bootstrap + - role: base-backend + - role: nftables + - role: nginx + tags: nginx + vars: + nginx_user: "www-data" + - role: dehydrated + tags: dehydrated + expand: yes + vars: + ssl_domains: + # with dehydrated the first entry is the cert FQDN + # and the other ones are alternative names + - "backend-hel.ooni.org" + - role: ooni-backend + vars: + ssl_domain: backend-hel.ooni.org diff --git a/ansible/deploy-tier0.yml b/ansible/deploy-tier0.yml new file mode 100644 index 00000000..7c11a8c6 --- /dev/null +++ b/ansible/deploy-tier0.yml @@ -0,0 +1,22 @@ +--- +- name: Include monitoring playbook + ansible.builtin.import_playbook: deploy-monitoring.yml + +- name: Include ooni-backend playbook + ansible.builtin.import_playbook: deploy-ooni-backend.yml + +- name: Include clickhouse playbook + ansible.builtin.import_playbook: deploy-clickhouse.yml + +- name: Deploy oonidata worker nodes + hosts: + - data1.htz-fsn.prod.ooni.nu + become: true + tags: + - oonidata_worker + roles: + - oonidata + vars: + enable_jupyterhub: false + enable_oonipipeline_worker: true + clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni" diff --git a/ansible/deploy-tier2.yml b/ansible/deploy-tier2.yml new file mode 100644 index 00000000..8f87a663 --- /dev/null +++ b/ansible/deploy-tier2.yml @@ -0,0 +1,25 @@ +--- +- name: Setup OpenVPN server + hosts: openvpn-server1.ooni.io + become: true + remote_user: root + roles: + - ssh_users + +- name: Deploy notebook host + hosts: notebook.ooni.org + become: true + tags: + - notebook + vars: + enable_oonipipeline_worker: false + roles: + - oonidata + +# commented out due to the fact it requires manual config of ~/.ssh/config +#- name: Setup codesign box +# hosts: codesign-box +# become: true +# remote_user: ubuntu +# roles: +# - codesign_box diff --git a/ansible/group_vars/all/vars.yml b/ansible/group_vars/all/vars.yml index 17712861..c0b94053 100644 --- a/ansible/group_vars/all/vars.yml +++ b/ansible/group_vars/all/vars.yml @@ -2,11 +2,17 @@ ssh_users: agrabeli: login: agrabeli comment: Maria Xynou - keys: ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD0JSwM+t3Uz9lS3Mjoz9oo4vOToWyzboZhYQbP8JY5HvFtAvWanWHnUBO91t6hkgKIMiUqhdCJn26fqkhSGe/bRBaFUocOmuyfcmZoRdi0qzAskmycJsj/w6vWR4x6MYkmJvSeI/MGxjEFt4s2MfOG1tP8CBLUYft9qUleeJa7Jln8c+xbnqB7YngaI190icQHE9NuIB2CXvzbmo3tLtHNMagEwI7VoBDj6mxzTxBd9JhuhF4w5uGxxm0Gp1hzk+15obNnaBS+Anr7jXz8FPwwxCH+XhBZxB1PPpcIayKrf9iLyGtwmhkdDoWCqYAr1mue3LxFso+TZF4bwE4Cjt1 agrabelh@agrabelh"] + keys: + [ + "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDD0JSwM+t3Uz9lS3Mjoz9oo4vOToWyzboZhYQbP8JY5HvFtAvWanWHnUBO91t6hkgKIMiUqhdCJn26fqkhSGe/bRBaFUocOmuyfcmZoRdi0qzAskmycJsj/w6vWR4x6MYkmJvSeI/MGxjEFt4s2MfOG1tP8CBLUYft9qUleeJa7Jln8c+xbnqB7YngaI190icQHE9NuIB2CXvzbmo3tLtHNMagEwI7VoBDj6mxzTxBd9JhuhF4w5uGxxm0Gp1hzk+15obNnaBS+Anr7jXz8FPwwxCH+XhBZxB1PPpcIayKrf9iLyGtwmhkdDoWCqYAr1mue3LxFso+TZF4bwE4Cjt1 agrabelh@agrabelh", + ] art: login: art comment: Arturo Filasto - keys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsibU0nsQFFIdolD1POzXOws4VetV0ZNByINRzY8Hx0 arturo@ooni.org"] + keys: + [ + "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsibU0nsQFFIdolD1POzXOws4VetV0ZNByINRzY8Hx0 arturo@ooni.org", + ] majakomel: login: majakomel comment: Maja Komel @@ -23,7 +29,9 @@ ssh_users: keys: - "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQDBXprrutdT6AhrV9hWBKjyzq6RqGmCBWpWxi3qwJyRcBJfkiEYKV9QWl3H0g/Sg9JzLd9lWG2yfAai7cyBAT4Ih0+OhwQ0V7wkhBn4YkNjs7d4BGPHjuLIywS9VtmiyH7VafikMjmqPLL/uPBIbRrx9RuSfLkAuN9XFZpVmqzWY8ePpcRCvnG6ucPxEY8o+4j5nfTrgxSaIT31kH16/PFJe07tn1SZjxZE4sZTz/p9xKt6s8HXmlP3RdnXSpXWmH8ZwYDrNhkcH8m6mC3giiqSKThFdwvQVflRRvn9pAlUOhy6KIBtAt1KobVJtOCPrrkcLhQ1C+2P9wKhfYspCGrScFGnrUqumLxPpwlqILxJvmgqGAtkm8Ela9f2D9sEv8CUv5x9XptZKlyRhtOLixvLYoJlwfXXnmXa8T1pg8+4063BhHUOu/bg0InpSp3hdscOfk0R8FtDlXnn6COwbPXynIt4PxzIxD/WQhP0ymgH3ky6ClB5wRBVhOqYvxQw32n2QFS9A5ocga+nATiOE7BTOufgmDCA/OIXfJ/GukXRaMCBsvlx7tObHS1LOMt0I+WdoOEjI0ARUrFzwoiTrs9QYmd922e7S35EnheT3JjnCTjebJrCNtwritUy8vjsN/M27wJs7MAXleT7drwXXnm+3xYrH+4KQ+ru0dxMe1zfBw== aanorbel@gmail.com" -admin_usernames: [ art, mehul ] -root_usernames: [ art, mehul ] -non_admin_usernames: [ ] -deactivated_usernames: [ sbs, federico, sarath ] +admin_usernames: [art, mehul] +root_usernames: [art, mehul] +non_admin_usernames: [] +deactivated_usernames: [sbs, federico, sarath] + +prometheus_metrics_password: "{{ lookup('amazon.aws.aws_secret', 'oonidevops/ooni_services/prometheus_metrics_password', profile='oonidevops_user_prod') }}" diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 8e7388e8..f1ac5248 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -7,6 +7,8 @@ nftables_clickhouse_allow: ip: 168.119.7.188 - fqdn: notebook.ooni.org ip: 138.201.19.39 + - fqdn: clickhouseproxy.dev.ooni.io + ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}" nftables_zookeeper_allow: - fqdn: data1.htz-fsn.prod.ooni.nu @@ -24,7 +26,7 @@ clickhouse_config: max_connections: 4096 keep_alive_timeout: 3 max_concurrent_queries: 100 - max_server_memory_usage: 0 + max_server_memory_usage: 21001001000 max_thread_pool_size: 10000 max_server_memory_usage_to_ram_ratio: 0.9 total_memory_profiler_step: 4194304 @@ -154,6 +156,10 @@ clickhouse_distributed_ddl: clickhouse_default_profiles: default: readonly: 2 + max_memory_usage: 11001001000 + use_uncompressed_cache: 0 + load_balancing: random + max_partitions_per_insert_block: 100 readonly: readonly: 1 write: @@ -194,3 +200,17 @@ clickhouse_default_quotas: result_rows: 0 read_rows: 0 execution_time: 0 + +clickhouse_prometheus: + endpoint: "/metrics" + port: 9363 + metrics: true + events: true + asynchronous_metrics: true + status_info: true + +prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:8100/metrics + - location: /metrics/clickhouse + proxy_pass: http://127.0.0.1:9363/metrics diff --git a/ansible/inventory b/ansible/inventory index 25f1f5df..a44f8d45 100644 --- a/ansible/inventory +++ b/ansible/inventory @@ -1,22 +1,24 @@ -[all] -# This requires manual setup of ~/.ssh/config -#codesign-box +[all:children] +htz-fsn +ghs-ams -[prod] -data.ooni.org -oonidata.ooni.org -monitoring.ooni.org -openvpn-server1.ooni.io +## Role tags + +[clickhouse] notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu -[dev] -oonidatatest.ooni.nu +## Location tags -[clickhouse] +[htz-fsn] +data.ooni.org +monitoring.ooni.org notebook.ooni.org data1.htz-fsn.prod.ooni.nu data2.htz-fsn.prod.ooni.nu data3.htz-fsn.prod.ooni.nu + +[ghs-ams] +openvpn-server1.ooni.io diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 63d2b448..17bcd402 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -7,63 +7,8 @@ tags: - bootstrap -- name: Update monitoring config - hosts: monitoring.ooni.org - become: true - tags: - - monitoring - roles: - - prometheus - - prometheus_blackbox_exporter - - prometheus_alertmanager - -- name: Setup OpenVPN server - hosts: openvpn-server1.ooni.io - become: true - remote_user: root - roles: - - ssh_users - -- name: Deploy oonidata clickhouse hosts - hosts: - - data1.htz-fsn.prod.ooni.nu - #- data2.htz-fsn.prod.ooni.nu - - data3.htz-fsn.prod.ooni.nu - - notebook.ooni.org - become: true - tags: - - clickhouse - roles: - #- tailnet - - oonidata_clickhouse - -- name: Deploy oonidata worker nodes - hosts: - - data1.htz-fsn.prod.ooni.nu - become: true - tags: - - oonidata_worker - roles: - - oonidata - vars: - enable_jupyterhub: false - enable_oonipipeline_worker: true - clickhouse_url: "clickhouse://write:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_write_password', profile='oonidevops_user_prod') | hash('sha256') }}@clickhouse1.prod.ooni.io/ooni" - -- name: Deploy notebook host - hosts: notebook.ooni.org - become: true - tags: - - notebook - vars: - enable_oonipipeline_worker: false - roles: - - oonidata +- name: Include tier0 playbook + ansible.builtin.import_playbook: deploy-tier0.yml -# commented out due to the fact it requires manual config of ~/.ssh/config -#- name: Setup codesign box -# hosts: codesign-box -# become: true -# remote_user: ubuntu -# roles: -# - codesign_box +- name: Include tier2 playbook + ansible.builtin.import_playbook: deploy-tier2.yml diff --git a/ansible/requirements.yml b/ansible/requirements.yml index 0a2eae7d..52ae85ea 100644 --- a/ansible/requirements.yml +++ b/ansible/requirements.yml @@ -1,7 +1,6 @@ - src: willshersystems.sshd - src: nginxinc.nginx - src: geerlingguy.certbot -- src: geerlingguy.node_exporter - src: artis3n.tailscale - src: https://github.com/idealista/clickhouse_role scm: git diff --git a/ansible/roles/base-backend/README.adoc b/ansible/roles/base-backend/README.adoc new file mode 100644 index 00000000..ac3f7039 --- /dev/null +++ b/ansible/roles/base-backend/README.adoc @@ -0,0 +1 @@ +Configure base host based on backend hosts diff --git a/ansible/roles/base-backend/handlers/main.yml b/ansible/roles/base-backend/handlers/main.yml new file mode 100644 index 00000000..4a8d06e8 --- /dev/null +++ b/ansible/roles/base-backend/handlers/main.yml @@ -0,0 +1,15 @@ +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart chrony + ansible.builtin.systemd: + name: chrony.service + state: restarted + +- name: restart netdata + ansible.builtin.systemd: + name: netdata.service + state: restarted diff --git a/ansible/roles/base-backend/meta/main.yml b/ansible/roles/base-backend/meta/main.yml new file mode 100644 index 00000000..5de9bc56 --- /dev/null +++ b/ansible/roles/base-backend/meta/main.yml @@ -0,0 +1,6 @@ +--- +dependencies: + - role: adm + become: false + remote_user: root + gather_facts: false diff --git a/ansible/roles/base-backend/tasks/main.yml b/ansible/roles/base-backend/tasks/main.yml new file mode 100644 index 00000000..00a7352a --- /dev/null +++ b/ansible/roles/base-backend/tasks/main.yml @@ -0,0 +1,140 @@ +--- +- name: motd + shell: echo "" > /etc/motd + +- name: Remove apt repo + tags: apt + file: + path: /etc/apt/sources.list.d/ftp_nl_debian_org_debian.list + state: absent + +- name: Remove apt repo + tags: apt + file: + path: /etc/apt/sources.list.d/security_debian_org.list + state: absent + +- name: Create internal-deb repo GPG pubkey + tags: apt + template: + src: templates/internal-deb.gpg + dest: /etc/ooni/internal-deb.gpg + mode: 0644 + owner: root + +- name: Set apt repos + tags: apt + template: + src: templates/sources.list + dest: /etc/apt/sources.list + mode: 0644 + owner: root + +- name: Install gpg + tags: base-packages + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - gpg + - gpg-agent + +- name: Update apt cache + tags: apt + apt: + update_cache: yes + +- name: Installs base packages + tags: base-packages + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - bash-completion + - byobu + - chrony + - etckeeper + - fail2ban + - git + - iotop + - jupyter-notebook + - manpages + - ncdu + - netdata-core + - netdata-plugins-bash + - netdata-plugins-python + - netdata-web + - nftables + - nullmailer + - prometheus-node-exporter + - pv + # needed by ansible + - python3-apt + - rsync + - ssl-cert + - strace + - tcpdump + - tmux + - vim + +- name: Autoremove + tags: autoremove + apt: + autoremove: yes + +- name: Clean cache + tags: apt + apt: + autoclean: yes + +- name: allow netdata.service + tags: netdata + blockinfile: + path: /etc/ooni/nftables/tcp/19999.nft + create: yes + block: | + add rule inet filter input ip saddr {{ lookup('dig', 'prometheus.ooni.org/A') }} tcp dport 19999 counter accept comment "netdata.service" + notify: + - reload nftables + +- name: configure netdata.service + tags: netdata + template: + src: netdata.conf + dest: /etc/netdata/netdata.conf + +- name: disable netdata emails + tags: netdata + blockinfile: + path: /etc/netdata/conf.d/health_alarm_notify.conf + create: yes + block: | + # Managed by ansible, see roles/base-bookworm/tasks/main.yml + SEND_EMAIL="NO" + +- name: Set timezone + tags: timezone + timezone: + name: Etc/UTC + notify: + - restart chrony + +- name: configure netdata chrony + tags: netdata, timezone + blockinfile: + path: /etc/netdata/python.d/chrony.conf + create: yes + block: | + # Managed by ansible, see roles/base-bookworm/tasks/main.yml + update_every: 5 + local: + command: 'chronyc -n tracking' + +- name: configure netdata chrony + tags: netdata, timezone + lineinfile: + path: /usr/lib/netdata/conf.d/python.d.conf + regexp: '^chrony:' + line: 'chrony: yes' + notify: + - restart netdata diff --git a/ansible/roles/base-backend/templates/internal-deb.gpg b/ansible/roles/base-backend/templates/internal-deb.gpg new file mode 100644 index 00000000..28126a36 --- /dev/null +++ b/ansible/roles/base-backend/templates/internal-deb.gpg @@ -0,0 +1,14 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ +JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh +dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb +AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb +gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo +Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX +HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh +tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy +6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s +oN7jfdbznrWVigE= +=PtYb +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/base-backend/templates/journald.conf b/ansible/roles/base-backend/templates/journald.conf new file mode 100644 index 00000000..d7ae85e1 --- /dev/null +++ b/ansible/roles/base-backend/templates/journald.conf @@ -0,0 +1,8 @@ +[Journal] +Storage=persistent +Compress=yes +#RateLimitIntervalSec=30s +#RateLimitBurst=10000 +SystemMaxFileSize=200M +RuntimeMaxFileSize=1G +ForwardToSyslog=no diff --git a/ansible/roles/base-backend/templates/netdata.conf b/ansible/roles/base-backend/templates/netdata.conf new file mode 100644 index 00000000..e2bef302 --- /dev/null +++ b/ansible/roles/base-backend/templates/netdata.conf @@ -0,0 +1,32 @@ +# Managed by ansible, see roles/base-bookworm/tasks/main.yml +[global] + run as user = netdata + web files owner = root + web files group = root + bind socket to IP = 0.0.0.0 + +[plugins] + python.d = yes + + +[statsd] + enabled = yes + # decimal detail = 1000 + update every (flushInterval) = 1 + # udp messages to process at once = 10 + # create private charts for metrics matching = * + max private charts allowed = 10000 + max private charts hard limit = 10000 + private charts memory mode = ram + private charts history = 300 + # histograms and timers percentile (percentThreshold) = 95.00000 + # add dimension for number of events received = no + # gaps on gauges (deleteGauges) = no + # gaps on counters (deleteCounters) = no + # gaps on meters (deleteMeters) = no + # gaps on sets (deleteSets) = no + # gaps on histograms (deleteHistograms) = no + # gaps on timers (deleteTimers) = no + # listen backlog = 4096 + # default port = 8125 + # bind to = udp:localhost:8125 tcp:localhost:8125 diff --git a/ansible/roles/base-backend/templates/ooni_internal.sources b/ansible/roles/base-backend/templates/ooni_internal.sources new file mode 100644 index 00000000..f85bc625 --- /dev/null +++ b/ansible/roles/base-backend/templates/ooni_internal.sources @@ -0,0 +1,7 @@ +Architectures: amd64 +Suites: unstable +Uris: https://ooni-internal-deb.s3.eu-central-1.amazonaws.com +Types: deb +Components: main +Enabled: yes +Signed-By: /etc/ooni/internal-deb.gpg diff --git a/ansible/roles/base-backend/templates/resolved.conf b/ansible/roles/base-backend/templates/resolved.conf new file mode 100644 index 00000000..aa68eaf1 --- /dev/null +++ b/ansible/roles/base-backend/templates/resolved.conf @@ -0,0 +1,9 @@ +# Deployed by ansible +# See roles/base-bookworm/templates/resolved.conf + +[Resolve] +DNS=9.9.9.9 +FallbackDNS=1.1.1.1 8.8.8.8 +DNSOverTLS=opportunistic +DNSSEC=allow-downgrade +Cache=yes diff --git a/ansible/roles/base-backend/templates/sources.list b/ansible/roles/base-backend/templates/sources.list new file mode 100644 index 00000000..7432ddad --- /dev/null +++ b/ansible/roles/base-backend/templates/sources.list @@ -0,0 +1,6 @@ +# Managed by ansible +# roles/base-bookworm/templates/sources.list + +deb http://deb.debian.org/debian bookworm main contrib non-free-firmware +deb http://deb.debian.org/debian-security/ bookworm-security main contrib non-free-firmware +deb http://deb.debian.org/debian bookworm-backports main diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index ecf1d46f..500d58ff 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -55,11 +55,6 @@ tags: - nftables -- ansible.builtin.include_role: - name: prometheus_node_exporter - tags: - - node_exporter - - name: Configure journald tags: - journald diff --git a/ansible/roles/dehydrated/README.adoc b/ansible/roles/dehydrated/README.adoc new file mode 100644 index 00000000..477601de --- /dev/null +++ b/ansible/roles/dehydrated/README.adoc @@ -0,0 +1,10 @@ + +Configure dehydrated to generate certificates (locally to each server) + +- listen on port 443 for ACME challenge + +- ansible --diff is supported + +- generate certificate expirations metrics for node exporter + +- changes to /etc are also tracked locally by etckeeper diff --git a/ansible/roles/dehydrated/meta/main.yml b/ansible/roles/dehydrated/meta/main.yml new file mode 100644 index 00000000..e7e996b0 --- /dev/null +++ b/ansible/roles/dehydrated/meta/main.yml @@ -0,0 +1,5 @@ +--- +dependencies: + - nginx-buster +... + diff --git a/ansible/roles/dehydrated/tasks/main.yml b/ansible/roles/dehydrated/tasks/main.yml new file mode 100644 index 00000000..0bfaf7c3 --- /dev/null +++ b/ansible/roles/dehydrated/tasks/main.yml @@ -0,0 +1,108 @@ +--- +- name: Installs packages + tags: dehydrated + apt: + install_recommends: no + cache_valid_time: 86400 + name: + - dehydrated + +#- name: create dehydrated hook file +# # This hook is called after getting a new cert to deploy it +# template: +# src: templates/hook.sh +# dest: /etc/dehydrated/hook.sh +# mode: 0755 +# owner: root +# +# +#- name: set dehydrated hook +# blockinfile: +# path: /etc/dehydrated/config +# block: | +# HOOK="/etc/dehydrated/hook.sh" + +- name: Add ACME dedicated sites-enabled file + tags: dehydrated + template: + src: templates/letsencrypt-http + # the server block matches all SSL FQDNs and must be + # parsed first, hence 00- + dest: /etc/nginx/sites-enabled/00-letsencrypt-http + mode: 0644 + owner: root + +- name: Add canary file to ensure /.well-known/acme-challenge is reachable by let's encrypt + tags: dehydrated + copy: + content: | + Generated by ansible using ansible/roles/dehydrated/tasks/main.yml. + + Also, meow!!! + dest: /var/lib/dehydrated/acme-challenges/ooni-acme-canary + mode: 0644 + owner: root + +- name: reload nginx + tags: dehydrated + shell: systemctl reload nginx.service + +- name: allow incoming TCP connections to Nginx on port 80 + tags: dehydrated + blockinfile: + path: /etc/ooni/nftables/tcp/80.nft + create: yes + block: | + add rule inet filter input tcp dport 80 counter accept comment "incoming HTTP" + +- name: reload nftables service + tags: dehydrated + shell: systemctl reload nftables.service + +- name: Configure domains {{ ssl_domains }} + # https://github.com/dehydrated-io/dehydrated/blob/master/docs/domains_txt.md + tags: dehydrated + template: + src: templates/domains.txt.j2 + dest: /etc/dehydrated/domains.txt + +- name: Register account if needed + tags: dehydrated + ansible.builtin.shell: + cmd: "test -d /var/lib/dehydrated/accounts || dehydrated --register --accept-terms" + +- name: Install dehydrated.service + tags: dehydrated + template: + src: templates/dehydrated.service + dest: /etc/systemd/system/dehydrated.service + mode: 0644 + owner: root + +- name: Install dehydrated.timer + tags: dehydrated + template: + src: templates/dehydrated.timer + dest: /etc/systemd/system/dehydrated.timer + mode: 0644 + owner: root + +- name: Ensure timer runs + tags: dehydrated + systemd: + name: dehydrated.timer + state: started + enabled: yes + +- name: Run dehydrated service immediately + # creates: + # /var/lib/dehydrated/certs//chain.pem cert.pem privkey.pem fullchain.pem + tags: dehydrated + systemd: + name: dehydrated.service + state: started + enabled: yes + +- name: reload nginx + tags: dehydrated + shell: systemctl reload nginx.service diff --git a/ansible/roles/dehydrated/templates/dehydrated.service b/ansible/roles/dehydrated/templates/dehydrated.service new file mode 100644 index 00000000..50ffdc46 --- /dev/null +++ b/ansible/roles/dehydrated/templates/dehydrated.service @@ -0,0 +1,13 @@ +[Unit] +Description=Run dehydrated certificate refresh + +[Service] +Type=oneshot +#User=dehydrated +#Group=dehydrated +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/dehydrated +PrivateTmp=yes +ExecStart=/usr/bin/dehydrated --cron +ExecStartPost=+/bin/systemctl reload nginx.service diff --git a/ansible/roles/dehydrated/templates/dehydrated.timer b/ansible/roles/dehydrated/templates/dehydrated.timer new file mode 100644 index 00000000..5e6ea784 --- /dev/null +++ b/ansible/roles/dehydrated/templates/dehydrated.timer @@ -0,0 +1,9 @@ +[Unit] +Description=Run dehydrated certificate refresh + +[Timer] +OnCalendar=Mon 13:00 + +[Install] +WantedBy=timers.target + diff --git a/ansible/roles/dehydrated/templates/domains.txt.j2 b/ansible/roles/dehydrated/templates/domains.txt.j2 new file mode 100644 index 00000000..5850d203 --- /dev/null +++ b/ansible/roles/dehydrated/templates/domains.txt.j2 @@ -0,0 +1 @@ +{% for d in ssl_domains %}{{ d }} {% endfor %} diff --git a/ansible/roles/dehydrated/templates/hook.sh b/ansible/roles/dehydrated/templates/hook.sh new file mode 100644 index 00000000..26193aeb --- /dev/null +++ b/ansible/roles/dehydrated/templates/hook.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +# Deployed by ansible +# see ansible/roles/dehydrated/templates/hook.sh +# +deploy_cert() { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" + # This hook is called once for each certificate that has been produced. + # Parameters: + # - DOMAIN The primary domain name, i.e. the certificate common name (CN). + # - KEYFILE The path of the file containing the private key. + # - CERTFILE The path of the file containing the signed certificate. + # - FULLCHAINFILE The path of the file containing the full certificate chain. + # - CHAINFILE The path of the file containing the intermediate certificate(s). + # - TIMESTAMP Timestamp when the specified certificate was created. + + logger "Deploying SSL certificate $DOMAIN $KEYFILE $CERTFILE $FULLCHAINFILE $CHAINFILE $TIMESTAMP" + # cp ... + #systemctl reload nginx +} diff --git a/ansible/roles/dehydrated/templates/letsencrypt-http b/ansible/roles/dehydrated/templates/letsencrypt-http new file mode 100644 index 00000000..41fda273 --- /dev/null +++ b/ansible/roles/dehydrated/templates/letsencrypt-http @@ -0,0 +1,13 @@ +# Generated by ansible +# roles/dehydrated/templates/letsencrypt-http + +server { + # Listen on port 80 for *any* domain + listen 80; + server_name _; + + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } +} diff --git a/ansible/roles/nginx/defaults/main.yml b/ansible/roles/nginx/defaults/main.yml new file mode 100644 index 00000000..4c0ac11a --- /dev/null +++ b/ansible/roles/nginx/defaults/main.yml @@ -0,0 +1 @@ +nginx_user: nginx diff --git a/ansible/roles/nginx/templates/nginx.conf b/ansible/roles/nginx/templates/nginx.conf index f43bf7c5..7b1b594c 100644 --- a/ansible/roles/nginx/templates/nginx.conf +++ b/ansible/roles/nginx/templates/nginx.conf @@ -1,122 +1,61 @@ -# NB: system nginx uses `www-data` user! -user nginx; -worker_processes 2; +# Managed by ansible +# roles/nginx/templates/nginx.conf +# -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; +user {{ nginx_user }}; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; events { - worker_connections 1024; + worker_connections 768; + # multi_accept on; } http { - include /etc/nginx/mime.types; - default_type application/octet-stream; - - geo $is_ooni { - # TODO: this is not implemented ATM - default 0; - } - - map $http_x_request_id $has_request_id { # check for `X-Request-ID` - "" 0; - default 1; - } - - map "$is_ooni:$has_request_id" $ooni_request_id { - "1:1" $http_x_request_id; # use `X-Request-ID` if it's okay - default $request_id; - } - - # IPv4 is anonymized to /24, IPv6 to /48 - according to OONI Data Policy. - # https://ooni.torproject.org/about/data-policy/ - # IP is recorded to track possible abusers, not to distinguish users, so the - # address is truncated down to ISP (min routable prefix) instead of hashing. - map $remote_addr $ooni_remote_addr { - default "0.0.0.0"; - # variables in map value require nginx/1.11.0+ - "~(?P\d+\.\d+\.\d+)\.\d+" "$ip.0"; - # :: means at least TWO zero 16bit fields, https://tools.ietf.org/html/rfc5952#section-4.2.2 - "~(?P[0-9a-f]+:[0-9a-f]+:[0-9a-f]+):[0-9a-f:]+" "$ip::"; - "~(?P[0-9a-f]+:[0-9a-f]+)::[0-9a-f:]+" "$ip::"; - "~(?P[0-9a-f]+)::[0-9a-f:]+" "$ip::"; - } - - # $server_name is important as mtail does not distinguish log lines from - # different files, $host is required to log actual `Host` header. - # $request is split into separate fields to ease awk and mtail parsing. - # $scheme is used instead of $https to ease eye-reading. - # TCP_INFO is logged for random fun. - log_format mtail_pub - '$time_iso8601\t$msec\t$server_name\t' - '$ooni_remote_addr\t' # pub/int diff - '$request_completion\t$request_time\t$status\t$bytes_sent\t$body_bytes_sent\t' - '$upstream_cache_status\t$upstream_addr\t$upstream_status\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time\t' - '$scheme\t$server_protocol\t$request_length\t$request_method\t$host\t$request_uri\t' - '$tcpinfo_rtt\t$tcpinfo_rttvar\t' - '$http_referer\t$http_user_agent\t$ooni_request_id'; - - log_format mtail_int - '$time_iso8601\t$msec\t$server_name\t' - '$remote_addr\t' # pub/int diff - '$request_completion\t$request_time\t$status\t$bytes_sent\t$body_bytes_sent\t' - '$upstream_cache_status\t$upstream_addr\t$upstream_status\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time\t' - '$scheme\t$server_protocol\t$request_length\t$request_method\t$host\t$request_uri\t' - '$tcpinfo_rtt\t$tcpinfo_rttvar\t' - '$http_referer\t$http_user_agent\t$ooni_request_id'; - - log_format oolog '$ooni_remote_addr - $remote_user [$time_local] ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" "$host"'; - - log_format oolog_mtail '$time_iso8601\t$msec\t$server_name\t' - '$ooni_remote_addr\t' # pub/int diff - '$request_completion\t$request_time\t$status\t$bytes_sent\t$body_bytes_sent\t' - '$upstream_cache_status\t$upstream_addr\t$upstream_status\t$upstream_connect_time\t$upstream_header_time\t$upstream_response_time\t' - '$scheme\t$server_protocol\t$request_length\t$request_method\t$host\t$request_uri\t' - '$tcpinfo_rtt\t$tcpinfo_rttvar\t' - '$http_referer\t$http_user_agent\t$ooni_request_id'; - - access_log /var/log/nginx/access.log mtail_int; - - sendfile on; - tcp_nopush on; # TCP_CORK HTTP headers with sendfile() body into single packet - - keepalive_timeout 120 120; # Firefox has 115s, http://kb.mozillazine.org/Network.http.keep-alive.timeout - - server_tokens off; - - # SSL based on https://wiki.mozilla.org/Security/Server_Side_TLS (doc v4.1) - ssl_session_timeout 1d; - ssl_session_cache shared:GLOBAL:1m; # 1m of cache is ~4000 sessions - ssl_session_tickets off; # needs accurate key rotation - ssl_dhparam /etc/nginx/ffdhe2048_dhparam.pem; # https://tools.ietf.org/html/rfc7919 - ssl_prefer_server_ciphers on; - #TODO: ssl_stapling on; # needs `resolver` or `ssl_stapling_file` - #TODO: ssl_stapling_verify on; # needs `ssl_trusted_certificate` - #TODO: resolver ; - # Define in server{} - # - include /etc/nginx/ssl_modern.conf | /etc/nginx/ssl_intermediate.conf - # - ssl_certificate /etc/letsencrypt/live/example.org/fullchain.pem; - # - ssl_certificate_key /etc/letsencrypt/live/example.org/privkey.pem - # - ssl_trusted_certificate /etc/letsencrypt/live/example.org/chain.pem; # for ssl_stapling_verify - # - add_header Strict-Transport-Security max-age=15768000; # HSTS (15768000 seconds = 6 months) - ### - - gzip on; - gzip_types text/html text/plain text/css text/xml text/javascript application/x-javascript application/json application/xml; # default is only `text/html` - gzip_disable "msie6"; - #gzip_proxied any; - - # Host, X-Real-IP, X-Forwarded-For, X-Forwarded-Proto are from - # file /etc/nginx/proxy_params from nginx-common package - # NB: adding `proxy_set_header` in another location overwrites whole set! - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Request-ID $ooni_request_id; - - include /etc/nginx/conf.d/*.conf; - include /etc/nginx/sites-enabled/*; + + # Basic Settings + + sendfile on; + tcp_nopush on; # TCP_CORK HTTP headers with sendfile() body into single packet + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Logging Settings + + # anonymize ipaddr + map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; + } + + # log anonymized ipaddr and caching status + log_format ooni_nginx_fmt '$remote_addr_anon $upstream_cache_status [$time_local] ' + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'; + + access_log syslog:server=unix:/dev/log ooni_nginx_fmt; + error_log syslog:server=unix:/dev/log; + + # Gzip Settings + + gzip on; + + # gzip_vary on; + # gzip_proxied any; + # gzip_comp_level 6; + # gzip_buffers 16 8k; + # gzip_http_version 1.1; + # gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript; + + # Virtual Host Configs + + include /etc/nginx/conf.d/*.conf; + include /etc/nginx/sites-enabled/*; } diff --git a/ansible/roles/ooni-backend/handlers/main.yml b/ansible/roles/ooni-backend/handlers/main.yml new file mode 100644 index 00000000..84d0f4f1 --- /dev/null +++ b/ansible/roles/ooni-backend/handlers/main.yml @@ -0,0 +1,6 @@ +--- +- name: reload nftables + service: name=nftables state=reloaded + +- name: restart clickhouse + service: name=clickhouse-server state=restarted diff --git a/ansible/roles/ooni-backend/meta/main.yml b/ansible/roles/ooni-backend/meta/main.yml new file mode 100644 index 00000000..c82f9e2d --- /dev/null +++ b/ansible/roles/ooni-backend/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: nftables diff --git a/ansible/roles/ooni-backend/tasks/main.yml b/ansible/roles/ooni-backend/tasks/main.yml new file mode 100644 index 00000000..a6ee12d6 --- /dev/null +++ b/ansible/roles/ooni-backend/tasks/main.yml @@ -0,0 +1,697 @@ +--- + +## API ## + +- name: install API if not present + # do not update package if present + tags: api + apt: + cache_valid_time: '{{ apt_cache_valid_time }}' + name: ooni-api + state: present + update_cache: yes + +- name: create Nginx cache dir + file: + path: /var/cache/nginx/ooni-api + state: directory + +- name: configure test api + when: inventory_hostname == 'backend-hel.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: [] + # bucket_name and collector_id must match the uploader + collector_id: 2 + bucket_name: ooni-data-eu-fra-test + github_push_repo: "ooni-bot/test-lists" + github_origin_repo: "ooni/test-lists" + login_base_url: "https://test-lists.test.ooni.org/login" + pg_uri: "" + clickhouse_url: clickhouse://api:api@localhost/default + # mail_smtp_password: "DISABLED" + # jwt_encryption_key and account_id_hashing_key are taken from the vault + +- name: configure backend-fsn api + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api + template: + src: api.conf + dest: /etc/ooni/api.conf + owner: ooniapi + group: ooniapi + mode: 0640 + vars: + collectors: ['backend-fsn.ooni.org'] + # bucket_name and collector_id must match the uploader + collector_id: 1 + bucket_name: ooni-data-eu-fra + github_push_repo: "ooni/test-lists" + github_origin_repo: "citizenlab/test-lists" + login_base_url: "https://test-lists.ooni.org/login" + pg_uri: "" + clickhouse_url: clickhouse://api:api@localhost/default + base_url: "https://api.ooni.io" + +- name: create Psiphon conffile + tags: api + copy: + content: "{{ psiphon_config }}" + dest: /etc/ooni/psiphon_config.json + +- name: Write Tor targets conffile + tags: api + template: + src: tor_targets.json + dest: /etc/ooni/tor_targets.json + +- name: configure api uploader using test bucket + when: inventory_hostname == 'backend-hel.ooni.org' + tags: api + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + vars: + # bucket_name and collector_id must match the API + bucket_name: ooni-data-eu-fra-test + collector_id: 2 + +- name: configure FSN api uploader using PROD bucket + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api + template: + src: templates/api-uploader.conf + dest: /etc/ooni/api-uploader.conf + vars: + # bucket_name and collector_id must match the API + bucket_name: ooni-data-eu-fra + collector_id: 1 + +## Haproxy and nginx ## + +- name: Overwrite API nginx test conf + when: inventory_hostname == 'backend-hel.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-test.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: install haproxy if not present + when: inventory_hostname in ('backend-hel.ooni.org') + tags: webserv + apt: + cache_valid_time: 86400 + name: haproxy + state: present + +- name: Deploy haproxy conf + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, webserv + template: + src: templates/haproxy.cfg + dest: /etc/haproxy/haproxy.cfg + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: Delete old files + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, webserv + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: + - /etc/nginx/sites-enabled/00-letsencrypt-http + - /etc/nginx/sites-enabled/deb_ooni_org + - /etc/nginx/sites-enabled/deb_ooni_org_http + +- name: Deploy dehydrated conf + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated.config + dest: /etc/dehydrated/config + mode: 0755 + owner: root + +- name: Deploy dehydrated conf + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated.config + dest: /etc/dehydrated/config + mode: 0755 + owner: root + +- name: Deploy dehydrated haproxy hook + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, webserv + template: + src: templates/dehydrated_haproxy_hook.sh + dest: /etc/dehydrated/haproxy_hook.sh + mode: 0755 + owner: root + +- name: Overwrite API nginx FSN conf + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: api, webserv + template: + src: templates/nginx-api-fsn.conf + dest: /etc/nginx/sites-available/ooni-api.conf + mode: 0755 + owner: root + vars: + # Uses dehydrated + certpath: /var/lib/dehydrated/certs/ + +- name: Deploy API gunicorn conf + tags: api + template: + src: api.gunicorn.py + dest: /etc/ooni/api.gunicorn.py + owner: ooniapi + group: ooniapi + mode: 0640 + +- name: Create symlink for API nginx conf + tags: api + file: + src=/etc/nginx/sites-available/ooni-api.conf + dest=/etc/nginx/sites-enabled/ooni-api.conf + state=link + +- name: Configure deb.ooni.org forwarder on FSN host + when: inventory_hostname in ('backend-fsn.ooni.org', ) + tags: deb_ooni_org + # Uses dehydrated + template: + src: deb_ooni_org.nginx.conf + dest: /etc/nginx/sites-enabled/deb_ooni_org + +- name: Configure deb-ci.ooni.org forwarder on test host + when: inventory_hostname == 'backend-hel.ooni.org' + tags: deb_ooni_org + blockinfile: + path: /etc/nginx/sites-enabled/deb_ooni_org_http + create: yes + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + server { + listen 80; + server_name deb-ci.ooni.org; + location / { + proxy_pass https://ooni-internal-deb.s3.eu-central-1.amazonaws.com/; + } + } + +- name: create badges dir + tags: api + file: + path: /var/www/package_badges/ + state: directory + +- name: Safely reload Nginx + # TODO remove restart after transition to haproxy + tags: api, deb_ooni_org, webserv + shell: nginx -t && systemctl reload nginx + +- name: Restart Nginx + tags: webserv + shell: nginx -t && systemctl restart nginx + +- name: Restart haproxy + # reload is not enough + when: inventory_hostname in ('backend-hel.ooni.org') + tags: api, deb_ooni_org, webserv + shell: systemctl restart haproxy + +- name: allow incoming TCP connections to API + tags: api + blockinfile: + path: /etc/ooni/nftables/tcp/443.nft + create: yes + block: | + add rule inet filter input tcp dport 443 counter accept comment "incoming HTTPS" + +- name: allow incoming TCP connections to haproxy metrics + tags: webserv + template: + src: 444.nft + dest: /etc/ooni/nftables/tcp/444.nft + +#- name: reload nftables service +# tags: api +# systemd: +# name: nftables.service +# state: reloaded + +- name: reload nftables service + tags: api, webserv + shell: systemctl reload nftables.service + + +## Fastpath ## + +- name: install fastpath if not present + # do not update package if present + when: inventory_hostname != 'backend-fsn.ooni.org' + tags: fastpath + apt: + cache_valid_time: 86400 + name: fastpath + state: present + +- name: configure fastpath on test + when: inventory_hostname == 'backend-hel.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: clickhouse://fastpath:fastpath@localhost/default + +- name: configure fastpath on FSN + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: fastpath + template: + src: fastpath.conf + dest: /etc/ooni/fastpath.conf + owner: fastpath + group: fastpath + mode: 0640 + vars: + clickhouse_url: clickhouse://fastpath:fastpath@localhost/default + + + +## Event detector ## + +#- name: install detector +# tags: detector +# apt: +# cache_valid_time: 86400 +# name: detector +# +#- name: configure detector +# tags: detector +# blockinfile: +# path: /etc/ooni/detector.conf +# create: yes +# block: | +# # Managed by ansible, see roles/ooni-backend/tasks/main.yml + + +## Analysis daemon ## + +- name: install analysis + # do not update package if present + when: inventory_hostname != 'backend-fsn.ooni.org' + tags: analysis + apt: + cache_valid_time: 86400 + name: analysis=1.4~pr408-209 + force: True + state: present + +- name: configure analysis + tags: analysis-conf + template: + src: analysis.conf + dest: /etc/ooni/analysis.conf + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + + +## Test helper rotation ## + +- name: configure test helper rotation + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + blockinfile: + path: /etc/ooni/rotation.conf + create: yes + mode: 0400 + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + [DEFAULT] + # Digital Ocean token + token = {{ digital_ocean_token }} + active_droplets_count = 4 + size_slug = s-1vcpu-1gb + image_name = debian-11-x64 + draining_time_minutes = 1440 + dns_zone = th.ooni.org + +- name: configure test helper rotation certbot + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + blockinfile: + path: /etc/ooni/certbot-digitalocean + create: yes + mode: 0400 + block: | + # Managed by ansible, see roles/ooni-backend/tasks/main.yml + dns_digitalocean_token = {{ digital_ocean_token }} + +- name: configure test helper rotation setup script + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + template: + src: rotation_setup.sh + dest: /etc/ooni/rotation_setup.sh + +- name: create test helper rotation nginx template + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + template: + src: rotation_nginx_conf + dest: /etc/ooni/rotation_nginx_conf + +- name: generate test helper rotation SSH keypair + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + openssh_keypair: + path: /etc/ooni/testhelper_ssh_key + owner: root + group: root + mode: 0400 + type: ed25519 + register: pubkey + +- name: print SSH pubkey + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + debug: msg={{ pubkey.public_key }} + +- name: Enable and start rotation service + tags: rotation + when: inventory_hostname == 'backend-fsn.ooni.org' + systemd: + daemon_reload: yes + enabled: yes + name: ooni-rotation.timer + state: started + + +## Tor daemon and onion service ## + +## TODO(decfox): get rid of this? +- name: configure tor onion service hostname + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + blockinfile: + path: /var/lib/tor/ooni_onion_service/hostname + create: yes + owner: debian-tor + group: debian-tor + mode: 0644 + block: guegdifjy7bjpequ.onion + +- name: configure tor onion service private_key + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + blockinfile: + path: /var/lib/tor/ooni_onion_service/private_key + create: yes + owner: debian-tor + group: debian-tor + mode: 0600 + block: "{{ amspg_ooni_org_onion_key }}" + +- name: set tor onion service directory + when: inventory_hostname == 'ams-pg.ooni.org' + tags: tor + shell: | + chown debian-tor:debian-tor /var/lib/tor/ooni_onion_service + chmod 0700 /var/lib/tor/ooni_onion_service + + +# # Clickhouse # # + +- name: install APT HTTPS support + # do not update package if present + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + apt: + cache_valid_time: 86400 + state: present + name: + - apt-transport-https + - ca-certificates + - dirmngr + +- name: install clickhouse keys + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + command: apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754 + +- name: set clickhouse repos + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/apt/sources.list.d/clickhouse.list + create: yes + block: | + deb https://packages.clickhouse.com/deb lts main + +- name: pin clickhouse release train + when: inventory_hostname in ('backend-fsn.ooni.org', ) + tags: clickhouse + blockinfile: + path: /etc/apt/preferences.d/clickhouse-server + create: yes + block: | + Package: clickhouse-server + Pin: version 21.8.12.* + Pin-Priority: 999 + +- name: pin clickhouse release train + when: inventory_hostname in ('backend-hel.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/apt/preferences.d/clickhouse-server + create: yes + block: | + Package: clickhouse-server + Pin: version 23.8.2.* + Pin-Priority: 999 + +- name: install clickhouse on backend-fsn + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + - clickhouse-common-static={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 21.8.12.* + +- name: install clickhouse on backend-hel.ooni.org + when: inventory_hostname == 'backend-hel.ooni.org' + tags: clickhouse + apt: + # refresh cache + cache_valid_time: 0 + name: + - clickhouse-server={{ clickhouse_pkg_ver }} + - clickhouse-client={{ clickhouse_pkg_ver }} + - clickhouse-common-static={{ clickhouse_pkg_ver }} + vars: + clickhouse_pkg_ver: 23.8.2.* + +- name: install clickhouse conf override + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + template: + src: clickhouse_config.xml + dest: /etc/clickhouse-server/config.d/ooni_conf.xml + owner: clickhouse + group: clickhouse + mode: 0400 + notify: restart clickhouse + +- name: allow incoming TCP connections from monitoring to Clickhouse prometheus interface + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/ooni/nftables/tcp/9363.nft + create: yes + block: | + add rule inet filter input ip saddr 5.9.112.244 tcp dport 9363 counter accept comment "clickhouse prometheus from monitoring.ooni.org" + notify: reload nftables + +- name: allow incoming TCP connections from jupiter on monitoring.ooni.org to Clickhouse + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + blockinfile: + path: /etc/ooni/nftables/tcp/9000.nft + create: yes + block: | + add rule inet filter input ip saddr 5.9.112.244 tcp dport 9000 counter accept comment "clickhouse from monitoring.ooni.org" + notify: reload nftables + +- name: Run clickhouse + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + systemd: + name: clickhouse-server.service + state: started + enabled: yes + +## Clickhouse access control ## +# https://clickhouse.com/docs/en/operations/access-rights/#enabling-access-control + +- name: Clickhouse - test admin user - failure is ok to ignore + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse-users + command: clickhouse-client -u admin --password admin -q 'select 1' + ignore_errors: true + register: admin_check + +- name: install tor python3-lxml + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + apt: + cache_valid_time: 86400 + name: python3-lxml + +- name: Clickhouse - set flag + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + # The users.xml file itself needs to be edited for this to work + xml: + path: /etc/clickhouse-server/users.xml + backup: yes + xpath: /clickhouse/users/default/{{ item }} + value: "1" + loop: + - access_management + - named_collection_control + - show_named_collections + - show_named_collections_secrets + register: users_xml + +- name: Clickhouse - restart immediately if needed + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + systemd: + name: clickhouse-server + state: restarted + +- name: Clickhouse - create admin + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + command: clickhouse-client -q "CREATE USER OR REPLACE admin IDENTIFIED WITH sha256_password BY 'admin' HOST LOCAL GRANTEES ANY" + # The server might be still starting: retry as needed + retries: 10 + delay: 5 + register: result + until: result.rc == 0 + +- name: Clickhouse - grant admin rights + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + command: clickhouse-client -q 'GRANT ALL ON *.* TO admin WITH GRANT OPTION' + +- name: Clickhouse - create readonly profile + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + template: + src: clickhouse_readonly.xml + dest: /etc/clickhouse-server/users.d/make_default_readonly.xml + owner: clickhouse + group: clickhouse + mode: 0640 + + #- name: Clickhouse - restore users.xml + # when: admin_check is defined and admin_check is failed + # tags: clickhouse-users + # command: mv {{ users_xml.backup_file }} /etc/clickhouse-server/users.xml + +- name: Clickhouse - restart immediately if needed + when: admin_check is defined and admin_check is failed + tags: clickhouse-users + systemd: + name: clickhouse-server + state: restarted + +- name: Clickhouse - setup users and permissions + tags: clickhouse-users + command: clickhouse-client -u admin --password admin -q "{{ item }}" + loop: + - "CREATE USER OR REPLACE api IDENTIFIED WITH sha256_password BY 'api' HOST LOCAL" + - "GRANT ALL ON *.* TO api" + - "CREATE USER OR REPLACE fastpath IDENTIFIED WITH sha256_password BY 'fastpath' HOST LOCAL" + - "GRANT ALL ON *.* TO fastpath" + +## end of Clickhouse access control ## + + + +- name: Run feeder on backend-hel + when: inventory_hostname == 'backend-hel.ooni.org' + tags: clickhouse + blockinfile: + path: /etc/ooni/clickhouse_feeder.conf + create: yes + block: | + [DEFAULT] + pg_dbuser = readonly + pg_dbhost = localhost + +- name: run feeder on backend-fsn + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: clickhouse + blockinfile: + path: /etc/ooni/clickhouse_feeder.conf + create: yes + block: | + [DEFAULT] + pg_dbuser = readonly + pg_dbhost = backend-hel.ooni.org + +- name: Run feeder + when: inventory_hostname in ('backend-fsn.ooni.org', 'backend-hel.ooni.org') + tags: clickhouse + systemd: + name: ooni-clickhouse-feeder.service + state: started + enabled: yes + +- name: Run DB backup on backend-hel + when: inventory_hostname == 'backend-hel.ooni.org' + tags: dbbackup + template: + src: db-backup.conf + dest: /etc/ooni/db-backup.conf + mode: 0600 + vars: + public_bucket_name: ooni-data-eu-fra-test + +- name: Run DB backup on FSN + when: inventory_hostname == 'backend-fsn.ooni.org' + tags: dbbackup + template: + src: db-backup.conf + dest: /etc/ooni/db-backup.conf + mode: 0600 + vars: + public_bucket_name: ooni-data-eu-fra diff --git a/ansible/roles/ooni-backend/templates/444.nft b/ansible/roles/ooni-backend/templates/444.nft new file mode 100644 index 00000000..03f5106f --- /dev/null +++ b/ansible/roles/ooni-backend/templates/444.nft @@ -0,0 +1,2 @@ +# roles/ooni-backend/templates/444.nft +add rule inet filter input tcp dport 444 counter accept comment "incoming haproxy metrics" diff --git a/ansible/roles/ooni-backend/templates/analysis.conf b/ansible/roles/ooni-backend/templates/analysis.conf new file mode 100644 index 00000000..4df8a8ae --- /dev/null +++ b/ansible/roles/ooni-backend/templates/analysis.conf @@ -0,0 +1,9 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# [s3bucket] +# bucket_name = ooni-data-eu-fra-test +# aws_access_key_id = +# aws_secret_access_key = + +[backup] +# space separated +table_names = citizenlab fastpath jsonl diff --git a/ansible/roles/ooni-backend/templates/api-uploader.conf b/ansible/roles/ooni-backend/templates/api-uploader.conf new file mode 100644 index 00000000..2de0e399 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api-uploader.conf @@ -0,0 +1,9 @@ +# OONI API measurement uploader - Python ini format +# Deployed by ansible, see roles/ooni-backend/templates/api-uploader.conf +[DEFAULT] +# arn:aws:iam::676739448697:user/ooni-pipeline, AWS: OONI Open Data +aws_access_key_id = AKIAJURD7T4DTN5JMJ5Q +aws_secret_access_key = {{ s3_ooni_open_data_access_key }} +bucket_name = {{ bucket_name }} +msmt_spool_dir = /var/lib/ooniapi/measurements +collector_id = {{ collector_id }} diff --git a/ansible/roles/ooni-backend/templates/api.conf b/ansible/roles/ooni-backend/templates/api.conf new file mode 100644 index 00000000..25d1d0c6 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api.conf @@ -0,0 +1,60 @@ +# Deployed by ansible +# See ooni-backend/tasks/main.yml ooni-backend/templates/api.conf +# Syntax: treat it as a Python file, but only uppercase variables are used +COLLECTORS = {{ collectors }} +COLLECTOR_ID = {{ collector_id }} + +# Read-only database access +# The password is already made public +DATABASE_URI_RO = "{{ pg_uri }}" + +DATABASE_STATEMENT_TIMEOUT = 20 + +{% if clickhouse_url|length %} +USE_CLICKHOUSE = True +{% else %} +USE_CLICKHOUSE = False +{% endif %} + +CLICKHOUSE_URL = "{{ clickhouse_url }}" + + +BASE_URL = "{{ base_url }}" + +AUTOCLAVED_BASE_URL = "http://datacollector.infra.ooni.io/ooni-public/autoclaved/" +CENTRIFUGATION_BASE_URL = "http://datacollector.infra.ooni.io/ooni-public/centrifugation/" + +S3_ACCESS_KEY_ID = "AKIAJURD7T4DTN5JMJ5Q" +S3_BUCKET_NAME = "{{ bucket_name }}" +S3_SECRET_ACCESS_KEY = "CHANGEME" +S3_SESSION_TOKEN = "CHANGEME" +S3_ENDPOINT_URL = "CHANGEME" + +PSIPHON_CONFFILE = "/etc/ooni/psiphon_config.json" +TOR_TARGETS_CONFFILE = "/etc/ooni/tor_targets.json" + +JWT_ENCRYPTION_KEY = "{{ jwt_encryption_key }}" +ACCOUNT_ID_HASHING_KEY = "{{ account_id_hashing_key }}" + +SESSION_EXPIRY_DAYS = 180 +LOGIN_EXPIRY_DAYS = 365 + +# Registration email delivery +MAIL_SERVER = "mail.riseup.net" +MAIL_PORT = 465 +MAIL_USE_SSL = True +MAIL_USERNAME = "ooni-mailer" +MAIL_PASSWORD = "{{ mail_smtp_password }}" +MAIL_SOURCE_ADDRESS = "contact@ooni.org" +LOGIN_BASE_URL = "{{ login_base_url }}" + +GITHUB_WORKDIR = "/var/lib/ooniapi/citizenlab" +GITHUB_TOKEN = "{{ github_token }}" +GITHUB_USER = "ooni-bot" +GITHUB_ORIGIN_REPO = "{{ github_origin_repo }}" +GITHUB_PUSH_REPO = "{{ github_push_repo }}" + +# Measurement spool directory +MSMT_SPOOL_DIR = "/var/lib/ooniapi/measurements" +GEOIP_ASN_DB = "/var/lib/ooniapi/asn.mmdb" +GEOIP_CC_DB = "/var/lib/ooniapi/cc.mmdb" diff --git a/ansible/roles/ooni-backend/templates/api.gunicorn.py b/ansible/roles/ooni-backend/templates/api.gunicorn.py new file mode 100644 index 00000000..f86b6f67 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/api.gunicorn.py @@ -0,0 +1,12 @@ +# Gunicorn configuration file +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# and templates/api.gunicorn.py + +workers = 12 + +loglevel = "info" +proc_name = "ooni-api" +reuse_port = True +# Disabled statsd: https://github.com/benoitc/gunicorn/issues/2843 +#statsd_host = "127.0.0.1:8125" +#statsd_prefix = "ooni-api" diff --git a/ansible/roles/ooni-backend/templates/clickhouse_config.xml b/ansible/roles/ooni-backend/templates/clickhouse_config.xml new file mode 100644 index 00000000..548c2a81 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_config.xml @@ -0,0 +1,41 @@ + + + + + information + + +{% if inventory_hostname == 'backend-fsn.ooni.org' %} + production + 20100100100 + +{% else %} + {{ inventory_hostname.replace(".ooni.org", "") }} +{% endif %} + +{% if inventory_hostname == 'backend-hel.ooni.org' %} + 500100100 + 3100100100 +{% endif %} + + + 0.0.0.0 + + + + + + + + + /metrics + 9363 + true + true + true + true + + diff --git a/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml b/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml new file mode 100644 index 00000000..73645616 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_readonly.xml @@ -0,0 +1,9 @@ + + + + + + readonly + + + diff --git a/ansible/roles/ooni-backend/templates/clickhouse_users.xml b/ansible/roles/ooni-backend/templates/clickhouse_users.xml new file mode 100644 index 00000000..49fd011a --- /dev/null +++ b/ansible/roles/ooni-backend/templates/clickhouse_users.xml @@ -0,0 +1,31 @@ + + + + + + + 1 + + + + + + + readonly + + 0.0.0.0 + + + + + + {{ clickhouse_writer_password|hash('sha256') }} + + 127.0.0.1 + + + + + + + diff --git a/ansible/roles/ooni-backend/templates/db-backup.conf b/ansible/roles/ooni-backend/templates/db-backup.conf new file mode 100644 index 00000000..4302f0ec --- /dev/null +++ b/ansible/roles/ooni-backend/templates/db-backup.conf @@ -0,0 +1,17 @@ +{ + "ver": 0, + "action": "export", + "public_aws_access_key_id": "AKIAJURD7T4DTN5JMJ5Q", + "public_aws_secret_access_key": "{{ s3_ooni_open_data_access_key }}", + "public_bucket_name": "{{ public_bucket_name }}", + "clickhouse_url": "clickhouse://localhost/default", + "__description": "tables can be backed up as: ignore, full, incremental, partition", + "backup_tables": { + "citizenlab": "ignore", + "fastpath": "ignore", + "jsonl": "ignore", + "msmt_feedback": "ignore", + "test_helper_instances": "ignore", + "url_priorities": "ignore" + } +} diff --git a/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf new file mode 100644 index 00000000..c069fd55 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/deb_ooni_org.nginx.conf @@ -0,0 +1,64 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# log anonymized ipaddr +log_format deb_ooni_org_logfmt '$remote_addr_anon [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + listen 80; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + resolver 127.0.0.1; + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info deb_ooni_org_logfmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} diff --git a/ansible/roles/ooni-backend/templates/dehydrated.config b/ansible/roles/ooni-backend/templates/dehydrated.config new file mode 100644 index 00000000..7a0293a2 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/dehydrated.config @@ -0,0 +1,7 @@ +# Deployed by ansible +# See roles/ooni-backend/templates/dehydrated.config +CONFIG_D=/etc/dehydrated/conf.d +BASEDIR=/var/lib/dehydrated +WELLKNOWN="${BASEDIR}/acme-challenges" +DOMAINS_TXT="/etc/dehydrated/domains.txt" +HOOK="/etc/dehydrated/haproxy_hook.sh" diff --git a/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh b/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh new file mode 100644 index 00000000..0e5b41f3 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/dehydrated_haproxy_hook.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Deployed by ansible +# See roles/ooni-backend/templates/dehydrated_haproxy_hook.sh +# +# Deploys chained privkey and certificates for haproxy +# Reloads haproxy as needed + +deploy_cert() { + local DOMAIN="${1}" KEYFILE="${2}" CERTFILE="${3}" FULLCHAINFILE="${4}" CHAINFILE="${5}" TIMESTAMP="${6}" + # Called once for each certificate + # /var/lib/dehydrated/certs/backend-hel.ooni.org/privkey.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/cert.pem /var/lib/dehydrated/certs/backend-hel.ooni.org/fullchain.pem > /var/lib/dehydrated/certs/backend-hel.ooni.org/haproxy.pem + # cp "${KEYFILE}" "${FULLCHAINFILE}" /etc/nginx/ssl/; chown -R nginx: /etc/nginx/ssl + logger "deploy_cert hook reading ${KEYFILE} ${CERTFILE} ${FULLCHAINFILE}" + cat "${KEYFILE}" "${CERTFILE}" "${FULLCHAINFILE}" > "${KEYFILE}.haproxy" + logger "deploy_cert reloading haproxy" + systemctl reload haproxy.service +} + +HANDLER="$1"; shift +if [[ "${HANDLER}" =~ ^(deploy_cert)$ ]]; then + "$HANDLER" "$@" +fi diff --git a/ansible/roles/ooni-backend/templates/fastpath.conf b/ansible/roles/ooni-backend/templates/fastpath.conf new file mode 100644 index 00000000..031f49a0 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/fastpath.conf @@ -0,0 +1,15 @@ +# See roles/ooni-backend/tasks/main.yml +[DEFAULT] +collectors = localhost +{% if psql_uri is defined %} +# The password is already made public +db_uri = {{ psql_uri }} +{% else %} +db_uri = +{% endif %} +clickhouse_url = {{ clickhouse_url }} + +# S3 access credentials +# Currently unused +s3_access_key = +s3_secret_key = diff --git a/ansible/roles/ooni-backend/templates/haproxy.cfg b/ansible/roles/ooni-backend/templates/haproxy.cfg new file mode 100644 index 00000000..025a4fc2 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/haproxy.cfg @@ -0,0 +1,122 @@ +## Deployed by ansible, see roles/ooni-backend/templates/haproxy.cfg + +# Proxies to: +# - local nginx +# - remote test helpers +# See http://interactive.blockdiag.com/?compression=deflate&src=eJyFjjELwjAQhXd_xeFuEdpBEAURBwfBXSSk6ZkEr7mSZGgR_7tNXdoiuD2--7j3SmL1rKzU8FoAFEUOqz0Y2XhuuxSHICKLiCEKg9Sg3_bmSHHaujaxISRyuJ7hRrJEgh0slVTGOr28Txz2yvQvvYw44R617XGXMTubWU7HzXq26kfl8XISykgidBphVP-whLPuOtRRhIaZ_ogVlt8d7PVYDXkS3x_pgmPP + +global + log /dev/log local0 info alert + log /dev/log local1 notice alert + chroot /var/lib/haproxy + stats socket /run/haproxy/admin.sock mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + + # Default SSL material locations + ca-base /etc/ssl/certs + crt-base /etc/ssl/private + + # See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate + ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384 + ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256 + ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + errorfile 400 /etc/haproxy/errors/400.http + errorfile 403 /etc/haproxy/errors/403.http + errorfile 408 /etc/haproxy/errors/408.http + errorfile 500 /etc/haproxy/errors/500.http + errorfile 502 /etc/haproxy/errors/502.http + errorfile 503 /etc/haproxy/errors/503.http + errorfile 504 /etc/haproxy/errors/504.http + + log-format "%[var(txn.src_ipaddr_masked)] %ft > %b > %s %TR/%Tw/%Tc/%Tr/%Ta %ST %B %CC %CS %tsc %ac/%fc/%bc/%sc/%rc %sq/%bq %hr %hs %{+Q}r" + +frontend haproxy_metrics + # Metrics exposed on TLS port 444 + # File generated by /etc/dehydrated/haproxy_hook.sh + bind :444 ssl crt /var/lib/dehydrated/certs/"{{ inventory_hostname }}"/privkey.pem.haproxy + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # /__haproxy_stats stats page + stats enable + stats uri /__haproxy_stats + stats refresh 5s + + # /__haproxy_prom_metrics prometheus metrics + http-request use-service prometheus-exporter if { path /__haproxy_prom_metrics } + + +frontend public_tls + # TLS on port 443 + # File generated by /etc/dehydrated/haproxy_hook.sh + bind :443 ssl crt /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem.haproxy + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # test helpers + default_backend lb_test_helpers + + # deb.ooni.org + acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org + use_backend deb_ooni_org if ACL_deb_ooni_org + + # Nginx + use_backend nginx if !{ path / } || !{ method POST } + + +frontend public_80 + # Forwarded to Nginx for ACME and deb.ooni.org + bind :80 + + http-request set-var(txn.src_ipaddr_masked) src,ipmask(24,64) + + # ACME + use_backend nginx if { path_beg /.well-known/acme-challenge } + + # deb.ooni.org + acl ACL_deb_ooni_org hdr(host) -i deb.ooni.org + use_backend deb_ooni_org if ACL_deb_ooni_org + + + +backend nginx + # Local Nginx is in front of the API and more. See diagram. + default-server check + option forwardfor + #option httpchk GET / + # forward to local nginx + server nginx localhost:17744 + + +backend lb_test_helpers + # Remote testn helpers + default-server check + option forwardfor + http-check send meth POST uri / hdr Content-Type application/json body "{}" + http-check send-state + http-check comment "TH POST with empty JSON" + + server th0 0.th.ooni.org:443 ssl verify none + server th1 1.th.ooni.org:443 ssl verify none + server th2 2.th.ooni.org:443 ssl verify none + server th3 3.th.ooni.org:443 ssl verify none + #option httpchk + + +backend deb_ooni_org + #default-server check + option forwardfor + server s3-ooni-deb ooni-deb.s3.eu-central-1.amazonaws.com ssl verify none + diff --git a/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf b/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf new file mode 100644 index 00000000..4e3cf934 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-ams-pg.conf @@ -0,0 +1,297 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-ams-pg.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + # HSTS (ngx_http_headers_module is required) (63072000 seconds) + add_header Strict-Transport-Security "max-age=63072000" always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + # Registry + # Should match: + # - /api/v1/login + # - /api/v1/register + # - /api/v1/update + location ~^/api/v1/(login|register|update) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://registry.ooni.io:443; + } + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + } + + # Orchestrate + # Should match: + # - /api/v1/test-list + location ~^/api/v1/(test-list|urls) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://orchestrate.ooni.io:443; + } + + # Web Connectivity Test Helper + # Should match: + # - / + # - /status + # + # The fact that it responds to / means that we may have to differentiate + # via the Host record. + # TODO We should check if clients will respect a suffix added to by the + # bouncer in the returned field, otherwise new clients should use another + # form + location ~^/web-connectivity/(status) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://wcth.ooni.io; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + location /metrics { + return 200 ''; + } + + # Expose (only) Netdata badges + location ~ ^/netdata/badge { + rewrite ^/netdata/badge /api/v1/badge.svg break; + proxy_pass http://127.0.0.1:19999; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Expose package version badges + location /package_badges { + root /var/www; + add_header Pragma "no-cache"; + add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; + } + + # Temporary redirection to backend-FSN + location ~ ^/api/v1/(aggregation|measurements|raw_measurement|measurement_meta) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(asn_by_month|countries|countries_by_month|check_report_id|country_overview|global_overview|global_overview_by_month|im_networks|im_stats|network_stats) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(test_coverage|website_networks|website_stats|website_urls|vanilla_tor_stats|test_names) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/_/circumvention_stats_by_country { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = / { + # match "/" strictly, not as a prefix + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/static/ { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + # open and close reports, submit msmt + location ~ ^/report/ { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Auth, URL sumbission, URL priorities + location ~ ^/api/v1/(url-submission|get_account_role|set_account_role|set_session_expunge|user_login|user_register|user_logout) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/api/_/(url-priorities|account_metadata) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + location ~ ^/api/v1/(collectors|test-helpers|torsf_stats) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location ~ ^/(robots.txt|files) { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/tor-targets { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/urls { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /bouncer/net-tests { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + location = /api/v1/test-list/psiphon-config { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + #location ~ ^/api/_/(test_names) { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + ## /files* tree + #location ~ ^/files { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + #location ~ ^/(health) { + # proxy_pass https://backend-fsn.ooni.org; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + #} + + # Temporary redirect + location = /api/v1/check-in { + proxy_pass https://backend-fsn.ooni.org; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # new API + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + + gzip_comp_level 6; + gzip_min_length 1240; + gzip_proxied any; + gzip_types *; + gzip_vary on; + } +} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf new file mode 100644 index 00000000..9d6e1451 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-fsn.conf @@ -0,0 +1,260 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-fsn.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# anonymize forwarded ipaddr +map $http_x_forwarded_for $remote_fwd_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + + listen 443 ssl http2 default_server; + listen [::]:443 ssl http2 default_server; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + gzip_types text/plain application/xml application/json; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + ssl_certificate {{ certpath }}{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key {{ certpath }}{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate {{ certpath }}{{ inventory_hostname }}/chain.pem; # for ssl_stapling_verify + + # Use the intermediate configuration to support legacy probes + # https://ssl-config.mozilla.org/#server=nginx&version=1.14.2&config=intermediate&openssl=1.1.1d&guideline=5.6 + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # OCSP stapling + ssl_stapling on; + ssl_stapling_verify on; + + # verify chain of trust of OCSP response using Root CA and Intermediate certs + #ssl_trusted_certificate /path/to/root_CA_cert_plus_intermediates; + + resolver 127.0.0.1; + + # Registry + # Should match: + # - /api/v1/login + # - /api/v1/register + # - /api/v1/update + location ~^/api/v1/(login|register|update) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://registry.ooni.io:443; + } + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Orchestrate + # Should match: + # - /api/v1/test-list + location ~^/api/v1/(test-list|urls) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://orchestrate.ooni.io:443; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + } + + # Web Connectivity Test Helper + # Should match: + # - / + # - /status + # + # The fact that it responds to / means that we may have to differentiate + # via the Host record. + # TODO We should check if clients will respect a suffix added to by the + # bouncer in the returned field, otherwise new clients should use another + # form + location ~^/web-connectivity/(status) { + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + + proxy_pass https://wcth.ooni.io; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + location /metrics { + return 200 ''; + } + + # Expose event detector RSS/atom feeds + location ~ ^/detector { + root /var/lib; + default_type application/xml; + } + + # Expose (only) Netdata badges + location ~ ^/netdata/badge { + rewrite ^/netdata/badge /api/v1/badge.svg break; + proxy_pass http://127.0.0.1:19999; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + } + + # Expose package version badges + location /package_badges { + root /var/www; + add_header Pragma "no-cache"; + add_header Cache-Control "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # 2022-09-01 20:08 CEST temporarily block a bot scraping /files/download/* + location ~^/files/download/ { + return 301 https://explorer.ooni.org/; + } + + # new API + location / { + + # Protect /apidocs invoked with url= and/or urls= args + if ($uri ~ "^/apidocs") { set $block_apidocs X; } + if ($args ~ "url=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($args ~ "urls=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($block_apidocs ~ "XY") { return 403; } # nested "if" are not supported + + deny 216.244.66.0/24; # DotBot/1.2 + deny 114.119.128.0/19; # PetalBot + allow all; + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + + # match test-helper POST to / and forward traffic to a TH + if ($request_uri = "/") { set $forward_to_th "YE"; } + if ($request_method = POST) { set $forward_to_th "${forward_to_th}S"; } + if ($forward_to_th = "YES") { + proxy_pass https://0.th.ooni.org; + } + + set $external_remote_addr $remote_addr; + if ($remote_addr = "188.166.93.143") { + # If remote_addr is ams-pg-test trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr = "142.93.237.101") { + # If remote_addr is ams-pg trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + proxy_set_header X-Real-IP $external_remote_addr; + + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + } +} + +# Used by Netdata to monitor Nginx +server { + listen 127.0.0.1:80; + server_name localhost; + location = /stub_status { + stub_status; + } +} diff --git a/ansible/roles/ooni-backend/templates/nginx-api-test.conf b/ansible/roles/ooni-backend/templates/nginx-api-test.conf new file mode 100644 index 00000000..092d40db --- /dev/null +++ b/ansible/roles/ooni-backend/templates/nginx-api-test.conf @@ -0,0 +1,157 @@ +# Managed by ansible +# roles/ooni-backend/templates/nginx-api-test.conf + +# Use 2-level cache, 20MB of RAM + 5GB on disk, +proxy_cache_path /var/cache/nginx/ooni-api levels=1:2 keys_zone=apicache:100M + max_size=5g inactive=24h use_temp_path=off; + +# anonymize ipaddr +map $remote_addr $remote_addr_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + +# anonymize forwarded ipaddr +map $http_x_forwarded_for $remote_fwd_anon { + ~(?P\d+\.\d+\.\d+)\. $ip.0; + ~(?P[^:]+:[^:]+): $ip::; + default 0.0.0.0; +} + + +# log anonymized ipaddr and caching status +log_format ooni_api_fmt '$remote_addr_anon $remote_fwd_anon $upstream_cache_status [$time_local] ' + '"$request" $status snt:$body_bytes_sent rt:$request_time uprt:$upstream_response_time "$http_referer" "$http_user_agent"'; + +server { + # TODO(bassosimone): we need support for cleartext HTTP to make sure that requests + # over Tor correctly land to the proper backend. We are listening on this custom port + # and we are configuring Tor such that it routes traffic to this port. + listen 127.0.0.1:17744; + server_name _; + access_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,tag=ooniapi,severity=info; + gzip on; + gzip_types text/plain application/xml application/json; + + # TODO: we could use different client_max_body_size and SSL configurations for probe service paths + # and everyhing else + client_max_body_size 200M; # for measurement POST + + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + + # use systemd-resolved + resolver 127.0.0.53; + + # Selectively route test-list/urls to the API + location ~^/api/v1/test-list/urls { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + location /whoami { + return 200 "{{ inventory_hostname }}"; + } + + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + + # 2022-09-01 20:08 CEST temporarily block a bot scraping /files/download/* + location ~^/files/download/ { + return 301 https://explorer.ooni.org/; + } + + # new API + location / { + + # Protect /apidocs invoked with url= and/or urls= args + if ($uri ~ "^/apidocs") { set $block_apidocs X; } + if ($args ~ "url=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($args ~ "urls=" ) { set $block_apidocs "${block_apidocs}Y"; } + if ($block_apidocs ~ "XY") { return 403; } # nested "if" are not supported + + deny 216.244.66.0/24; # DotBot/1.2 + deny 114.119.128.0/19; # PetalBot + allow all; + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + + set $external_remote_addr $remote_addr; + if ($remote_addr = "188.166.93.143") { + # If remote_addr is ams-pg-test trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + if ($remote_addr = "142.93.237.101") { + # If remote_addr is ams-pg trust the X-Real-IP header + set $external_remote_addr $http_x_real_ip; + } + proxy_set_header X-Real-IP $external_remote_addr; + + proxy_cache apicache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + proxy_cache_methods HEAD GET; + # Cache only 200, 301, and 302 by default and for very short. + # Overridden by the API using the Expires header + proxy_cache_valid 200 301 302 10s; + proxy_cache_valid any 0; + add_header x-cache-status $upstream_cache_status; + add_header X-Cache-Status $upstream_cache_status; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Frame-Options DENY always; + add_header X-Content-Type-Options nosniff always; + } + + # Expose the measurement spool directory + location /measurement_spool/ { + alias /var/lib/ooniapi/measurements/incoming/; + autoindex off; + sendfile on; + tcp_nopush on; + if_modified_since off; + expires off; + etag off; + } +} + +server { + # Forward deb.ooni.org to S3 + listen 17744; + server_name deb.ooni.org; + access_log syslog:server=unix:/dev/log,severity=info ooni_api_fmt; + error_log syslog:server=unix:/dev/log,severity=info; + gzip on; + resolver 127.0.0.53; + # Serve ACME challenge from disk + location ^~ /.well-known/acme-challenge { + alias /var/lib/dehydrated/acme-challenges; + } + location / { + proxy_pass https://ooni-deb.s3.eu-central-1.amazonaws.com/; + } +} diff --git a/ansible/roles/ooni-backend/templates/rotation_nginx_conf b/ansible/roles/ooni-backend/templates/rotation_nginx_conf new file mode 100644 index 00000000..63255e51 --- /dev/null +++ b/ansible/roles/ooni-backend/templates/rotation_nginx_conf @@ -0,0 +1,70 @@ +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# and roles/ooni-backend/templates/rotation_nginx_conf +# Deployed by rotation tool to the test-helper hosts +proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=thcache:100M + max_size=5g inactive=24h use_temp_path=off; + +server { + listen 443 ssl http2; + listen [::]:443 ssl http2; + server_name _; + gzip on; + ssl_certificate /etc/ssl/private/th_fullchain.pem; + ssl_certificate_key /etc/ssl/private/th_privkey.pem; + ssl_session_timeout 5m; + ssl_session_cache shared:MozSSL:30m; + ssl_session_tickets off; + ssl_protocols TLSv1.2 TLSv1.3; + ssl_ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384; + ssl_prefer_server_ciphers off; + add_header Strict-Transport-Security "max-age=63072000" always; + ssl_stapling on; + ssl_stapling_verify on; + resolver 127.0.0.1; + # local test helper + location / { + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 900; + proxy_pass http://127.0.0.1:8080; + + proxy_cache thcache; + proxy_cache_min_uses 1; + proxy_cache_lock on; + proxy_cache_lock_timeout 30; + proxy_cache_lock_age 30; + proxy_cache_use_stale error timeout invalid_header updating; + # Cache POST without headers set by the test helper! + proxy_cache_methods POST; + proxy_cache_key "$request_uri|$request_body"; + proxy_cache_valid 200 10m; + proxy_cache_valid any 0; + add_header X-Cache-Status $upstream_cache_status; + + } +} + +# Used by Netdata to monitor Nginx +server { + listen 127.0.0.1:80; + server_name localhost; + + allow 5.9.112.244; # monitoring host + deny all; + + location = /stub_status { + stub_status; + } +} + +# Used by Prometheus to reach the TH +server { + listen 9001; + server_name localhost; + + allow 5.9.112.244; # monitoring host + deny all; + + location = /metrics { + proxy_pass http://127.0.0.1:9091; + } +} diff --git a/ansible/roles/ooni-backend/templates/rotation_setup.sh b/ansible/roles/ooni-backend/templates/rotation_setup.sh new file mode 100644 index 00000000..5706150c --- /dev/null +++ b/ansible/roles/ooni-backend/templates/rotation_setup.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Managed by ansible, see roles/ooni-backend/tasks/main.yml +# +# Configure test-helper droplet +# This script is run remotely on newly spawned VM by https://github.com/ooni/backend/blob/master/analysis/rotation.py +# It runs as root and with CWD=/ +# +set -euo pipefail +exec 1>/var/log/vm_rotation_setup.log 2>&1 +echo > /etc/motd + +echo "Configuring APT" +echo "deb [trusted=yes] https://ooni-deb.s3.eu-central-1.amazonaws.com unstable main" > /etc/apt/sources.list.d/ooni.list +cat < /etc/apt/trusted.gpg.d/ooni.gpg +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYGISFRYJKwYBBAHaRw8BAQdA4VxoR0gSsH56BbVqYdK9HNQ0Dj2YFVbvKIIZ +JKlaW920Mk9PTkkgcGFja2FnZSBzaWduaW5nIDxjb250YWN0QG9wZW5vYnNlcnZh +dG9yeS5vcmc+iJYEExYIAD4WIQS1oI8BeW5/UhhhtEk3LR/ycfLdUAUCYGISFQIb +AwUJJZgGAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRA3LR/ycfLdUFk+AQCb +gsUQsAQGxUFvxk1XQ4RgEoh7wy2yTuK8ZCkSHJ0HWwD/f2OAjDigGq07uJPYw7Uo +Ih9+mJ/ubwiPMzUWF6RSdgu4OARgYhIVEgorBgEEAZdVAQUBAQdAx4p1KerwcIhX +HfM9LbN6Gi7z9j4/12JKYOvr0d0yC30DAQgHiH4EGBYIACYWIQS1oI8BeW5/Uhhh +tEk3LR/ycfLdUAUCYGISFQIbDAUJJZgGAAAKCRA3LR/ycfLdUL4cAQCs53fLphhy +6JMwVhRs02LXi1lntUtw1c+EMn6t7XNM6gD+PXpbgSZwoV3ZViLqr58o9fZQtV3s +oN7jfdbznrWVigE= +=PtYb +-----END PGP PUBLIC KEY BLOCK----- +EOF + +# Vector +cat < /etc/apt/trusted.gpg.d/vector.gpg +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v2 + +mQENBF9gFZ0BCADETtIHM8y5ehMoyNiZcriK+tHXyKnbZCKtMCKcC4ll94/6pekQ +jKIPWg8OXojkCtwua/TsddtQmOhUxAUtv6K0jO8r6sJ8rezMhuNH8J8rMqWgzv9d +2+U7Z7GFgcP0OeD+KigtnR8uyp50suBmEDC8YytmmbESmG261Y38vZME0VvQ+CMy +Yi/FvKXBXugaiCtaz0a5jVE86qSZbKbuaTHGiLn05xjTqc4FfyP4fi4oT2r6GGyL +Bn5ob84OjXLQwfbZIIrNFR10BvL2SRLL0kKKVlMBBADodtkdwaTt0pGuyEJ+gVBz +629PZBtSrwVRU399jGSfsxoiLca9//c7OJzHABEBAAG0OkNsb3Vkc21pdGggUGFj +a2FnZSAodGltYmVyL3ZlY3RvcikgPHN1cHBvcnRAY2xvdWRzbWl0aC5pbz6JATcE +EwEIACEFAl9gFZ0CGy8FCwkIBwMFFQoJCAsFFgIDAQACHgECF4AACgkQNUPbLQor +xLhf6gf8DyfIpKjvEeW/O8lRUTpkiPKezJbb+udZboCXJKDD02Q9PE3hfEfQRr5X +muytL7YMPvzqBVuP3xV5CN3zvtiQQbZiDhstImVyd+t24pQTkjzkvy+A2yvUuIkE +RWxuey41f5FNj/7wdfJnHoU9uJ/lvsb7DLXw7FBMZFNBR6LED/d+b61zMzVvmFZA +gsrCGwr/jfySwnpShmKdJaMTHQx0qt2RfXwNm2V6i900tAuMUWnmUIz5/9vENPKm +0+31I43a/QgmIrKEePhwn2jfA1oRlYzdv+PbblSTfjTStem+GqQkj9bZsAuqVH8g +3vq0NvX0k2CLi/W9mTiSdHXFChI15A== +=k36w +-----END PGP PUBLIC KEY BLOCK----- +EOF + +echo "deb https://repositories.timber.io/public/vector/deb/debian bullseye main" > /etc/apt/sources.list.d/vector.list + +echo "Installing packages" +export DEBIAN_FRONTEND=noninteractive +apt-get update -q +apt-get purge -qy unattended-upgrades rsyslog +apt-get upgrade -qy +apt-get install -qy --no-install-recommends chrony netdata oohelperd netdata-plugins-python + +systemctl daemon-reload +systemctl restart systemd-journald.service +logger start +systemctl restart systemd-journald.service + +apt-get install -qy --no-install-recommends vector + +echo "Configuring Vector" +# The certs are copied over by rotation.py +cat > /etc/vector/vector.toml < /etc/netdata/netdata.conf < /var/run/rotation_setup_completed diff --git a/ansible/roles/ooni-backend/templates/tor_targets.json b/ansible/roles/ooni-backend/templates/tor_targets.json new file mode 100644 index 00000000..933c4ede --- /dev/null +++ b/ansible/roles/ooni-backend/templates/tor_targets.json @@ -0,0 +1,304 @@ +{ + "128.31.0.39:9101": { + "address": "128.31.0.39:9101", + "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "name": "moria1", + "protocol": "or_port_dirauth" + }, + "128.31.0.39:9131": { + "address": "128.31.0.39:9131", + "fingerprint": "9695DFC35FFEB861329B9F1AB04C46397020CE31", + "name": "moria1", + "protocol": "dir_port" + }, + "131.188.40.189:443": { + "address": "131.188.40.189:443", + "fingerprint": "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281", + "name": "gabelmoo", + "protocol": "or_port_dirauth" + }, + "131.188.40.189:80": { + "address": "131.188.40.189:80", + "fingerprint": "F2044413DAC2E02E3D6BCF4735A19BCA1DE97281", + "name": "gabelmoo", + "protocol": "dir_port" + }, + "154.35.175.225:443": { + "address": "154.35.175.225:443", + "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "name": "Faravahar", + "protocol": "or_port_dirauth" + }, + "154.35.175.225:80": { + "address": "154.35.175.225:80", + "fingerprint": "CF6D0AAFB385BE71B8E111FC5CFF4B47923733BC", + "name": "Faravahar", + "protocol": "dir_port" + }, + "171.25.193.9:443": { + "address": "171.25.193.9:443", + "fingerprint": "BD6A829255CB08E66FBE7D3748363586E46B3810", + "name": "maatuska", + "protocol": "dir_port" + }, + "171.25.193.9:80": { + "address": "171.25.193.9:80", + "fingerprint": "BD6A829255CB08E66FBE7D3748363586E46B3810", + "name": "maatuska", + "protocol": "or_port_dirauth" + }, + "193.23.244.244:443": { + "address": "193.23.244.244:443", + "fingerprint": "7BE683E65D48141321C5ED92F075C55364AC7123", + "name": "dannenberg", + "protocol": "or_port_dirauth" + }, + "193.23.244.244:80": { + "address": "193.23.244.244:80", + "fingerprint": "7BE683E65D48141321C5ED92F075C55364AC7123", + "name": "dannenberg", + "protocol": "dir_port" + }, + "199.58.81.140:443": { + "address": "199.58.81.140:443", + "fingerprint": "74A910646BCEEFBCD2E874FC1DC997430F968145", + "name": "longclaw", + "protocol": "or_port_dirauth" + }, + "199.58.81.140:80": { + "address": "199.58.81.140:80", + "fingerprint": "74A910646BCEEFBCD2E874FC1DC997430F968145", + "name": "longclaw", + "protocol": "dir_port" + }, + "204.13.164.118:443": { + "address": "204.13.164.118:443", + "fingerprint": "24E2F139121D4394C54B5BCC368B3B411857C413", + "name": "bastet", + "protocol": "or_port_dirauth" + }, + "204.13.164.118:80": { + "address": "204.13.164.118:80", + "fingerprint": "24E2F139121D4394C54B5BCC368B3B411857C413", + "name": "bastet", + "protocol": "dir_port" + }, + "2d7292b5163fb7de5b24cd04032c93a2d4c454431de3a00b5a6d4a3309529e49": { + "address": "193.11.166.194:27020", + "fingerprint": "86AC7B8D430DAC4117E9F42C9EAED18133863AAF", + "params": { + "cert": [ + "0LDeJH4JzMDtkJJrFphJCiPqKx7loozKN7VNfuukMGfHO0Z8OGdzHVkhVAOfo1mUdv9cMg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "3fa772a44e07856b4c70e958b2f6dc8a29450a823509d5dbbf8b884e7fb5bb9d": { + "address": "192.95.36.142:443", + "fingerprint": "CDF2E852BF539B82BD10E27E9115A31734E378C2", + "params": { + "cert": [ + "qUVQ0srL1JI/vO6V6m/24anYXiJD3QP2HgzUKQtQ7GRqqUvs7P+tG43RtAqdhLOALP7DJQ" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "45.66.33.45:443": { + "address": "45.66.33.45:443", + "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", + "name": "dizum", + "protocol": "or_port_dirauth" + }, + "45.66.33.45:80": { + "address": "45.66.33.45:80", + "fingerprint": "7EA6EAD6FD83083C538F44038BBFA077587DD755", + "name": "dizum", + "protocol": "dir_port" + }, + "49116bf72d336bb8724fd3a06a5afa7bbd4e7baef35fbcdb9a98d13e702270ad": { + "address": "146.57.248.225:22", + "fingerprint": "10A6CD36A537FCE513A322361547444B393989F0", + "params": { + "cert": [ + "K1gDtDAIcUfeLqbstggjIw2rtgIKqdIhUlHp82XRqNSq/mtAjp1BIC9vHKJ2FAEpGssTPw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "4a330634c5d678887f0f7c299490af43a6ac9fa944a6cc2140ab264c9ec124a0": { + "address": "209.148.46.65:443", + "fingerprint": "74FAD13168806246602538555B5521A0383A1875", + "params": { + "cert": [ + "ssH+9rP8dG2NLDN2XuFw63hIO/9MNNinLmxQDpVa+7kTOa9/m+tGWT1SmSYpQ9uTBGa6Hw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "548eebff71da6128321c3bc1c3ec12b5bfff277ef5cde32709a33e207b57f3e2": { + "address": "37.218.245.14:38224", + "fingerprint": "D9A82D2F9C2F65A18407B1D2B764F130847F8B5D", + "params": { + "cert": [ + "bjRaMrr1BRiAW8IE9U5z27fQaYgOhX1UCmOpg2pFpoMvo6ZgQMzLsaTzzQNTlm7hNcb+Sg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "5aeb9e43b43fc8a809b8d25aae968395a5ceea0e677caaf56e1c0a2ba002f5b5": { + "address": "193.11.166.194:27015", + "fingerprint": "2D82C2E354D531A68469ADF7F878FA6060C6BACA", + "params": { + "cert": [ + "4TLQPJrTSaDffMK7Nbao6LC7G9OW/NHkUwIdjLSS3KYf0Nv4/nQiiI8dY2TcsQx01NniOg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "66.111.2.131:9001": { + "address": "66.111.2.131:9001", + "fingerprint": "BA44A889E64B93FAA2B114E02C2A279A8555C533", + "name": "Serge", + "protocol": "or_port_dirauth" + }, + "66.111.2.131:9030": { + "address": "66.111.2.131:9030", + "fingerprint": "BA44A889E64B93FAA2B114E02C2A279A8555C533", + "name": "Serge", + "protocol": "dir_port" + }, + "662218447d396b9d4f01b585457d267735601fedbeb9a19b86b942f238fe4e7b": { + "address": "51.222.13.177:80", + "fingerprint": "5EDAC3B810E12B01F6FD8050D2FD3E277B289A08", + "params": { + "cert": [ + "2uplIpLQ0q9+0qMFrK5pkaYRDOe460LL9WHBvatgkuRr/SL31wBOEupaMMJ6koRE6Ld0ew" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "75fe96d641a078fee06529af376d7f8c92757596e48558d5d02baa1e10321d10": { + "address": "45.145.95.6:27015", + "fingerprint": "C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C", + "params": { + "cert": [ + "TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "86.59.21.38:443": { + "address": "86.59.21.38:443", + "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "name": "tor26", + "protocol": "or_port_dirauth" + }, + "86.59.21.38:80": { + "address": "86.59.21.38:80", + "fingerprint": "847B1F850344D7876491A54892F904934E4EB85D", + "name": "tor26", + "protocol": "dir_port" + }, + "99e9adc8bba0d60982dbc655b5e8735d88ad788905c3713a39eff3224b617eeb": { + "address": "38.229.1.78:80", + "fingerprint": "C8CBDB2464FC9804A69531437BCF2BE31FDD2EE4", + "params": { + "cert": [ + "Hmyfd2ev46gGY7NoVxA9ngrPF2zCZtzskRTzoWXbxNkzeVnGFPWmrTtILRyqCTjHR+s9dg" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "9d735c6e70512123ab2c2fe966446b2345b352c512e9fb359f4b1673236e4d4a": { + "address": "38.229.33.83:80", + "fingerprint": "0BAC39417268B96B9F514E7F63FA6FBA1A788955", + "params": { + "cert": [ + "VwEFpk9F/UN9JED7XpG1XOjm/O8ZCXK80oPecgWnNDZDv5pdkhq1OpbAH0wNqOT6H6BmRQ" + ], + "iat-mode": [ + "1" + ] + }, + "protocol": "obfs4" + }, + "b7c0e3f183ad85a6686ec68344765cec57906b215e7b82a98a9ca013cb980efa": { + "address": "193.11.166.194:27025", + "fingerprint": "1AE2C08904527FEA90C4C4F8C1083EA59FBC6FAF", + "params": { + "cert": [ + "ItvYZzW5tn6v3G4UnQa6Qz04Npro6e81AP70YujmK/KXwDFPTs3aHXcHp4n8Vt6w/bv8cA" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "b8de51da541ced804840b1d8fd24d5ff1cfdf07eae673dae38c2bc2cce594ddd": { + "address": "85.31.186.26:443", + "fingerprint": "91A6354697E6B02A386312F68D82CF86824D3606", + "params": { + "cert": [ + "PBwr+S8JTVZo6MPdHnkTwXJPILWADLqfMGoVvhZClMq/Urndyd42BwX9YFJHZnBB3H0XCw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "d2d6e34abeda851f7cd37138ffafcce992b2ccdb0f263eb90ab75d7adbd5eeba": { + "address": "85.31.186.98:443", + "fingerprint": "011F2599C0E9B27EE74B353155E244813763C3E5", + "params": { + "cert": [ + "ayq0XzCwhpdysn5o0EyDUbmSOx3X/oTEbzDMvczHOdBJKlvIdHHLJGkZARtT4dcBFArPPg" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + }, + "f855ba38d517d8589c16e1333ac23c6e516532cf036ab6f47b15030b40a3b6a6": { + "address": "[2a0c:4d80:42:702::1]:27015", + "fingerprint": "C5B7CD6946FF10C5B3E89691A7D3F2C122D2117C", + "params": { + "cert": [ + "TD7PbUO0/0k6xYHMPW3vJxICfkMZNdkRrb63Zhl5j9dW3iRGiCx0A7mPhe5T2EDzQ35+Zw" + ], + "iat-mode": [ + "0" + ] + }, + "protocol": "obfs4" + } +} \ No newline at end of file diff --git a/ansible/roles/prometheus/templates/prometheus.yml b/ansible/roles/prometheus/templates/prometheus.yml index e8f9cd30..bed0464e 100755 --- a/ansible/roles/prometheus/templates/prometheus.yml +++ b/ansible/roles/prometheus/templates/prometheus.yml @@ -151,6 +151,30 @@ scrape_configs: - targets: - backend-fsn.ooni.org:9363 + - job_name: 'clickhouse cluster' + scrape_interval: 5s + scheme: http + metrics_path: "/metrics/clickhouse" + basic_auth: + username: 'prom' + password: '{{ prometheus_metrics_password_prod }}' + static_configs: + - targets: + - data1.htz-fsn.prod.ooni.nu:9100 + - data3.htz-fsn.prod.ooni.nu:9100 + + - job_name: 'node new' + scrape_interval: 5s + scheme: http + metrics_path: "/metrics/node_exporter" + basic_auth: + username: 'prom' + password: '{{ prometheus_metrics_password_prod }}' + static_configs: + - targets: + - data1.htz-fsn.prod.ooni.nu:9100 + - data3.htz-fsn.prod.ooni.nu:9100 + # See ansible/roles/ooni-backend/tasks/main.yml for the scraping targets - job_name: 'haproxy' scrape_interval: 5s diff --git a/ansible/roles/prometheus/vars/main.yml b/ansible/roles/prometheus/vars/main.yml index abf6d469..01ae359f 100644 --- a/ansible/roles/prometheus/vars/main.yml +++ b/ansible/roles/prometheus/vars/main.yml @@ -1,8 +1,6 @@ dom0_hosts: - ams-ps.ooni.nu - ams-slack-1.ooni.org - - ams-wcth2.ooni.nu - - ams-wcth3.ooni.nu - amsmatomo.ooni.nu - db-1.proteus.ooni.io - doams1-countly.ooni.nu @@ -15,8 +13,9 @@ blackbox_jobs: targets: # - "https://a.web-connectivity.th.ooni.io/status" - "https://wcth.ooni.io/status" - - "https://ams-wcth2.ooni.nu/status" - - "https://a.web-connectivity.th.ooni.io/status" # "https://ams-wcth3.ooni.nu/status" + # TODO add these records to the ALB config + #- "https://ams-wcth2.ooni.nu/status" + #- "https://a.web-connectivity.th.ooni.io/status" # "https://ams-wcth3.ooni.nu/status" # cloudfront - "https://d33d1gs9kpq1c5.cloudfront.net/status" diff --git a/ansible/roles/prometheus_node_exporter/defaults/main.yml b/ansible/roles/prometheus_node_exporter/defaults/main.yml new file mode 100644 index 00000000..3433498f --- /dev/null +++ b/ansible/roles/prometheus_node_exporter/defaults/main.yml @@ -0,0 +1,16 @@ +prometheus_nginx_proxy_config: + - location: /metrics/node_exporter + proxy_pass: http://127.0.0.1:8100/metrics + +node_exporter_version: '1.8.2' +node_exporter_arch: 'amd64' +node_exporter_download_url: https://github.com/prometheus/node_exporter/releases/download/v{{ node_exporter_version }}/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}.tar.gz + +node_exporter_bin_path: /usr/local/bin/node_exporter +node_exporter_host: 'localhost' +node_exporter_port: 8100 +node_exporter_options: '' + +node_exporter_state: started +node_exporter_enabled: true +node_exporter_restart: on-failure diff --git a/ansible/roles/prometheus_node_exporter/handlers/main.yml b/ansible/roles/prometheus_node_exporter/handlers/main.yml index 69a5b2fe..4ec66003 100644 --- a/ansible/roles/prometheus_node_exporter/handlers/main.yml +++ b/ansible/roles/prometheus_node_exporter/handlers/main.yml @@ -13,3 +13,8 @@ ansible.builtin.systemd_service: name: nginx state: restarted + +- name: restart node_exporter + service: + name: node_exporter + state: restarted diff --git a/ansible/roles/prometheus_node_exporter/tasks/install.yml b/ansible/roles/prometheus_node_exporter/tasks/install.yml new file mode 100644 index 00000000..2ad7ccd7 --- /dev/null +++ b/ansible/roles/prometheus_node_exporter/tasks/install.yml @@ -0,0 +1,60 @@ +--- +- name: Check current node_exporter version. + command: "{{ node_exporter_bin_path }} --version" + failed_when: false + changed_when: false + register: node_exporter_version_check + +- name: Download and unarchive node_exporter into temporary location. + unarchive: + src: "{{ node_exporter_download_url }}" + dest: /tmp + remote_src: true + mode: 0755 + when: > + node_exporter_version_check.stdout is not defined + or node_exporter_version not in node_exporter_version_check.stdout + register: node_exporter_download_check + +- name: Move node_exporter binary into place. + copy: + src: "/tmp/node_exporter-{{ node_exporter_version }}.linux-{{ node_exporter_arch }}/node_exporter" + dest: "{{ node_exporter_bin_path }}" + mode: 0755 + remote_src: true + notify: restart node_exporter + when: > + node_exporter_download_check is changed + or node_exporter_version_check.stdout | length == 0 + +- name: Create node_exporter user. + user: + name: node_exporter + shell: /sbin/nologin + state: present + +- name: Copy the node_exporter systemd unit file. + template: + src: node_exporter.service.j2 + dest: /etc/systemd/system/node_exporter.service + mode: 0644 + register: node_exporter_service + +- name: Reload systemd daemon if unit file is changed. + systemd: + daemon_reload: true + notify: restart node_exporter + when: node_exporter_service is changed + +- name: Ensure node_exporter is running and enabled at boot. + service: + name: node_exporter + state: "{{ node_exporter_state }}" + enabled: "{{ node_exporter_enabled }}" + +- name: Verify node_exporter is responding to requests. + uri: + url: "http://{% if node_exporter_host !='' %}{{ node_exporter_host }}{% else %}localhost{% endif %}:{{ node_exporter_port }}/" + return_content: true + register: metrics_output + failed_when: "'Metrics' not in metrics_output.content" diff --git a/ansible/roles/prometheus_node_exporter/tasks/main.yml b/ansible/roles/prometheus_node_exporter/tasks/main.yml index 0c4fc242..cf9f8229 100644 --- a/ansible/roles/prometheus_node_exporter/tasks/main.yml +++ b/ansible/roles/prometheus_node_exporter/tasks/main.yml @@ -4,15 +4,7 @@ - nginx - node_exporter -- ansible.builtin.include_role: - name: geerlingguy.node_exporter - vars: - node_exporter_host: "localhost" - node_exporter_port: 8100 - tags: - - monitoring - - node_exporter - - config +- include_tasks: install.yml - name: create ooni configuration directory ansible.builtin.file: @@ -30,7 +22,7 @@ name: prom password: "{{ prometheus_metrics_password }}" owner: root - group: www-data + group: nginx mode: 0640 tags: - monitoring @@ -55,7 +47,7 @@ nft_rules_tcp: - name: 9100 rules: - - add rule inet filter input tcp dport 9100 counter accept comment "Incoming prometheus monitoring" + - add rule inet filter input ip saddr 5.9.112.244 tcp dport 9100 counter accept comment "clickhouse prometheus from monitoring.ooni.org" tags: - monitoring - node_exporter diff --git a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 index 7d9fbab1..7e68c45c 100644 --- a/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 +++ b/ansible/roles/prometheus_node_exporter/templates/nginx-prometheus.j2 @@ -7,14 +7,18 @@ server { access_log /var/log/nginx/{{ inventory_hostname }}.access.log; error_log /var/log/nginx/{{ inventory_hostname }}.log warn; - location /metrics { + {% for config in prometheus_nginx_proxy_config %} + + location {{ config['location'] }} { auth_basic "Administrator’s Area"; auth_basic_user_file /etc/ooni/prometheus_passwd; - proxy_pass http://127.0.0.1:8100; + proxy_pass {{ config['proxy_pass'] }}; proxy_set_header X-Real-IP $remote_addr; proxy_set_header Host $host; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } -} \ No newline at end of file + + {% endfor %} +} diff --git a/ansible/roles/prometheus_node_exporter/templates/node_exporter.service.j2 b/ansible/roles/prometheus_node_exporter/templates/node_exporter.service.j2 new file mode 100644 index 00000000..42cb98cc --- /dev/null +++ b/ansible/roles/prometheus_node_exporter/templates/node_exporter.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=NodeExporter + +[Service] +TimeoutStartSec=0 +User=node_exporter +ExecStart={{ node_exporter_bin_path }} --web.listen-address={{ node_exporter_host }}:{{ node_exporter_port }} {{ node_exporter_options }} +Restart={{ node_exporter_restart }} + +[Install] +WantedBy=multi-user.target diff --git a/scripts/cluster-migration/benchmark.sql b/scripts/cluster-migration/benchmark.sql new file mode 100644 index 00000000..55e06781 --- /dev/null +++ b/scripts/cluster-migration/benchmark.sql @@ -0,0 +1,55 @@ +SELECT + countIf ( + anomaly = 't' + AND confirmed = 'f' + AND msm_failure = 'f' + ) AS anomaly_count, + countIf ( + confirmed = 't' + AND msm_failure = 'f' + ) AS confirmed_count, + countIf (msm_failure = 't') AS failure_count, + countIf ( + anomaly = 'f' + AND confirmed = 'f' + AND msm_failure = 'f' + ) AS ok_count, + COUNT(*) AS measurement_count, + domain +FROM + fastpath +WHERE + measurement_start_time >= '2024-11-01' + AND measurement_start_time < '2024-11-10' + AND probe_cc = 'IT' +GROUP BY + domain; + +SELECT + COUNT(*) AS measurement_count, + domain +FROM + analysis_web_measurement +WHERE + measurement_start_time >= '2024-11-01' + AND measurement_start_time < '2024-11-10' + AND probe_cc = 'IT' +GROUP BY + domain; + +ALTER TABLE ooni.analysis_web_measurement ON CLUSTER oonidata_cluster MODIFY +ORDER BY + ( + measurement_start_time, + probe_cc, + probe_asn, + domain, + measurement_uid + ) +ALTER TABLE ooni.analysis_web_measurement ON CLUSTER oonidata_cluster ADD INDEX IF NOT EXISTS measurement_start_time_idx measurement_start_time TYPE minmax GRANULARITY 2; + +ALTER TABLE ooni.analysis_web_measurement ON CLUSTER oonidata_cluster MATERIALIZE INDEX measurement_start_time_idx; + +ALTER TABLE ooni.analysis_web_measurement ON CLUSTER oonidata_cluster ADD INDEX IF NOT EXISTS probe_cc_idx probe_cc TYPE minmax GRANULARITY 1; + +ALTER TABLE ooni.analysis_web_measurement ON CLUSTER oonidata_cluster MATERIALIZE INDEX probe_cc_idx; \ No newline at end of file diff --git a/scripts/cluster-migration/db-sample.py b/scripts/cluster-migration/db-sample.py new file mode 100644 index 00000000..d4544135 --- /dev/null +++ b/scripts/cluster-migration/db-sample.py @@ -0,0 +1,33 @@ +from datetime import datetime, timedelta +import csv + +from tqdm import tqdm +from clickhouse_driver import Client as ClickhouseClient + + +START_TIME = datetime(2024, 11, 1, 0, 0, 0) +END_TIME = datetime(2024, 11, 10, 0, 0, 0) +SAMPLE_SIZE = 100 + + +def sample_to_file(table_name): + with ClickhouseClient.from_url("clickhouse://localhost/ooni") as click, open( + f"{table_name}-sample.csv", "w" + ) as out_file: + writer = csv.writer(out_file) + ts = START_TIME + while ts < END_TIME: + for row in click.execute_iter( + f""" + SELECT * FROM {table_name} + WHERE measurement_uid LIKE '{ts.strftime("%Y%m%d%H")}%' + ORDER BY measurement_uid LIMIT {SAMPLE_SIZE} + """ + ): + writer.writerow(row) + ts += timedelta(hours=1) + + +if __name__ == "__main__": + sample_to_file("obs_web") + sample_to_file("analysis_web_measurement") diff --git a/scripts/cluster-migration/migrate-tables.py b/scripts/cluster-migration/migrate-tables.py new file mode 100644 index 00000000..2a3d4bfb --- /dev/null +++ b/scripts/cluster-migration/migrate-tables.py @@ -0,0 +1,38 @@ +import os + +from tqdm import tqdm +from clickhouse_driver import Client as ClickhouseClient + + +WRITE_CLICKHOUSE_URL = os.environ["WRITE_CLICKHOUSE_URL"] + + +def stream_table(table_name, where_clause): + with ClickhouseClient.from_url("clickhouse://backend-fsn.ooni.org/") as click: + for row in click.execute_iter(f"SELECT * FROM {table_name} {where_clause}"): + yield row + + +def copy_table(table_name, where_clause): + with ClickhouseClient.from_url(WRITE_CLICKHOUSE_URL) as click_writer: + buf = [] + for row in tqdm(stream_table(table_name=table_name, where_clause=where_clause)): + buf.append(row) + if len(buf) > 50_000: + click_writer.execute(f"INSERT INTO {table_name} VALUES", buf) + buf = [] + + if len(buf) > 0: + click_writer.execute(f"INSERT INTO {table_name} VALUES", buf) + + +if __name__ == "__main__": + assert WRITE_CLICKHOUSE_URL, "WRITE_CLICKHOUSE_URL environment variable is not set" + print("## copying `fastpath` table") + copy_table("fastpath", "WHERE measurement_uid < '20241127'") + print("## copying `jsonl` table") + copy_table("jsonl", "WHERE measurement_uid < '20241127'") + print("## copying `citizenlab` table") + copy_table("citizenlab", "") + print("## copying `citizenlab_flip` table") + copy_table("citizenlab_flip", "") diff --git a/scripts/cluster-migration/schema.sql b/scripts/cluster-migration/schema.sql new file mode 100644 index 00000000..7588f060 --- /dev/null +++ b/scripts/cluster-migration/schema.sql @@ -0,0 +1,137 @@ +CREATE TABLE + ooni.jsonl ON CLUSTER oonidata_cluster ( + `report_id` String, + `input` String, + `s3path` String, + `linenum` Int32, + `measurement_uid` String, + `date` Date, + `source` String, + `update_time` DateTime64 (3) MATERIALIZED now64 () + ) ENGINE = ReplicatedReplacingMergeTree ( + '/clickhouse/{cluster}/tables/ooni/jsonl/{shard}', + '{replica}', + update_time + ) +ORDER BY + (report_id, input, measurement_uid) SETTINGS index_granularity = 8192; + +CREATE TABLE + ooni.fastpath ON CLUSTER oonidata_cluster ( + `measurement_uid` String, + `report_id` String, + `input` String, + `probe_cc` LowCardinality (String), + `probe_asn` Int32, + `test_name` LowCardinality (String), + `test_start_time` DateTime, + `measurement_start_time` DateTime, + `filename` String, + `scores` String, + `platform` String, + `anomaly` String, + `confirmed` String, + `msm_failure` String, + `domain` String, + `software_name` String, + `software_version` String, + `control_failure` String, + `blocking_general` Float32, + `is_ssl_expected` Int8, + `page_len` Int32, + `page_len_ratio` Float32, + `server_cc` String, + `server_asn` Int8, + `server_as_name` String, + `update_time` DateTime64 (3) MATERIALIZED now64 (), + `test_version` String, + `architecture` String, + `engine_name` LowCardinality (String), + `engine_version` String, + `test_runtime` Float32, + `blocking_type` String, + `test_helper_address` LowCardinality (String), + `test_helper_type` LowCardinality (String), + `ooni_run_link_id` Nullable (UInt64), + INDEX fastpath_rid_idx report_id TYPE minmax GRANULARITY 1, + INDEX measurement_uid_idx measurement_uid TYPE minmax GRANULARITY 8 + ) ENGINE = ReplicatedReplacingMergeTree ( + '/clickhouse/{cluster}/tables/ooni/fastpath/{shard}', + '{replica}', + update_time + ) +ORDER BY + ( + measurement_start_time, + report_id, + input, + measurement_uid + ) SETTINGS index_granularity = 8192; + +CREATE TABLE + ooni.citizenlab ON CLUSTER oonidata_cluster ( + `domain` String, + `url` String, + `cc` FixedString (32), + `category_code` String + ) ENGINE = ReplicatedReplacingMergeTree ( + '/clickhouse/{cluster}/tables/ooni/citizenlab/{shard}', + '{replica}' + ) +ORDER BY + (domain, url, cc, category_code) SETTINGS index_granularity = 4; + +CREATE TABLE + ooni.citizenlab_flip ON CLUSTER oonidata_cluster ( + `domain` String, + `url` String, + `cc` FixedString (32), + `category_code` String + ) ENGINE = ReplicatedReplacingMergeTree ( + '/clickhouse/{cluster}/tables/ooni/citizenlab_flip/{shard}', + '{replica}' + ) +ORDER BY + (domain, url, cc, category_code) SETTINGS index_granularity = 4; + +CREATE TABLE + analysis_web_measurement ON CLUSTER oonidata_cluster ( + `domain` String, + `input` String, + `test_name` String, + `probe_asn` UInt32, + `probe_as_org_name` String, + `probe_cc` String, + `resolver_asn` UInt32, + `resolver_as_cc` String, + `network_type` String, + `measurement_start_time` DateTime64 (3, 'UTC'), + `measurement_uid` String, + `ooni_run_link_id` String, + `top_probe_analysis` Nullable (String), + `top_dns_failure` Nullable (String), + `top_tcp_failure` Nullable (String), + `top_tls_failure` Nullable (String), + `dns_blocked` Float32, + `dns_down` Float32, + `dns_ok` Float32, + `tcp_blocked` Float32, + `tcp_down` Float32, + `tcp_ok` Float32, + `tls_blocked` Float32, + `tls_down` Float32, + `tls_ok` Float32 + ) ENGINE = ReplicatedReplacingMergeTree ( + '/clickhouse/{cluster}/tables/ooni/analysis_web_measurement/{shard}', + '{replica}' + ) +PARTITION BY + substring(measurement_uid, 1, 6) PRIMARY KEY measurement_uid +ORDER BY + ( + measurement_uid, + measurement_start_time, + probe_cc, + probe_asn, + domain + ) SETTINGS index_granularity = 8192; \ No newline at end of file diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 09e4636c..2b14235b 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -34,10 +34,13 @@ provider "aws" { # source_profile = oonidevops_user } -# In order for this provider to work you have to set the following environment -# variable to your DigitalOcean API token: -# DIGITALOCEAN_ACCESS_TOKEN= -provider "digitalocean" {} +data "aws_ssm_parameter" "do_token" { + name = "/oonidevops/secrets/digitalocean_access_token" +} + +provider "digitalocean" { + token = data.aws_ssm_parameter.do_token.value +} data "aws_availability_zones" "available" {} @@ -226,6 +229,10 @@ resource "aws_secretsmanager_secret_version" "oonipg_url" { ) } +data "aws_ssm_parameter" "clickhouse_readonly_url" { + name = "/oonidevops/secrets/clickhouse_readonly_url" +} + resource "random_id" "artifact_id" { byte_length = 4 } @@ -277,31 +284,6 @@ module "ooni_th_droplet" { dns_zone_ooni_io = local.dns_zone_ooni_io } -module "ooni_backendproxy" { - source = "../../modules/ooni_backendproxy" - - stage = local.environment - - vpc_id = module.network.vpc_id - subnet_id = module.network.vpc_subnet_public[0].id - private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block - dns_zone_ooni_io = local.dns_zone_ooni_io - - key_name = module.adm_iam_roles.oonidevops_key_name - instance_type = "t2.micro" - - backend_url = "https://backend-hel.ooni.org/" - wcth_addresses = module.ooni_th_droplet.droplet_ipv4_address - wcth_domain_suffix = "th.dev.ooni.io" - clickhouse_url = "backend-fsn.ooni.org" - clickhouse_port = "9000" - - tags = merge( - local.tags, - { Name = "ooni-tier0-backendproxy" } - ) -} - ### OONI Services Clusters module "ooniapi_cluster" { @@ -316,7 +298,7 @@ module "ooniapi_cluster" { asg_max = 6 asg_desired = 2 - instance_type = "t3a.medium" + instance_type = "t3a.micro" tags = merge( local.tags, @@ -346,8 +328,7 @@ module "ooniapi_ooniprobe_deployer" { module "ooniapi_ooniprobe" { source = "../../modules/ooniapi_service" - task_cpu = 256 - task_memory = 512 + task_memory = 64 # First run should be set on first run to bootstrap the task definition # first_run = true @@ -379,6 +360,86 @@ module "ooniapi_ooniprobe" { ) } +#### OONI Backend proxy service + +module "ooniapi_reverseproxy_deployer" { + source = "../../modules/ooniapi_service_deployer" + + service_name = "reverseproxy" + repo = "ooni/backend" + branch_name = "master" + buildspec_path = "ooniapi/services/reverseproxy/buildspec.yml" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + +module "ooniapi_reverseproxy" { + source = "../../modules/ooniapi_service" + + task_memory = 64 + + # First run should be set on first run to bootstrap the task definition + # first_run = true + + vpc_id = module.network.vpc_id + public_subnet_ids = module.network.vpc_subnet_public[*].id + private_subnet_ids = module.network.vpc_subnet_private[*].id + + service_name = "reverseproxy" + default_docker_image_url = "ooni/api-reverseproxy:latest" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + } + + task_environment = { + TARGET_URL = "https://backend-hel.ooni.org/" + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reverseproxy" } + ) +} + +module "ooni_backendproxy" { + source = "../../modules/ooni_backendproxy" + + stage = local.environment + + vpc_id = module.network.vpc_id + subnet_id = module.network.vpc_subnet_public[0].id + private_subnet_cidr = module.network.vpc_subnet_private[*].cidr_block + dns_zone_ooni_io = local.dns_zone_ooni_io + + key_name = module.adm_iam_roles.oonidevops_key_name + instance_type = "t3a.nano" + + backend_url = "https://backend-fsn.ooni.org/" + wcth_addresses = module.ooni_th_droplet.droplet_ipv4_address + wcth_domain_suffix = "th.ooni.org" + clickhouse_url = "clickhouse1.prod.ooni.io" + clickhouse_port = "9000" + + tags = merge( + local.tags, + { Name = "ooni-tier0-backendproxy" } + ) +} + + #### OONI Run service @@ -400,8 +461,7 @@ module "ooniapi_oonirun_deployer" { module "ooniapi_oonirun" { source = "../../modules/ooniapi_service" - task_cpu = 256 - task_memory = 512 + task_memory = 64 vpc_id = module.network.vpc_id public_subnet_ids = module.network.vpc_subnet_public[*].id @@ -438,7 +498,7 @@ module "ooniapi_oonifindings_deployer" { service_name = "oonifindings" repo = "ooni/backend" - branch_name = "master" + branch_name = "oonidata" buildspec_path = "ooniapi/services/oonifindings/buildspec.yml" codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn @@ -451,8 +511,7 @@ module "ooniapi_oonifindings_deployer" { module "ooniapi_oonifindings" { source = "../../modules/ooniapi_service" - task_cpu = 256 - task_memory = 512 + task_memory = 64 vpc_id = module.network.vpc_id public_subnet_ids = module.network.vpc_subnet_public[*].id @@ -469,6 +528,7 @@ module "ooniapi_oonifindings" { POSTGRESQL_URL = aws_secretsmanager_secret_version.oonipg_url.arn JWT_ENCRYPTION_KEY = aws_secretsmanager_secret_version.jwt_secret.arn PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + CLICKHOUSE_URL = data.aws_ssm_parameter.clickhouse_readonly_url.arn } ooniapi_service_security_groups = [ @@ -502,8 +562,7 @@ module "ooniapi_ooniauth_deployer" { module "ooniapi_ooniauth" { source = "../../modules/ooniapi_service" - task_cpu = 256 - task_memory = 512 + task_memory = 64 vpc_id = module.network.vpc_id public_subnet_ids = module.network.vpc_subnet_public[*].id @@ -559,7 +618,7 @@ module "ooniapi_frontend" { vpc_id = module.network.vpc_id subnet_ids = module.network.vpc_subnet_public[*].id - oonibackend_proxy_target_group_arn = module.ooni_backendproxy.alb_target_group_id + oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id @@ -595,7 +654,7 @@ locals { } resource "aws_route53_record" "ooniapi_frontend_main" { - name = local.ooniapi_frontend_main_domain_name + name = local.ooniapi_frontend_main_domain_name zone_id = local.ooniapi_frontend_main_domain_name_zone_id type = "A" diff --git a/tf/environments/prod/main.tf b/tf/environments/prod/main.tf index 477ffcdb..48902a89 100644 --- a/tf/environments/prod/main.tf +++ b/tf/environments/prod/main.tf @@ -287,6 +287,7 @@ module "ooni_th_droplet" { "3d:81:99:17:b5:d1:20:a5:fe:2b:14:96:67:93:d6:34", "f6:4b:8b:e2:0e:d2:97:c5:45:5c:07:a6:fe:54:60:0e" ] + dns_zone_ooni_io = local.dns_zone_ooni_io } @@ -315,6 +316,58 @@ module "ooni_backendproxy" { ) } +module "ooniapi_reverseproxy_deployer" { + source = "../../modules/ooniapi_service_deployer" + + service_name = "reverseproxy" + repo = "ooni/backend" + branch_name = "master" + buildspec_path = "ooniapi/services/reverseproxy/buildspec.yml" + codestar_connection_arn = aws_codestarconnections_connection.oonidevops.arn + + codepipeline_bucket = aws_s3_bucket.ooniapi_codepipeline_bucket.bucket + + ecs_service_name = module.ooniapi_reverseproxy.ecs_service_name + ecs_cluster_name = module.ooniapi_cluster.cluster_name +} + +module "ooniapi_reverseproxy" { + source = "../../modules/ooniapi_service" + + task_memory = 64 + + # First run should be set on first run to bootstrap the task definition + # first_run = true + + vpc_id = module.network.vpc_id + public_subnet_ids = module.network.vpc_subnet_public[*].id + private_subnet_ids = module.network.vpc_subnet_private[*].id + + service_name = "reverseproxy" + default_docker_image_url = "ooni/api-reverseproxy:latest" + stage = local.environment + dns_zone_ooni_io = local.dns_zone_ooni_io + key_name = module.adm_iam_roles.oonidevops_key_name + ecs_cluster_id = module.ooniapi_cluster.cluster_id + + task_secrets = { + PROMETHEUS_METRICS_PASSWORD = aws_secretsmanager_secret_version.prometheus_metrics_password.arn + } + + task_environment = { + TARGET_URL = "https://backend-fsn.ooni.org/" + } + + ooniapi_service_security_groups = [ + module.ooniapi_cluster.web_security_group_id + ] + + tags = merge( + local.tags, + { Name = "ooni-tier0-reverseproxy" } + ) +} + ### OONI Services Clusters module "ooniapi_cluster" { @@ -463,7 +516,7 @@ module "ooniapi_oonifindings_deployer" { module "ooniapi_oonifindings" { source = "../../modules/ooniapi_service" - first_run = true + # first_run = true vpc_id = module.network.vpc_id public_subnet_ids = module.network.vpc_subnet_public[*].id private_subnet_ids = module.network.vpc_subnet_private[*].id @@ -511,7 +564,7 @@ module "ooniapi_ooniauth_deployer" { module "ooniapi_ooniauth" { source = "../../modules/ooniapi_service" - #first_run = true + # first_run = true vpc_id = module.network.vpc_id private_subnet_ids = module.network.vpc_subnet_private[*].id @@ -569,7 +622,7 @@ module "ooniapi_frontend" { vpc_id = module.network.vpc_id subnet_ids = module.network.vpc_subnet_public[*].id - oonibackend_proxy_target_group_arn = module.ooni_backendproxy.alb_target_group_id + oonibackend_proxy_target_group_arn = module.ooniapi_reverseproxy.alb_target_group_id ooniapi_oonirun_target_group_arn = module.ooniapi_oonirun.alb_target_group_id ooniapi_ooniauth_target_group_arn = module.ooniapi_ooniauth.alb_target_group_id ooniapi_ooniprobe_target_group_arn = module.ooniapi_ooniprobe.alb_target_group_id diff --git a/tf/modules/ooni_backendproxy/main.tf b/tf/modules/ooni_backendproxy/main.tf index ad5b9bec..110461d3 100644 --- a/tf/modules/ooni_backendproxy/main.tf +++ b/tf/modules/ooni_backendproxy/main.tf @@ -12,16 +12,16 @@ resource "aws_security_group" "nginx_sg" { ingress { protocol = "tcp" - from_port = 80 - to_port = 80 - cidr_blocks = ["0.0.0.0/0"] + from_port = 9000 + to_port = 9000 + cidr_blocks = var.private_subnet_cidr } ingress { protocol = "tcp" - from_port = 9000 - to_port = 9000 - cidr_blocks = var.private_subnet_cidr + from_port = 80 + to_port = 80 + cidr_blocks = ["0.0.0.0/0"] } ingress { @@ -132,7 +132,7 @@ resource "aws_lb_target_group_attachment" "oonibackend_proxy" { resource "aws_route53_record" "clickhouse_proxy_alias" { zone_id = var.dns_zone_ooni_io - name = "clickhouse.${var.stage}.ooni.io" + name = "clickhouseproxy.${var.stage}.ooni.io" type = "CNAME" ttl = 300 diff --git a/tf/modules/ooni_th_droplet/main.tf b/tf/modules/ooni_th_droplet/main.tf index b62b47e9..9836ac62 100644 --- a/tf/modules/ooni_th_droplet/main.tf +++ b/tf/modules/ooni_th_droplet/main.tf @@ -34,6 +34,7 @@ resource "digitalocean_droplet" "ooni_th_docker" { lifecycle { create_before_destroy = true + ignore_changes = all } } resource "aws_route53_record" "ooni_th" { diff --git a/tf/modules/ooniapi_frontend/main.tf b/tf/modules/ooniapi_frontend/main.tf index c72937a2..d65f3b9d 100644 --- a/tf/modules/ooniapi_frontend/main.tf +++ b/tf/modules/ooniapi_frontend/main.tf @@ -182,7 +182,12 @@ resource "aws_lb_listener_rule" "ooniapi_oonifindings_rule" { condition { path_pattern { - values = ["/api/v1/incidents/*"] + values = [ + "/api/v1/incidents/*", + "/api/v1/aggregation/*", + "/api/v1/observations", + "/api/v1/analysis", + ] } } } diff --git a/tf/modules/ooniapi_service/main.tf b/tf/modules/ooniapi_service/main.tf index ad429a01..c5def884 100644 --- a/tf/modules/ooniapi_service/main.tf +++ b/tf/modules/ooniapi_service/main.tf @@ -40,11 +40,6 @@ resource "aws_cloudwatch_log_group" "ooniapi_service" { name = "ooni-ecs-group/${local.name}" } - -locals { - container_port = 80 -} - // This is done to retrieve the image name of the current task definition // It's important to keep aligned the container_name and task_definitions data "aws_ecs_container_definition" "ooniapi_service_current" { @@ -59,18 +54,17 @@ resource "aws_ecs_task_definition" "ooniapi_service" { container_definitions = jsonencode([ { - cpu = var.task_cpu, + memoryReservation = var.task_memory, essential = true, image = try( data.aws_ecs_container_definition.ooniapi_service_current[0].image, var.default_docker_image_url ), - memory = var.task_memory, name = local.name, portMappings = [ { - containerPort = local.container_port, + containerPort = 80 } ], diff --git a/tf/modules/ooniapi_service/templates/profile_policy.json b/tf/modules/ooniapi_service/templates/profile_policy.json index 5857ee55..3a772893 100644 --- a/tf/modules/ooniapi_service/templates/profile_policy.json +++ b/tf/modules/ooniapi_service/templates/profile_policy.json @@ -35,6 +35,16 @@ "Action": "secretsmanager:ListSecrets", "Resource": "*" }, + { + "Effect": "Allow", + "Action": [ + "ssm:GetParameter", + "ssm:GetParameters", + "ssm:GetParameterHistory", + "ssm:GetParametersByPath" + ], + "Resource": "arn:aws:ssm:*" + }, { "Effect": "Allow", "Action": [ diff --git a/tf/modules/ooniapi_service/variables.tf b/tf/modules/ooniapi_service/variables.tf index f83e16d7..bda90a72 100644 --- a/tf/modules/ooniapi_service/variables.tf +++ b/tf/modules/ooniapi_service/variables.tf @@ -44,13 +44,8 @@ variable "service_desired_count" { default = 1 } -variable "task_cpu" { - default = 256 - description = "https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size" -} - variable "task_memory" { - default = 512 + default = 64 description = "https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size" } @@ -79,4 +74,4 @@ variable "task_environment" { variable "ooniapi_service_security_groups" { description = "the shared web security group from the ecs cluster" type = list(string) -} +} \ No newline at end of file