Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ocp13 replacement #518

Open
wants to merge 18 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 13 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 11 additions & 13 deletions pillar/registry.sls
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
network:
host_id: ocp13
ipv4: 65.21.93.181
ipv6: 2a01:4f9:3b:45ca::2
host_id: ocp27
ipv4: 37.27.62.45
ipv6: 2a01:4f9:3081:3001::/64
netplan:
template: custom
configuration: |
network:
version: 2
renderer: networkd
ethernets:
enp9s0:
enp5s0:
addresses:
- 65.21.93.181/32
- 2a01:4f9:3b:45ca::2/64
- 37.27.62.45/32
- 2a01:4f9:3081:3001::2/64
routes:
- on-link: true
to: 0.0.0.0/0
via: 65.21.93.129
via: 37.27.62.1
- to: default
via: fe80::1
nameservers:
Expand Down Expand Up @@ -84,9 +84,10 @@ apache:
servername: rabbitmq.data.open-contracting.org

postgres:
version: 12
version: 16
# Public access allows Docker connections. Hetzner's firewall prevents non-local connections.
public_access: True
data_directory: '/data/storage/postgresql/16/main'
configuration:
name: registry
source: shared
Expand All @@ -96,7 +97,7 @@ postgres:
storage: hdd
type: oltp
content: |
data_directory = '/data/storage/postgresql/12/main'
data_directory = '/data/storage/postgresql/16/main'

# Avoid "checkpoints are occurring too frequently" due to intense writes (default 1GB).
max_wal_size = 10GB
Expand All @@ -112,9 +113,6 @@ docker:
uid: 1002
syslog_logging: True

python:
version: '3.10'

kingfisher_collect:
user: collect
group: deployer
Expand All @@ -132,7 +130,7 @@ kingfisher_collect:
docker_apps:
registry:
target: data-registry
exporter_host_dir: /data/storage/exporter_dumps
exporter_host_dir: /data/storage/exporter
env:
DJANGO_PROXY: True
ALLOWED_HOSTS: data.open-contracting.org
Expand Down
7 changes: 3 additions & 4 deletions pillar/registry_maintenance.sls
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,12 @@ maintenance:
enabled: True
patching: manual
rkhunter_customisation: |
SCRIPTWHITELIST=/usr/bin/egrep
SCRIPTWHITELIST=/usr/bin/fgrep
SCRIPTWHITELIST=/usr/bin/which
ALLOWHIDDENFILE=/etc/.resolv.conf.systemd-resolved.bak
ALLOWHIDDENFILE=/etc/.updated
ALLOWDEVFILE=/dev/shm/PostgreSQL.*
PORT_WHITELIST=TCP:60922
DISABLE_TESTS=running_procs
hardware_sensors: True
custom_sensors:
- coretemp
- nct6775
- k10temp
4 changes: 2 additions & 2 deletions salt-config/roster
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,7 @@ prometheus:
redmine:
host: ocp16.open-contracting.org
registry:
host: ocp13.open-contracting.org
port: 2223
host: ocp27.open-contracting.org
portland-dev:
host: ocp26.open-contracting.org
user: ocpadmin
Expand All @@ -43,4 +42,5 @@ portland-dev:
# ocp10 was kingfisher-archive (archive.kingfisher.open-contracting.org)
# ocp11 was covid19 (www.open-contracting.health)
# ocp12 was spoonbill-dev
# ocp13 was registry on Ubuntu 20
# ocp14 was redash (redash.open-contracting.org)
2 changes: 2 additions & 0 deletions salt/aws/init.sls
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ awscli:
- name: python3-pip
pip.installed:
- name: awscli
- extra_args:
- --break-system-packages
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment about why this option is necessary?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is required on systems with Python 3.11 and newer.
I have added a comment and a Jinja if statement so it only runs where needed.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this flag also needed on the pip.installed state in kingfisher/init.sls?

- require:
- pkg: awscli

Expand Down
2 changes: 1 addition & 1 deletion salt/core/rsyslog/files/docker.conf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Define the socket input
input(type="imuxsock" Socket="/var/spool/docker-custom.sock")
input(type="imuxsock" Socket="/var/spool/rsyslog/docker-custom.sock")

# Define a template for dynamic file names based on the Docker image name
# https://www.rsyslog.com/doc/configuration/templates.html#string
Expand Down
2 changes: 1 addition & 1 deletion salt/docker/files/daemon-logging.json
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
{
"log-driver": "syslog",
"log-opts": {
"syslog-address": "unixgram:///var/spool/docker-custom.sock",
"syslog-address": "unixgram:///var/spool/rsyslog/docker-custom.sock",
"tag": "docker-custom-{{.Name}}"
}
}
2 changes: 1 addition & 1 deletion salt/pelican/backend/init.sls
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ btree_gin:
# curl -sSf https://raw.githubusercontent.com/open-contracting/pelican-backend/main/pelican/migrations/002_constraints.sql | shasum -a 256
{%
for basename, source_hash in [
('001_base', 'c4b65862980146d0ba88e437b1dd129c5c641597656dcca6b89cfe4ecb7979df'),
('001_base', 'b6f2c25da154e1b4b8b55e1231039c84b4c0c3edab5d1c4c9e7dbd402b25ca36'),
('002_constraints', 'f298f0b8cb20d47f390b480d44d12c097e83b177dde56234dcbebc6ad3dcf229'),
]
%}
Expand Down
19 changes: 19 additions & 0 deletions salt/postgres/init.sls
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,25 @@ postgresql-reload:
{{ apache('postgres', {'configuration': 'default', 'servername': pillar.postgres.ssl.servername}) }}
{% endif %}

{% if salt['pillar.get']('postgres:data_directory') %}
{{ pillar.postgres.data_directory }}:
file.directory:
- name: {{ pillar.postgres.data_directory }}
- user: postgres
- group: postgres
- makedirs: True
- require:
- pkg: postgresql
cmd.run:
- name: /usr/lib/postgresql/{{ pillar.postgres.version }}/bin/initdb -D {{ pillar.postgres.data_directory }}
- runas: postgres
- creates: {{ pillar.postgres.data_directory }}/PG_VERSION
- require:
- pkg: postgresql
- watch_in:
- service: postgresql
{% endif %}

{% if pillar.postgres.configuration %}
/etc/postgresql/{{ pillar.postgres.version }}/main/conf.d/030_{{ pillar.postgres.configuration.name }}.conf:
file.managed:
Expand Down
2 changes: 1 addition & 1 deletion salt/prometheus/files/conf-prometheus.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ scrape_configs:
'kingfisher-main': 'ocp23.open-contracting.org',
'ocds-live.docs': 'ocp19.open-contracting.org',
'prometheus-server-node': 'ocp20.open-contracting.org',
'data-registry': 'ocp13.open-contracting.org',
'data-registry': 'ocp27.open-contracting.org',
'redmine': 'ocp16.open-contracting.org',
}|items %}
- job_name: '{{ job_name }}'
Expand Down
2 changes: 2 additions & 0 deletions salt/python/extensions.sls
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,5 @@ python c extensions:
- python{{ salt['pillar.get']('python:version', 3) }}-dev
- build-essential
- libffi-dev
- libxml2-dev
- libxslt1-dev