Merge pull request #404 from open-contracting/cove-ubuntu-20

cove-oc4ids Ubuntu upgrade
open-contracting · Feb 24, 2023 · 304a9d1 · 304a9d1
2 parents 1eb2d5b + 94cab10
commit 304a9d1
Show file tree

Hide file tree

Showing 26 changed files with 160 additions and 58 deletions.
diff --git a/docs/develop/update/apache.rst b/docs/develop/update/apache.rst
@@ -53,7 +53,6 @@ Add to your service's Pillar file:
          serveraliases: ['live.standard.open-contracting.org']
          context:
            ocds_cove_backend: https://cove.live3.cove.opencontracting.uk0.bigv.io
-           oc4ids_cove_backend: https://cove-live.oc4ids.opencontracting.uk0.bigv.io
            timeout: 1830  # 30 sec longer than cove's uwsgi.harakiri
 
 This will:

diff --git a/docs/develop/update/network.rst b/docs/develop/update/network.rst
@@ -15,27 +15,30 @@ Update the server's Pillar file:
 
 ``ipv6`` is optional.
 
-Netplan
--------
+Linux networking
+----------------
 
-`Netplan <https://netplan.io>`__ uses YAML files for configuration. Configurations are available for Linode and other hosts. The configuration is written to ``/etc/netplan/10-salt-networking.yaml``.
+systemd-networkd
+~~~~~~~~~~~~~~~~
+
+`systemd-networkd <https://manpages.ubuntu.com/manpages/jammy/man5/systemd.network.5.html>`__ is a system daemon to configure networking, and is our preferred solution for Linode instances. Configurations are available for `Linode`_ and other hosts. The configuration is written to ``/etc/systemd/network/05-eth0.network``.
 
-Linode
-~~~~~~
+Linode template
+^^^^^^^^^^^^^^^
 
 This configuration disables automatic IP configuration and configures static networking on IPv4 and IPv6.
 
 .. note::
 
    By default, a Linode server listens on – and prefers traffic to – its default IPv6 address. We use our own IPv6 block – ``2a01:7e00:e000:02cc::/64`` – to improve IP reputation and email deliverability.
 
-.. admonition:: One-time setup
+.. admonition::
 
-   `Open a support ticket with Linode <https://cloud.linode.com/support/tickets>`__ to request an IPv6 /64 block:
+   `Open a support ticket with Linode <https://cloud.linode.com/support/tickets>`__ to request an IPv6 /64 block, replacing ``ocpXX`` with the Linode instance's ID:
 
       Hello,
 
-      Please provision an IPv6 /64 block for our account.
+      Please can you provision an IPv6 /64 block for my server ocpXX.open-contracting.org.
 
       Thank you,
 
@@ -49,28 +52,52 @@ Update the server's Pillar file:
      host_id: ocp12
      ipv4: 198.51.100.34
      ipv6: 2001:db8::12
-     netplan:
+     networkd:
        template: linode
        addresses:
-         - 2001:db8::32/64 # SLAAC
+         - 2001:db8::/64
        gateway4: 198.51.100.1
-       gateway6: fe80::1
 
 To fill in the above, from the *Network* tab on the `Linode's <https://cloud.linode.com/linodes>`__ page, collect:
 
 ``ipv4``
   The *Address* with a *Type* of *IPv4 – Public*
-``addresses``
-  The *Address* with a *Type* of *IPv6 – SLAAC*, `suffixed by "/64" <https://www.linode.com/docs/guides/linux-static-ip-configuration/#general-information>`__
 ``gateway4``
   The *Default Gateway* with a *Type* of *IPv4 – Public*
-``gateway6``
-  The *Default Gateway* with a *Type* of *IPv6 – SLAAC*
+``addresses``
+  Other IP addresses attached to your instance (if any). Include the subnet block suffix, e.g.: `/64`
+
+Custom template
+^^^^^^^^^^^^^^^
+
+.. warning::
+
+   Only use a ``custom`` template if your needs are not met by any other template.
 
-For ``ipv6``, use our IPv6 block with the hostname's digits as the final group of the IPv6 address: for example, *2a01:7e00:e000:02cc::12* for *ocp12*.
+In the server's Pillar file, set ``network.networkd.template`` to ``custom`` and set ``network.networkd.configuration``:
 
-Other hosting providers
-~~~~~~~~~~~~~~~~~~~~~~~
+.. code-block:: yaml
+
+   [Match]
+   Name=eth0
+
+   [Network]
+   DHCP=no
+   DNS=203.0.113.1 203.0.113.2 2001:db8::32 2001:db8::64
+   Domains=open-contracting.org
+   IPv6PrivacyExtensions=false
+   IPv6AcceptRA=false
+
+   Address=198.51.100.34/24
+   Address=2001:db8::12/64
+
+   Gateway=Address=198.51.100.1
+   Gateway=fe80::1
+
+Netplan
+~~~~~~~
+
+`Netplan <https://netplan.io>`__ uses YAML files for configuration. Configurations are available for Linode and other hosts. The configuration is written to ``/etc/netplan/10-salt-networking.yaml``.
 
 .. note::
 

diff --git a/docs/reference/docs.rst b/docs/reference/docs.rst
@@ -6,7 +6,7 @@ This page serves as an orientation to how different components of the OCDS docum
 Servers
 -------
 
-The ``docs`` target serves OCDS documentation (e.g. `1.1 <https://standard.open-contracting.org/1.1/>`__), its profiles (e.g. `Public Private Partnerships <https://standard.open-contracting.org/profiles/ppp/latest/en/>`__) and draft documentation (below). It is a reverse proxy to the `OCDS Data Review Tool <https://standard.open-contracting.org/review/>`__ and the `OC4IDS Data Review Tool <https://standard.open-contracting.org/infrastructure/review/>`__. It also serves Elasticsearch.
+The ``docs`` target serves OCDS documentation (e.g. `1.1 <https://standard.open-contracting.org/1.1/>`__), its profiles (e.g. `Public Private Partnerships <https://standard.open-contracting.org/profiles/ppp/latest/en/>`__) and draft documentation (below). It is a reverse proxy to the `OCDS Data Review Tool <https://standard.open-contracting.org/review/>`. It also serves Elasticsearch.
 
 Version and language switchers
 ------------------------------

diff --git a/pillar/cove_oc4ids.sls b/pillar/cove_oc4ids.sls
@@ -1,3 +1,11 @@
+network:
+  host_id: ocp17
+  ipv4: 176.58.112.127
+  ipv6: "2a01:7e00:e000:04c1::"
+  networkd:
+    template: linode
+    gateway4: 176.58.112.1
+
 ssh:
   root:
     # Open Data Services Co-operative
@@ -10,9 +18,9 @@ python_apps:
       url: https://github.com/open-contracting/cove-oc4ids.git
     django:
       env:
-        ALLOWED_HOSTS: .standard.open-contracting.org,.oc4ids.opencontracting.uk0.bigv.io
+        ALLOWED_HOSTS: review-oc4ids.standard.open-contracting.org
         FATHOM_ANALYTICS_ID: UHUGOEOK
     apache:
-      servername: cove-live.oc4ids.opencontracting.uk0.bigv.io
+      servername: review-oc4ids.standard.open-contracting.org
       context:
         assets_base_url: /infrastructure
diff --git a/pillar/cove_oc4ids_maintenance.sls b/pillar/cove_oc4ids_maintenance.sls
@@ -0,0 +1,10 @@
+maintenance:
+  enabled: True
+  patching: manual
+  rkhunter_customisation: |
+    SCRIPTWHITELIST=/usr/bin/egrep
+    SCRIPTWHITELIST=/usr/bin/fgrep
+    SCRIPTWHITELIST=/usr/bin/which.debianutils
+    RTKT_FILE_WHITELIST=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9
+    USER_FILEPROP_FILES_DIRS=/usr/lib/x86_64-linux-gnu/libkeyutils.so.1.9
+    ALLOW_SSH_ROOT_USER=yes
diff --git a/pillar/cove_ocds.sls b/pillar/cove_ocds.sls
@@ -1,13 +1,21 @@
+network:
+  host_id: ocp18
+  ipv4: 176.58.107.239
+  # ipv6: "2a01:7e00:e000:04c1::18"
+  networkd:
+    template: linode
+    gateway4: 176.58.107.1
+
 python_apps:
   cove: # adds to cove.sls
     git:
       url: https://github.com/open-contracting/cove-ocds.git
     django:
       env:
-        ALLOWED_HOSTS: .standard.open-contracting.org,.cove.opencontracting.uk0.bigv.io
+        ALLOWED_HOSTS: review.standard.open-contracting.org
         FATHOM_ANALYTICS_ID: PPQKEZDX
         # HOTJAR_ID: 1501232
         # HOTJAR_SV: 6
         # HOTJAR_DATE_INFO: "4th March to 30th September 2020"
     apache:
-      servername: cove.live3.cove.opencontracting.uk0.bigv.io
+      servername: review.standard.open-contracting.org
diff --git a/pillar/docs.sls b/pillar/docs.sls
@@ -17,7 +17,6 @@ apache:
       servername: standard.open-contracting.org
       context:
         ocds_cove_backend: https://cove.live3.cove.opencontracting.uk0.bigv.io
-        oc4ids_cove_backend: https://cove-live.oc4ids.opencontracting.uk0.bigv.io
         timeout: 1830  # 30 sec longer than cove's uwsgi.harakiri
 
 elasticsearch:

diff --git a/pillar/redash.sls b/pillar/redash.sls
@@ -7,7 +7,6 @@ network:
     addresses:
       - 2a01:7e00::f03c:92ff:fea5:0e5f/64 # SLAAC
     gateway4: 139.162.199.1
-    gateway6: fe80::1
 
 vm:
   nr_hugepages: 128

diff --git a/pillar/redmine.sls b/pillar/redmine.sls
@@ -7,7 +7,6 @@ network:
     addresses:
       - 2a01:7e00::f03c:93ff:fe24:07b9/64 # SLAAC
     gateway4: 139.162.219.1
-    gateway6: fe80::1
 
 apache:
   public_access: True

diff --git a/pillar/top.sls b/pillar/top.sls
@@ -8,6 +8,7 @@ base:
   'cove-oc4ids':
     - cove
     - cove_oc4ids
+    - cove_oc4ids_maintenance
     - private.cove_oc4ids
     - maintenance
 

diff --git a/salt-config/roster b/salt-config/roster
@@ -1,7 +1,7 @@
 # Defines the potential targets of the salt-ssh command.
 
-cove-oc4ids:         ocp01.open-contracting.org
-cove-ocds:           ocp02.open-contracting.org
+cove-oc4ids:         ocp17.open-contracting.org
+cove-ocds:           ocp18.open-contracting.org
 docs:                ocp07.open-contracting.org
 kingfisher-process:  ocp04.open-contracting.org
 kingfisher-replica:  ocp05.open-contracting.org
@@ -13,6 +13,7 @@ registry:
   port: 2223
 
 # ocp0 was crm.open-contracting.org
+# ocp01 was cove-oc4ids on Ubuntu 18
 # ocp06 was covid19-dev
 # ocp08 was redash.open-contracting.org on Ubuntu 18
 # ocp09 was toucan.open-contracting.org

diff --git a/salt/apache/files/docs/robots.txt b/salt/apache/files/docs/robots.txt
@@ -10,7 +10,6 @@ Disallow: /legacy
 Disallow: /staging
 # Disallow Data Review Tool results.
 Disallow: /review/data
-Disallow: /infrastructure/review/data
 
 User-Agent: LinkChecker
 Allow: /
diff --git a/salt/apache/files/sites/docs.conf.include b/salt/apache/files/sites/docs.conf.include
@@ -27,7 +27,6 @@
             'versions': ['latest', '0.9'],
             'languages': ['en'],
             'redirect_excludes': ['schema'],
-            'cove_backend': oc4ids_cove_backend,
         },
         '': {
             'versions': ['latest', '1.1', '1.0'],
@@ -220,6 +219,9 @@ Redirect /profiles/gpa/master /profiles/gpa/latest
 # The Validator was renamed the Data Review Tool.
 Redirect /validator /review
 RedirectMatch ^/validator/(.*)$ /review/$1
+# The Data Review Tool was moved to its own subdomain.
+Redirect /infrastructure/review /review https://review-oc4ids.standard.open-contracting.org
+RedirectMatch ^/infrastructure/review(.*)$ https://review-oc4ids.standard.open-contracting.org/$1
 
 # See: https://crm.open-contracting.org/issues/4401
 {% for lang in options[''].languages %}

diff --git a/salt/apache/init.sls b/salt/apache/init.sls
@@ -11,18 +11,22 @@
 # ondrej/apache2 is still needed on Ubuntu 20.04 for MDContactEmail.
 # https://github.com/icing/mod_md/issues/203
 apache2:
+  {%- if grains.osmajorrelease in ("18", "20") %}
   pkgrepo.managed:
     - ppa: ondrej/apache2
+  {%- endif %}
   pkg.installed:
     - pkgs:
       - apache2
+    {%- if grains.osmajorrelease in ("18", "20") %}
       # Avoid "AH01882: Init: this version of mod_ssl was compiled against a newer library (OpenSSL 1.1.1g 21 Apr 2020,
       # version currently loaded is OpenSSL 1.1.1 11 Sep 2018) - may result in undefined or erroneous behavior"
       # https://github.com/open-contracting/deploy/issues/66#issuecomment-742898193
       - libssl1.1
       - openssl
     - require:
       - pkgrepo: apache2
+    {%- endif %}
   service.running:
     - name: apache2
     - enable: True

diff --git a/salt/core/apt/init.sls b/salt/core/apt/init.sls
@@ -7,3 +7,7 @@
 /etc/apt/apt.conf.d/99-connection-timeouts:
   file.managed:
     - source: salt://core/apt/files/99-connection-timeouts
+
+needrestart:
+  pkg.removed:
+    - name: needrestart
diff --git a/salt/core/firewall/files/firewall.sh b/salt/core/firewall/files/firewall.sh
@@ -64,31 +64,20 @@ else
 fi
 
 echo_verbose "Get iptables location"
-if [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "20.04" ]; then
+case "${ID}_${VERSION_ID}" in
+ubuntu_22.04 | ubuntu_20.04 | ubuntu_18.04 | debian_10 | debian_9 | debian_8)
     IPTABLESSAVLOC=/etc/iptables/rules.v4
     IP6TABLESSAVLOC=/etc/iptables/rules.v6
-elif [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "18.04" ]; then
-    IPTABLESSAVLOC=/etc/iptables/rules.v4
-    IP6TABLESSAVLOC=/etc/iptables/rules.v6
-elif [ "$ID" == "ubuntu" ] && [ "$VERSION_ID" == "16.04" ]; then
-    IPTABLESSAVLOC=/etc/iptables/rules.v4
-    IP6TABLESSAVLOC=/etc/iptables/rules.v6
-elif [ "$ID" == "debian" ] && [ "$VERSION_ID" == "8" ]; then
-    IPTABLESSAVLOC=/etc/iptables/rules.v4
-    IP6TABLESSAVLOC=/etc/iptables/rules.v6
-elif [ "$ID" == "debian" ] && [ "$VERSION_ID" == "7" ]; then
-    IPTABLESSAVLOC=/etc/iptables/rules.v4
-    IP6TABLESSAVLOC=/etc/iptables/rules.v6
-elif [ "$ID" == "centos" ] && [ "$VERSION_ID" == "7" ]; then
-    IPTABLESSAVLOC=/etc/sysconfig/iptables
-    IP6TABLESSAVLOC=/etc/sysconfig/ip6tables
-elif [ "$ID" == "redhat-derivative" ]; then
+    ;;
+centos_7 | redhat-derivative_)
     IPTABLESSAVLOC=/etc/sysconfig/iptables
     IP6TABLESSAVLOC=/etc/sysconfig/ip6tables
-else
+    ;;
+*)
     echo "Failed to determine iptables location! Please update this script appropriately."
     exit 7
-fi
+    ;;
+esac
 
 if [ "$MONITOR_APPBEAT" == "yes" ]; then
     echo_verbose "Get AppBeat IP addresses"

diff --git a/salt/core/init.sls b/salt/core/init.sls
@@ -9,9 +9,9 @@ git:
     - name: git
 
 # Required for salt to interact with apt.
-python-apt:
+python3-apt:
   pkg.installed:
-    - name: python-apt
+    - name: python3-apt
 
 # Required for some targets.
 debconf-utils:

diff --git a/salt/core/network/files/netplan_linode.yaml b/salt/core/network/files/netplan_linode.yaml
@@ -24,7 +24,7 @@ network:
         - {{ pillar.network.ipv6 }}/64
         {%- endif %}
       gateway4: {{ pillar.network.netplan.gateway4 }}
-      gateway6: {{ pillar.network.netplan.gateway6 }}
+      gateway6: fe80::1
       nameservers:
         search: ['open-contracting.org']
         addresses: ['178.79.182.5', '176.58.107.5', '176.58.116.5', '176.58.121.5', '151.236.220.5', '212.71.252.5', '212.71.253.5', '109.74.192.20', '109.74.193.20', '109.74.194.20', '2a01:7e00::9', '2a01:7e00::3', '2a01:7e00::c', '2a01:7e00::5', '2a01:7e00::6', '2a01:7e00::8', '2a01:7e00::b', '2a01:7e00::4', '2a01:7e00::7', '2a01:7e00::2']
diff --git a/salt/core/network/files/networkd_custom.network b/salt/core/network/files/networkd_custom.network
@@ -0,0 +1 @@
+{{ pillar.network.networkd.configuration }}
diff --git a/salt/core/network/files/networkd_linode.network b/salt/core/network/files/networkd_linode.network
@@ -0,0 +1,20 @@
+[Match]
+Name=eth0
+
+[Network]
+DHCP=no
+DNS=178.79.182.5 176.58.107.5 176.58.116.5 176.58.121.5 151.236.220.5 212.71.252.5 212.71.253.5 109.74.192.20 109.74.193.20 109.74.194.20 2a01:7e00::9 2a01:7e00::3 2a01:7e00::c 2a01:7e00::5 2a01:7e00::6 2a01:7e00::8 2a01:7e00::b 2a01:7e00::4 2a01:7e00::7 2a01:7e00::2
+Domains=open-contracting.org
+IPv6PrivacyExtensions=false
+IPv6AcceptRA=false
+
+Address={{ pillar.network.ipv4 }}/24
+{%- if 'ipv6' in pillar.network %}
+Address={{ pillar.network.ipv6 }}/64
+{%- endif %}
+{%- for address in pillar.network.networkd.get('addresses', []) %}
+Address={{ address }}
+{%- endfor %}
+
+Gateway={{ pillar.network.networkd.gateway4 }}
+Gateway=fe80::1
diff --git a/salt/core/network/init.sls b/salt/core/network/init.sls
@@ -34,7 +34,22 @@ set hostname:
     - onchanges:
       - file: /etc/mailname
 
-{%- if 'netplan' in pillar.network %}
+{%- if 'networkd' in pillar.network %}
+/etc/netplan/01-netcfg.yaml:
+  file.absent
+
+/etc/netplan/01-eth0.yaml:
+  file.absent
+
+/etc/systemd/network/05-eth0.network:
+  file.managed:
+    - source: salt://core/network/files/networkd_{{ pillar.network.networkd.template }}.network
+    - template: jinja
+
+systemd-networkd:
+  service.enabled:
+    - name: systemd-networkd
+{%- elif 'netplan' in pillar.network %}
 /etc/netplan/01-netcfg.yaml:
   file.absent