major: merge #180 from dev

The breaking changes are in build systems; the ansible has been extensively altered to provision ubuntu 22.04 instead of centos 7. The codebase is largely the same.

.cz.toml

@@ -0,0 +1,10 @@
+[tool.commitizen]
+name = "cz_nhm"
+version = "1.0.0"
+tag_format = "v$version"
+update_changelog_on_bump = true
+changelog_incremental = true
+version_files = [
+    "api/pyproject.toml:version",
+    "web/package.json:version"
+]

.github/workflows/bump.yml

@@ -0,0 +1,31 @@
+name: Bump version
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  bump-version:
+    if: "!startsWith(github.event.head_commit.message, 'bump:')"
+    runs-on: ubuntu-latest
+    name: "Bump version and create changelog"
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+      - name: Create bump and changelog
+        uses: commitizen-tools/commitizen-action@master
+        with:
+          github_token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          changelog_increment_filename: CURRENT.md
+          extra_requirements: "cz-nhm"
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        with:
+          body_path: "CURRENT.md"
+          tag_name: v${{ env.REVISION }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}

.github/workflows/sync.yml

@@ -0,0 +1,22 @@
+name: Sync branches
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  sync-branches:
+    runs-on: ubuntu-latest
+    name: "Sync dev branch to latest commit"
+    steps:
+      - name: Check out
+        uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+          fetch-depth: 0
+      - name: Sync dev branch
+        uses: connor-baer/action-sync-branch@main
+        with:
+          branch: dev
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}

.pre-commit-config.yaml

@@ -0,0 +1,34 @@
+exclude: /(vendor|dist)/
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.3.0
+    hooks:
+      - id: check-merge-conflict
+      - id: detect-private-key
+      - id: end-of-file-fixer
+      - id: name-tests-test
+        args: ["--pytest-test-first"]
+        exclude: ^tests/helpers/
+      - id: trailing-whitespace
+  - repo: https://github.com/commitizen-tools/commitizen
+    rev: v2.37.0
+    hooks:
+      - id: commitizen
+        additional_dependencies: ["cz-nhm"]
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black
+        args: ["--line-length=88", "--skip-string-normalization"]
+  - repo: https://github.com/PyCQA/docformatter
+    rev: v1.5.0
+    hooks:
+      - id: docformatter
+        args: ["-i", "--wrap-summaries=88", "--wrap-descriptions=88",
+               "--pre-summary-newline", "--make-summary-multi-line"]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v3.0.0-alpha.4
+    hooks:
+      - id: prettier
+        types_or: [ javascript, vue, less, sass, scss, css ]
+        args: [ '--single-quote' ]

README.md

@@ -1,99 +1,72 @@
-# Phenome 10K
+# Phenome10k
 
-## Development
-
-How to get it working:
+This is the repository for the code behind the website [phenome10k.org](https://www.phenome10k.org).
 
-First, make sure you have the following binaries available:
- - Python 3.8
- - Node 14.5
- - NPM 6.14
+***
 
-To create and/or activate the virtual environment:
-```bash
-source activate
-```
+## Development
 
-This basically just runs the venv scripts:
+The development environment uses a [Docker Compose](https://docs.docker.com/compose) configuration. This does not completely reflect the production server configuration.
 
-```bash
-# Create env
-python3.8 -m venv venv
-# Activate env
-source venv/bin/activate
+To run for the first time:
+```shell
+RUN_INIT=TRUE docker compose up -d
 ```
 
-To install requirments and run:
+On subsequent runs, `RUN_INIT` can be omitted.
 
-```bash
-# Install dependencies
-pip install -r requirements.txt
-npm i
-# Run database migrations
-flask db upgrade
-# Create the admin account
-flask set-admin-pw pass
-# build the front end
-npm build
-# Start the front end
-npm start &
-# Start the task queue
-scripts/tasks.sh &
-# Start
-flask run
-```
+An admin user will be created with the username/email `admin` and the password `pass`.
 
-If you want to run in developer/debug mode (with hot reload for both servers):
-
-```bash
-npm watch &
-FLASK_DEBUG=1 flask run
-```
+### Tests
 
-`npm start` has to be restarted every time the frontend is rebuilt. If you are making frequent changes, `npm watch` is much easier as it handles both the bundling _and_ the server restarts.
+Test coverage is very limited.
 
-The default user for a new site has the email address `admin` and the username `pass`.
-Make sure to change this!
+To run NodeJS tests (using [Jest](https://jestjs.io)):
+```shell
+docker compose exec app-node npm run test
+```
 
-To run tests with coverage, use the `tests` script, or use `pytest` with custom arguments.
+***
 
 ## Local staging
 
-Before running ansible you'll need to do a few things:
+There is a [Vagrant](https://www.vagrantup.com) configuration that more accurately reflects the production server setup. It's provisioned using ansible.
+
+As a staging environment, it requires some production values (e.g. hcaptcha keys) and can therefore only be run by individuals with access to certain Natural History Museum services.
 
-0. Make sure you're connected to the NHM internal network
-1. Generate a Vault token:
-   ```bash
-   source vault-token
+### Setup
+1. Install `hvac`: this is a Python library that needs to be installed with the same interpreter that ansible uses (probably your root/system Python)
+2. Generate a vault token:
+   ```shell
+   python ansible/get-token.py
    ```
-2. ```bash
-   pip install hvac
+3. Export the token to apply it to your current shell environment:
+   ```shell
+   export VAULT_TOKEN=your-token-here
    ```
+   Note that if you close your current shell, you'll have to do this again before Vagrant will work.
+4. Add `192.168.10.21   phenome10k.localhost` to `/etc/hosts`.
 
-You can run a local staging environment by spining up Vagrant:
-
-```
-vagrant up
+### Running Vagrant
+```shell
+vagrant up --no-parallel
 ```
 
-Edit your `/etc/hosts` file to direct `phenome10k.localhost` to `192.168.10.21`.
-The site will be available at phenome10k.localhost, and should reflect the production configuration
-of the service.
+The `--no-parallel` is particularly important if your system uses `libvirt` rather than Virtualbox.
+
+The site will be available at [phenome10k.localhost](http://phenome10k.localhost), and should reflect the production configuration of the service.
+
+***
 
 ## Production
 
 The site's infrastructure is an HA architecture is composed of:
  - 2+ load balancers in active-passive HA configuration
  - 2+ application servers
- - 1 data server (db and nfs), to be replaced in future with cloud-based storage
+ - 1 data server (db and nfs)
 
 The system can be deployed to production via ansible:
 
 ```bash
-ansible-playbook -iansible/inventories/production.ini ansible/playbook.yml -e@ansible/group_vars/production/main.yml -k -K  -ua-paulk6
+ansible-playbook ansible/playbook.yml -i ansible/inventories/production.ini -k -K --user YOUR-USERNAME
 ```
-
-
-## Adding database migrations
-
-Add or modify models in models.py then run `flask db revision`

Vagrantfile

@@ -11,13 +11,13 @@ mysql_password = "A1a2a_"
 # backwards compatibility). Please don't change it unless you know what
 # you're doing.
 Vagrant.configure("2") do |config|
-  nfs_ip = "192.168.10.5"
+  data_ip = "192.168.10.5"
 
   config.vm.synced_folder ".", "/vagrant", disabled: true
 
   config.vm.define "data" do |config|
-    config.vm.box = "centos/7"
-    config.vm.network "private_network", ip: nfs_ip
+    config.vm.box = "generic/ubuntu2204"
+    config.vm.network "private_network", ip: data_ip
     config.vbguest.no_install = true
   end
 
@@ -29,7 +29,7 @@ Vagrant.configure("2") do |config|
   app_ips.each_with_index do |app_ip, n|
     vmName = "app#{n + 1}"
     config.vm.define vmName do |config|
-      config.vm.box = "centos/7"
+      config.vm.box = "generic/ubuntu2204"
       config.vm.hostname = vmName
       config.vm.network "private_network", ip: app_ip
 
@@ -44,24 +44,31 @@ Vagrant.configure("2") do |config|
     end
   end
 
-  [1, 2].each do |n|
-    vmName = "lb#{n}"
+  lb_ips = [
+    "192.168.10.11",
+    "192.168.10.12"
+  ]
+
+  lb_ips.each_with_index do |lb_ip, n|
+    vmName = "lb#{n + 1}"
     config.vm.define vmName do |config|
-      config.vm.box = "centos/7"
-      config.vm.network "private_network", ip: "192.168.10.1#{n}"
+      config.vm.box = "generic/ubuntu2204"
+      config.vm.network "private_network", ip: lb_ip
       config.vbguest.no_install = true
     end
   end
 
   config.vm.provision :hosts, :sync_hosts => true
   config.vm.provision "ansible" do |ansible|
+    ansible.compatibility_mode = "2.0"
     ansible.playbook = "ansible/playbook.yml"
     ansible.groups = {
       "app" => ["app1", "app2"],
       "lb" => ["lb1", "lb2"],
       "all:vars" => {
-        "nfs_server" => nfs_ip,
-        "app_servers" => app_ips
+        "data_server" => data_ip,
+        "app_servers" => app_ips,
+        "lb_servers" => lb_ips
       }
     }
     ansible.host_vars = {

ansible/get-token.py

@@ -2,7 +2,7 @@
 
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
-from urllib.error import HTTPError
+from urllib.error import HTTPError, URLError
 import json
 import sys
 import getpass
@@ -11,34 +11,39 @@
 user = input()
 pswd = getpass.getpass('Vault LDAP password: ')
 
-vault_url = 'https://man-vault-2.nhm.ac.uk:8200/v1/auth/ldap/login/' + user # Set destination URL here
-post_fields = {'password': pswd}     # Set POST fields here
+vault_url = (
+    'https://man-vault-2.nhm.ac.uk:8200/v1/auth/ldap/login/' + user
+)  # Set destination URL here
+post_fields = {'password': pswd}  # Set POST fields here
 
 request = Request(vault_url, json.dumps(post_fields).encode())
 try:
-  resp = urlopen(request).read().decode()
-  auth = json.loads(resp)['auth']
-  token = auth['client_token']
-  duration = auth['lease_duration']
-  unit = 'seconds'
-  if duration >= 60:
-  	duration = duration / 60
-  	unit = 'minutes'
+    resp = urlopen(request).read().decode()
+    auth = json.loads(resp)['auth']
+    token = auth['client_token']
+    duration = auth['lease_duration']
+    unit = 'seconds'
+    if duration >= 60:
+        duration = duration / 60
+        unit = 'minutes'
 
-  	if duration >= 60:
-  		duration = duration / 60
-  		unit = 'hours'
+        if duration >= 60:
+            duration = duration / 60
+            unit = 'hours'
 
-  		if duration >= 24:
-  			duration = duration / 24
-  			unit = 'days'
-  print(f'Got token OK. It\'s valid for {duration:g} {unit}.', file=sys.stderr)
-  print(token)
+            if duration >= 24:
+                duration = duration / 24
+                unit = 'days'
+    print(f'Got token OK. It\'s valid for {duration:g} {unit}.', file=sys.stderr)
+    print(token)
 except HTTPError as e:
-  print(f'Failed to get a token, sorry. Vault says:', file=sys.stderr)
-  resp = e.read().decode()
-  errors = json.loads(resp)['errors']
-  sys.exit('\n'.join(errors))
+    print(f'Failed to get a token, sorry. Vault says:', file=sys.stderr)
+    resp = e.read().decode()
+    errors = json.loads(resp)['errors']
+    sys.exit('\n'.join(errors))
 except URLError as e:
-  print(f'Could not connect to vault server. Are you on the museum network?', file=sys.stderr)
-  sys.exit(e)
+    print(
+        f'Could not connect to vault server. Are you on the museum network?',
+        file=sys.stderr,
+    )
+    sys.exit(e)