From de83349cbae496ec90f8d06943f874af27f39d2f Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:03:44 +0000 Subject: [PATCH 1/9] Fix tests --- envs/example.env | 8 ++----- .../hospitals/tests/test_viewsets.py | 10 ++++----- rcpch_nhs_organisations/hospitals/urls.py | 2 +- rcpch_nhs_organisations/settings.py | 22 +++---------------- s/django-shell | 3 +++ 5 files changed, 14 insertions(+), 31 deletions(-) create mode 100755 s/django-shell diff --git a/envs/example.env b/envs/example.env index 31866e7..74559ac 100644 --- a/envs/example.env +++ b/envs/example.env @@ -1,9 +1,5 @@ # these env vars are ONLY for development -RCPCH_NHS_ORGANISATIONS_SECRET_KEY=mysecretkey -DJANGO_ALLOWED_HOSTS=0.0.0.0 -DJANGO_CSRF_TRUSTED_ORIGINS=https://localhost,https://0.0.0.0 - # ENVIRONMENT VARIABLES FOR LOCAL DEVELOPMENT ONLY # THIS FILE MUST NEVER BE ADDED TO VERSION CONTROL # SECTIONS IN ALPHABETICAL ORDER ASCENDING @@ -21,10 +17,10 @@ DJANGO_ALLOWED_HOSTS="localhost,127.0.0.1,rcpch-nhs-organisations.localhost" DEBUG="True" # Set DEBUG=True for Local dev and Development, not Staging or Live DJANGO_STARTUP_COMMAND="python manage.py runserver 0.0.0.0:8003" # for local development with auto-reload # DJANGO_STARTUP_COMMAND="gunicorn --bind=0.0.0.0:8003 --timeout 600 rcpch-audit-engine.wsgi" # for live deployment -RCPCH_NHS_ORGANISATIONS_SECRET_KEY="###########" +RCPCH_NHS_ORGANISATIONS_SECRET_KEY="mysecretkey" # DJANGO POSTGRES DATABASE CONNECTION -PGDATABASE="nhsorganisationsdb" +PGDATABASENAME="nhsorganisationsdb" PGUSER="nhsorganisationsuser" PGPASSWORD="password" PGHOST="postgis" diff --git a/rcpch_nhs_organisations/hospitals/tests/test_viewsets.py b/rcpch_nhs_organisations/hospitals/tests/test_viewsets.py index 5fdd965..58cd65e 100644 --- a/rcpch_nhs_organisations/hospitals/tests/test_viewsets.py +++ b/rcpch_nhs_organisations/hospitals/tests/test_viewsets.py @@ -85,24 +85,24 @@ def local_authority_districts(): @pytest.mark.django_db def test_list_local_authority_districts(api_client, local_authority_districts): - url = reverse("local_authority_district-list") + url = reverse("local_authority_districts-list") response = api_client.get(url) assert response.status_code == status.HTTP_200_OK - assert len(response.data["features"]) == 3 + assert len(response.data) == 3 @pytest.mark.django_db def test_within_radius(api_client, local_authority_districts): - url = reverse("local_authority_district-within-radius") + url = reverse("local_authority_districts-within-radius") response = api_client.get( url, {"lat": 53.0, "long": -3.0, "radius": 500000} ) # within 500km assert response.status_code == status.HTTP_200_OK - assert len(response.data["features"]) == 3 + assert len(response.data) == 3 response = api_client.get( url, {"lat": 53.0, "long": -3.0, "radius": 5} ) # within 5 km assert response.status_code == status.HTTP_200_OK - assert len(response.data["features"]) == 0 + assert len(response.data) == 0 diff --git a/rcpch_nhs_organisations/hospitals/urls.py b/rcpch_nhs_organisations/hospitals/urls.py index e075bf6..7d2a54f 100644 --- a/rcpch_nhs_organisations/hospitals/urls.py +++ b/rcpch_nhs_organisations/hospitals/urls.py @@ -49,7 +49,7 @@ router.register( r"local_authority_districts", viewset=LocalAuthorityDistrictViewSet, - basename="london_borough", + basename="local_authority_districts", ) # NHS England Region endpoints router.register( diff --git a/rcpch_nhs_organisations/settings.py b/rcpch_nhs_organisations/settings.py index 6a4119d..8e7b3c4 100644 --- a/rcpch_nhs_organisations/settings.py +++ b/rcpch_nhs_organisations/settings.py @@ -35,28 +35,12 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = os.getenv("DEBUG", "False") == "True" -ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",") + [ - "127.0.0.1", - "localhost", - "0.0.0.0", -] - if os.getenv("WEBSITE_SITE_NAME"): ALLOWED_HOSTS = ["*"] else: - ALLOWED_HOSTS = [ - "127.0.0.1", - "localhost", - "0.0.0.0", - "rcpch-nhs-organisations.localhost", - ] - -CSRF_TRUSTED_ORIGINS = os.getenv("DJANGO_CSRF_TRUSTED_ORIGINS", "").split(",") + [ - "https://rcpch-nhs-organisations.localhost", - "https://127.0.0.1", - "https://localhost", - "https://0.0.0.0", -] + ALLOWED_HOSTS = os.getenv("DJANGO_ALLOWED_HOSTS", "").split(",") + +CSRF_TRUSTED_ORIGINS = os.getenv("DJANGO_CSRF_TRUSTED_ORIGINS", "").split(",") # Enables Django to use the X-Forwarded-Host header in preference to the Host header. # Fixes CSRF errors when using Caddy to forward requests to Django. diff --git a/s/django-shell b/s/django-shell new file mode 100755 index 0000000..fe3ac27 --- /dev/null +++ b/s/django-shell @@ -0,0 +1,3 @@ +#!/bin/bash -e + +docker compose exec -it django python manage.py shell \ No newline at end of file From f4ba580a7b502b4ff531bb0fd6fad37548d67d33 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:14:43 +0000 Subject: [PATCH 2/9] Unpin drf --- requirements/common-requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/common-requirements.txt b/requirements/common-requirements.txt index 28a34ea..4837e65 100644 --- a/requirements/common-requirements.txt +++ b/requirements/common-requirements.txt @@ -4,8 +4,7 @@ python-dateutil==2.9.0.post0 # third party imports django==4.2.17 -# PINNED: see https://github.com/rcpch/rcpch-nhs-organisations/issues/55 -djangorestframework==3.14.0 +djangorestframework==3.15.2 djangorestframework-gis==1.0 drf-spectacular==0.27.1 django-filter==24.1 From 851167ebae70417da3cf7895f80e988a08163e27 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:20:48 +0000 Subject: [PATCH 3/9] Add tests to pr check --- .github/workflows/pr.yml | 26 ++++++++++++++++++++++++++ s/pr-check | 4 ++++ s/test | 2 +- s/up | 4 ++-- 4 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/pr.yml create mode 100755 s/pr-check diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml new file mode 100644 index 0000000..1f1c7bb --- /dev/null +++ b/.github/workflows/pr.yml @@ -0,0 +1,26 @@ +# Runs the Pytest test suite when PRs are made against any branch + +name: Run pr-check script on all PRs + +on: + pull_request: + branches: + - "*" + +jobs: + build-and-run-pytest-suite-in-docker-compose: + runs-on: ubuntu-latest + + steps: + - name: Checkout full code repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # to allow git_context_processor.py to still work + + - name: Create .env file from example + run: cp envs/example.env envs/.env + + - name: Run pr-check script + run: s/pr-check + + diff --git a/s/pr-check b/s/pr-check new file mode 100755 index 0000000..64e3f91 --- /dev/null +++ b/s/pr-check @@ -0,0 +1,4 @@ +#!/bin/bash -e + +s/up -d +s/test \ No newline at end of file diff --git a/s/test b/s/test index dfea5d6..4674186 100755 --- a/s/test +++ b/s/test @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -e # scripts may need to be made executable on some platforms before they can be run # chmod +x is the command to do this on unixy systems diff --git a/s/up b/s/up index 832d894..459b47f 100755 --- a/s/up +++ b/s/up @@ -1,7 +1,7 @@ -#!/bin/bash +#!/bin/bash -e # scripts may need to be made executable on some platforms before they can be run # chmod +x is the command to do this on unixy systems # starts all docker compose services -docker compose up +docker compose up $* From e2f84809a8a96700a7c78e690643dc98a7140838 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:27:22 +0000 Subject: [PATCH 4/9] See if --wait fixes the DNS lookup --- s/pr-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s/pr-check b/s/pr-check index 64e3f91..76f750e 100755 --- a/s/pr-check +++ b/s/pr-check @@ -1,4 +1,4 @@ #!/bin/bash -e -s/up -d +s/up --detach --wait s/test \ No newline at end of file From 8d7eddcc58d02c25937bebbf2ea6870694d6b885 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:31:12 +0000 Subject: [PATCH 5/9] Test if the DNS issue fixes itself after a while --- s/pr-check | 1 + 1 file changed, 1 insertion(+) diff --git a/s/pr-check b/s/pr-check index 76f750e..382ad8e 100755 --- a/s/pr-check +++ b/s/pr-check @@ -1,4 +1,5 @@ #!/bin/bash -e s/up --detach --wait +sleep 10 s/test \ No newline at end of file From 06bde063fdebf84ee03062b7ac50978460984823 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:48:41 +0000 Subject: [PATCH 6/9] Try a real healthcheck? --- docker-compose.yml | 5 +++++ s/pr-check | 1 - 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d660de0..f2453ec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,6 +44,11 @@ services: volumes: - postgis-data:/var/lib/postgresql/data restart: always + healthcheck: + test: pg_isready + interval: 1s + timeout: 5s + retries: 30 mkdocs: <<: *django # this will inherit all the settings from the django service diff --git a/s/pr-check b/s/pr-check index 382ad8e..76f750e 100755 --- a/s/pr-check +++ b/s/pr-check @@ -1,5 +1,4 @@ #!/bin/bash -e s/up --detach --wait -sleep 10 s/test \ No newline at end of file From 1382cc7145542a1f6d59f0ebc3dc129dd02feba7 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 11:51:39 +0000 Subject: [PATCH 7/9] Run interactively to try and get logs for why postgres won't start --- s/pr-check | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/s/pr-check b/s/pr-check index 76f750e..6091abe 100755 --- a/s/pr-check +++ b/s/pr-check @@ -1,4 +1,5 @@ #!/bin/bash -e -s/up --detach --wait +# s/up --detach --wait +s/up s/test \ No newline at end of file From 6447995b291871b288daeae67e62acd5897e6e10 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 12:09:20 +0000 Subject: [PATCH 8/9] Fix missing postgres env vars and add sleep :shrug: --- docker-compose.yml | 5 ----- envs/example.env | 12 +++++++----- rcpch_nhs_organisations/settings.py | 10 +++++----- s/pr-check | 5 +++-- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f2453ec..d660de0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -44,11 +44,6 @@ services: volumes: - postgis-data:/var/lib/postgresql/data restart: always - healthcheck: - test: pg_isready - interval: 1s - timeout: 5s - retries: 30 mkdocs: <<: *django # this will inherit all the settings from the django service diff --git a/envs/example.env b/envs/example.env index 74559ac..3b734dc 100644 --- a/envs/example.env +++ b/envs/example.env @@ -20,11 +20,13 @@ DJANGO_STARTUP_COMMAND="python manage.py runserver 0.0.0.0:8003" # for local dev RCPCH_NHS_ORGANISATIONS_SECRET_KEY="mysecretkey" # DJANGO POSTGRES DATABASE CONNECTION -PGDATABASENAME="nhsorganisationsdb" -PGUSER="nhsorganisationsuser" -PGPASSWORD="password" -PGHOST="postgis" -PGPORT="5432" +POSTGRES_DB="nhsorganisationsdb" +POSTGRES_USER="nhsorganisationsuser" +POSTGRES_PASSWORD="password" +POSTGRES_HOST="postgis" +POSTGRES_PORT="5432" + + # EMAIL EMAIL_DEFAULT_FROM_EMAIL="########" diff --git a/rcpch_nhs_organisations/settings.py b/rcpch_nhs_organisations/settings.py index 8e7b3c4..b4c563a 100644 --- a/rcpch_nhs_organisations/settings.py +++ b/rcpch_nhs_organisations/settings.py @@ -103,11 +103,11 @@ DATABASES = { "default": { "ENGINE": "django.contrib.gis.db.backends.postgis", - "NAME": os.environ.get("PGDATABASENAME"), - "USER": os.environ.get("PGUSER"), - "PASSWORD": os.environ.get("PGPASSWORD"), - "PORT": os.environ.get("PGPORT"), - "HOST": os.environ.get("PGHOST"), + "NAME": os.environ.get("POSTGRES_DB"), + "USER": os.environ.get("POSTGRES_USER"), + "PASSWORD": os.environ.get("POSTGRES_PASSWORD"), + "PORT": os.environ.get("POSTGRES_PORT"), + "HOST": os.environ.get("POSTGRES_HOST"), } } diff --git a/s/pr-check b/s/pr-check index 6091abe..930b512 100755 --- a/s/pr-check +++ b/s/pr-check @@ -1,5 +1,6 @@ #!/bin/bash -e -# s/up --detach --wait -s/up +s/up --detach +# I tried --wait and a healthcheck but still got connection refused from postgres +sleep 5 s/test \ No newline at end of file From 4c1465e0d548cda20999fe150025eef8d2c5c199 Mon Sep 17 00:00:00 2001 From: Michael Barton Date: Mon, 6 Jan 2025 12:12:17 +0000 Subject: [PATCH 9/9] upps forgot lfs --- .github/workflows/pr.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1f1c7bb..982d3b5 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -12,10 +12,9 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout full code repo - uses: actions/checkout@v4 + - uses: actions/checkout@v4 with: - fetch-depth: 0 # to allow git_context_processor.py to still work + lfs: true # ensures large files are downloaded using Git LFS - name: Create .env file from example run: cp envs/example.env envs/.env