Tp2000 1327 upgrade python 3.12 (#1246)

* gitignore all venv* directories. * Bump to Python version 3.10.x Increment version of Python from 3.8.x to 3.10.x where necessary. Some package dependency changes also applied - e.g. removing unnecessary backports. * Apply black formatting flagged in CI. * version upgrade to 3.11 * remove django-fakeredis and use fakeredis config * updated comments * alphabetical order :) * friday mistakes * upgrade elastic-apm * dependency fun * got the wrong version * downgrading elastic apm * upgrade boto3 * use correct version * trying an older boto3 * Python 3.12 * Update lxml * Upgrade moto to solve ssl wrap_socket error * Update responses * Lower moto version and rollback responses * Try moto 3.0.7 * Change coverage version * Try updating pytest * Undo pytest upgrade * Update coverage to 7.4.0 * Make xml bomb bigger * Update readme * Add indent * Fix error in workbasket test * Remove test, update pytest * Try boto upgrade to fix botocore packaging issue * Try lower boto version * Try downgrade elastic-apm * Update requirements.txt Undoing last commit * Try elastic-apm 6.15.1 * Add message to requirements.txt * Update .pre-commit-config.yaml * Remove uility function for 3.8 * Black * Update pii-secret-check-hooks --------- Co-authored-by: Paul Pepper <paul.pepper@digital.trade.gov.uk> Co-authored-by: Paul Pepper <85895113+paulpepper-trade@users.noreply.github.com> Co-authored-by: Anthoni Gleeson <anthoni.gleeson@digital.trade.gov.uk>
uktrade · Jul 23, 2024 · 8d07ef2 · 8d07ef2
1 parent 5624528
commit 8d07ef2
Show file tree

Hide file tree

Showing 16 changed files with 39 additions and 62 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -36,22 +36,22 @@ repos:
     hooks:
       - id: flynt
   - repo: https://github.com/uktrade/pii-secret-check-hooks
-    rev: 0.0.0.35
+    rev: 0.0.0.36
     hooks:
     -   id: pii_secret_filename
         files: ''
         language: python
-        language_version: python3.9
+        language_version: python3.12
         pass_filenames: true
         require_serial: true
     -   id: pii_secret_file_content
         files: ''
         language: python
-        language_version: python3.9
+        language_version: python3.12
         pass_filenames: true
         require_serial: true
     -   id: hooks_version_check
         name: Checking local hooks against latest release
         verbose: true
         require_serial: true
-
+
diff --git a/Dockerfile b/Dockerfile
@@ -12,7 +12,7 @@
 
 ############################################################
 
-FROM python:3.9-bookworm
+FROM python:3.12-bookworm
 
 LABEL maintainer="webops@digital.trade.gov.uk"
 

diff --git a/Dockerfile.pytest b/Dockerfile.pytest
@@ -2,7 +2,7 @@
 ###############
 # Build on Python-based image using default target platform (TARGETPLATFORM).
 
-FROM python:3.9-bookworm
+FROM python:3.12-bookworm
 
 ARG TARGETPLATFORM
 ARG TARGETARCH

diff --git a/README.rst b/README.rst
@@ -17,7 +17,7 @@ Prerequisites
 
 The following dependencies are required to run this app:
 
-- Python_ 3.9.x
+- Python_ 3.12.x
 - Node.js_ 20.10.0 (LTS)
 - PostgreSQL_ 12
 - Redis_ 5.x

diff --git a/common/tests/test_files/bomb.xml b/common/tests/test_files/bomb.xml
@@ -3,5 +3,7 @@
 <!ENTITY b "&a;&a;&a;&a;&a;&a;&a;&a;">
 <!ENTITY c "&b;&b;&b;&b;&b;&b;&b;&b;">
 <!ENTITY d "&c;&c;&c;&c;&c;&c;&c;&c;">
+<!ENTITY e "&d;&d;&d;&d;&d;&d;&d;&d;">
+<!ENTITY f "&e;&e;&e;&e;&e;&e;&e;&e;">
 ]>
-<bomb>&c;</bomb>
+<bomb>&f;</bomb>
diff --git a/common/util.py b/common/util.py
@@ -52,21 +52,9 @@
 
 major, minor, patch = python_version_tuple()
 
-# The preferred style of combining @classmethod and @property is only in 3.9.
-# When we stop support for 3.8, we should remove both of these branches.
-if int(major) == 3 and int(minor) < 9:
-    # https://stackoverflow.com/a/13624858
-    class classproperty(object):
-        def __init__(self, fget):
-            self.fget = fget
 
-        def __get__(self, owner_self, owner_cls):
-            return self.fget(owner_cls)
-
-else:
-
-    def classproperty(fn):
-        return classmethod(property(fn))
+def classproperty(fn):
+    return classmethod(property(fn))
 
 
 def is_truthy(value: Union[str, bool]) -> bool:

diff --git a/conftest.py b/conftest.py
@@ -1099,7 +1099,7 @@ def make_storage_mock(s3, storage_class, **override_settings):
                 "LocationConstraint": settings.AWS_S3_REGION_NAME,
             },
         )
-    except s3.exceptions.BucketAlreadyExists:
+    except (s3.exceptions.BucketAlreadyExists, s3.exceptions.BucketAlreadyOwnedByYou):
         return storage
 
     return storage

diff --git a/importer/tests/cache/test_redis_cache.py b/importer/tests/cache/test_redis_cache.py
@@ -1,10 +1,8 @@
 import pytest
-from django_fakeredis.fakeredis import FakeRedis
 
 from importer.cache import redis
 
 
-@FakeRedis("django.core.cache")
 def test_put_stores_value():
     """For all parameterized cache engines, test that the put method stores a
     value in the respective cache."""
@@ -13,7 +11,6 @@ def test_put_stores_value():
     assert object_cache.get("test") == 123
 
 
-@FakeRedis("django.core.cache")
 def test_put_stores_value_when_value_exists():
     """For all parameterized cache engines, test that the multiple put calls to
     same key overwrites stored value in the respective caches."""
@@ -23,7 +20,6 @@ def test_put_stores_value_when_value_exists():
     assert object_cache.get("test") == 456
 
 
-@FakeRedis("django.core.cache")
 def test_pop_when_no_values_available_returns_none():
     """For all parameterized cache engines, test that pop calls to non-existent
     key returns None and does not throw exception etc."""
@@ -32,7 +28,6 @@ def test_pop_when_no_values_available_returns_none():
     assert object_cache.pop("zoo") is None
 
 
-@FakeRedis("django.core.cache")
 def test_pop_removes_and_returns_value_when_present():
     """For all parameterized cache engines, test that pop returns and removes
     value from cache."""
@@ -43,7 +38,6 @@ def test_pop_removes_and_returns_value_when_present():
     assert object_cache.pop("test") is None
 
 
-@FakeRedis("django.core.cache")
 def test_pop_returns_none_if_not_present():
     """For all parameterized cache engines, test that pop returns None when
     value not present."""
@@ -52,7 +46,6 @@ def test_pop_returns_none_if_not_present():
     assert object_cache.pop("zzz") is None
 
 
-@FakeRedis("django.core.cache")
 @pytest.mark.skip(reason="no way of currently testing this without redis")
 def test_keys_return_keys_correctly_when_populated():
     """For all parameterized cache engines, test that keys returns as expected
@@ -63,7 +56,6 @@ def test_keys_return_keys_correctly_when_populated():
     assert object_cache.keys() == {"test": ""}.keys()
 
 
-@FakeRedis("django.core.cache")
 @pytest.mark.skip(reason="no way of currently testing this without redis")
 def test_keys_return_keys_correctly_when_empty():
     """For all parameterized cache engines, test that keys returns as expected
@@ -72,7 +64,6 @@ def test_keys_return_keys_correctly_when_empty():
     assert object_cache.keys() == {}.keys()
 
 
-@FakeRedis("django.core.cache")
 def test_dump_return_correctly():
     """
     For all parameterized cache engines, test that dump returns as expected.
@@ -85,7 +76,6 @@ def test_dump_return_correctly():
     assert object_cache.dump() is None
 
 
-@FakeRedis("django.core.cache")
 @pytest.mark.skip(reason="no way of currently testing this without redis")
 def test_clear_returns_correctly():
     """For all parameterized cache engines, test that clear does clear the

diff --git a/importer/tests/test_importer_command.py b/importer/tests/test_importer_command.py
@@ -46,7 +46,7 @@ def get_value_from_line_name(output_lines, line_name):
         name is found amongst output_lines, else None.
         """
         matched_line = next(l for l in output_lines if l.startswith(line_name))
-        tokens = re.split("\W+", matched_line)
+        tokens = re.split(r"\W+", matched_line)
         return tokens[1] if tokens and len(tokens) > 1 else None
 
     assert str(importing_goods_import_batch.pk) == get_value_from_line_name(

diff --git a/publishing/tests/test_tasks.py b/publishing/tests/test_tasks.py
@@ -46,7 +46,7 @@ def test_create_and_upload_envelope(
         name
         for name in s3_object_names(expected_bucket)
         if re.match(
-            f"^envelope/{envelope_name}__.*\.xml$",
+            rf"^envelope/{envelope_name}__.*\.xml$",
             name,
         )
     )

diff --git a/pyproject.toml b/pyproject.toml
@@ -1,7 +1,7 @@
 [project]
 name = "tamato"
 version = "0.0.1"
-requires-python = ">=3.9"
+requires-python = ">=3.12"
 license = {file = "LICENSE"}
 description = "UK Tariff Management Tool"
 authors = [

diff --git a/requirements.txt b/requirements.txt
@@ -6,9 +6,9 @@ beautifulsoup4==4.10.0
 better-exceptions==0.3.3
 black==24.3.0
 bleach==6.1.0
-boto3==1.18.51
+boto3==1.20.27
 celery[redis]>=5.2.2
-coverage[toml]==5.4
+coverage[toml]==7.4.0
 crispy-forms-gds @ git+https://github.com/uktrade/crispy-forms-gds.git@0ed314933ce601e208c2f263bde3943e63b1e554
 dbt-copilot-python==0.2.1
 defusedxml==0.7.*
@@ -19,7 +19,6 @@ django-csp==3.6
 django-cte==1.3.1
 django-dotenv==1.4.2
 django-extensions==3.2.3
-django-fakeredis>=0.1.2
 django-filter==23.5
 django-formtools==2.3
 django-fsm==2.8.1
@@ -30,15 +29,16 @@ django-redis==5.4.0
 django-rest-polymorphic==0.1.9
 django-sequences==3.0
 django-staff-sso-client==4.2.1
-django-storages==1.14.2
+django-storages==1.14.3
 django-test-migrations==1.3.0
 django-webpack-loader==3.0.1
 django==4.2.14
 djangorestframework==3.15.2
 drf-extra-fields==3.7.0
 drf-flex-fields==1.0.2
-elastic-apm==6.7.2
+elastic-apm==6.15.1 # On GOV.UK PaaS, upgrading beyond this causes apm server compatibility issues
 factory-boy==3.3.0
+fakeredis==2.23.0
 flower==1.2.0
 freezegun==1.1.0
 gevent==23.9.1
@@ -50,7 +50,7 @@ lxml==5.2.2
 markdown==3.6
 markdownify==0.12.1
 markupsafe==2.1.2
-moto==2.1.0
+moto==3.0.7
 notifications-python-client==6.4.1
 openpyxl==3.0.7
 pandas~=2.0
@@ -66,7 +66,7 @@ pytest-django==4.8.0
 pytest-forked==1.4.0
 pytest-responses==0.5.0
 pytest-xdist==2.5.0
-pytest==7.1.2
+pytest==7.4.0
 python-magic==0.4.25
 requests-oauthlib==1.3.0
 requests-mock==1.10.0
@@ -88,5 +88,5 @@ tenacity==8.3.0
 urllib3==1.26.19
 Werkzeug==3.0.3
 whitenoise==5.2.0
-wrapt==1.12.1
+wrapt==1.14.1
 xmldiff==2.4
diff --git a/runtime.txt b/runtime.txt
@@ -1 +1 @@
-python-3.9.x
+python-3.12.x
diff --git a/settings/test.py b/settings/test.py
@@ -1,3 +1,5 @@
+from fakeredis import FakeConnection
+
 from settings.common import *
 
 ENV = "test"
@@ -23,10 +25,15 @@
 AWS_S3_ENDPOINT_URL = os.environ.get("TEST_AWS_S3_ENDPOINT_URL")
 AWS_S3_REGION_NAME = os.environ.get("TEST_AWS_S3_REGION_NAME", "eu-west-2")
 
-# Cache settings - put things in memory to minimise dependencies.
+# Cache settings - put things in fake redis cache.
 CACHES = {
     "default": {
-        "BACKEND": "django.core.cache.backends.dummy.DummyCache",
+        "BACKEND": "django_redis.cache.RedisCache",
+        "LOCATION": "redis://127.0.0.1:6379/1",  # Local dummy address
+        "OPTIONS": {
+            "REDIS_CLIENT_CLASS": "fakeredis.FakeRedis",
+            "CONNECTION_POOL_KWARGS": {"connection_class": FakeConnection},
+        },
     },
 }
 

diff --git a/workbaskets/tests/test_models.py b/workbaskets/tests/test_models.py
@@ -15,8 +15,6 @@
 from common.tests.factories import SeedFileTransactionFactory
 from common.tests.factories import TransactionFactory
 from common.tests.factories import WorkBasketFactory
-from common.tests.util import TestRule
-from common.tests.util import add_business_rules
 from common.tests.util import assert_transaction_order
 from common.validators import UpdateType
 from tasks.models import UserAssignment
@@ -318,18 +316,6 @@ def test_workbasket_approval_updates_transactions(
     assert_transaction_order(Transaction.objects.all())
 
 
-def test_workbasket_clean_does_not_run_business_rules():
-    """Workbaskets should not have business rule validation included as part of
-    their model cleaning, because business rules are slow to run and for a
-    moderate sized workbasket will time out a web request."""
-
-    model = factories.TestModel1Factory.create()
-    with add_business_rules(type(model), TestRule):
-        model.transaction.workbasket.full_clean()
-
-    assert TestRule.validate.not_called()
-
-
 def test_current_transaction_returns_last_approved_transaction(
     session_request,
     approved_transaction,

diff --git a/workbaskets/tests/test_views.py b/workbaskets/tests/test_views.py
@@ -135,11 +135,15 @@ def test_download(
     s3_resource,
     s3_object_names,
     settings,
+    s3,
 ):
     client.force_login(valid_user)
     bucket = "hmrc"
     settings.HMRC_STORAGE_BUCKET_NAME = bucket
-    s3_resource.create_bucket(Bucket="hmrc")
+    try:
+        s3_resource.create_bucket(Bucket="hmrc")
+    except s3.exceptions.BucketAlreadyOwnedByYou:
+        pass
     with patch(
         "exporter.storages.HMRCStorage.save",
         wraps=MagicMock(side_effect=hmrc_storage.save),