From 8da9016ac0b39698e1e5d62f3806d2a59f29d851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 7 Aug 2023 16:33:47 +0200 Subject: [PATCH 01/25] escape _ --- docs/releases/0.2.15.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/0.2.15.rst b/docs/releases/0.2.15.rst index ca782e04..944ca207 100644 --- a/docs/releases/0.2.15.rst +++ b/docs/releases/0.2.15.rst @@ -11,6 +11,6 @@ Support for SPA themes and some htmx fixes * The template for image galleries can now be overwritten by themes * Fixed audio player on htmx pagination * Fixed galleries on htmx pagination -* Don't remove newlines in *_html because it breaks preformatted code blocks +* Don't remove newlines in ``*_html`` because it breaks preformatted code blocks * Combine wagtail api pages endpoint with django-filter to allow filtering by date facets and fulltext search * Add facet count api endpoints From 633dce2e250df38a305476356ccaf3f816f31e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Fri, 11 Aug 2023 13:51:28 +0200 Subject: [PATCH 02/25] #99 renamed s3_backup to media_backup and make it read its configuration from settings.STORAGES --- cast/management/commands/media_backup.py | 33 ++++++++++++++++++++++++ cast/management/commands/s3_backup.py | 16 ------------ cast/utils.py | 4 +-- docs/backup.rst | 9 +++++++ docs/index.rst | 1 + docs/releases/0.2.18.rst | 4 +++ docs/releases/index.rst | 1 + docs/settings.rst | 31 +++++++++++++++++++++- 8 files changed, 80 insertions(+), 19 deletions(-) create mode 100644 cast/management/commands/media_backup.py delete mode 100644 cast/management/commands/s3_backup.py create mode 100644 docs/backup.rst create mode 100644 docs/releases/0.2.18.rst diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py new file mode 100644 index 00000000..d5cc0723 --- /dev/null +++ b/cast/management/commands/media_backup.py @@ -0,0 +1,33 @@ +try: + from django.core.files.storage import InvalidStorageError, storages + + DJANGO_VERSION_VALID = True +except ImportError: + DJANGO_VERSION_VALID = False +from django.core.management.base import BaseCommand + +from ...utils import storage_walk_paths + + +class Command(BaseCommand): + help = ( + "backup media files from production to backup storage " + "(requires Django >= 4.2 and production and backup storage configured)" + ) + + def handle(self, *args, **options): + if not DJANGO_VERSION_VALID: + # make sure we run at least Django 4.2 + print("Django version >= 4.2 is required") + exit(1) + try: + production_storage, backup_storage = storages["production"], storages["backup"] + except InvalidStorageError: + print("production or backup storage not configured") + return + for num, path in enumerate(storage_walk_paths(production_storage)): + if not backup_storage.exists(path): + with production_storage.open(path, "rb") as in_f: + backup_storage.save(path, in_f) + if num % 100 == 0: + print(".", end="", flush=True) diff --git a/cast/management/commands/s3_backup.py b/cast/management/commands/s3_backup.py deleted file mode 100644 index 4be1ca45..00000000 --- a/cast/management/commands/s3_backup.py +++ /dev/null @@ -1,16 +0,0 @@ -from django.core.files.storage import default_storage, get_storage_class -from django.core.management.base import BaseCommand - -from ...utils import storage_walk_paths - - -class Command(BaseCommand): - help = "backup media files from s3 to local media root" - - def handle(self, *args, **options): - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() - for path in storage_walk_paths(s3): - if not default_storage.exists(path): - print(path) - with s3.open(path, "rb") as in_f: - default_storage.save(path, in_f) diff --git a/cast/utils.py b/cast/utils.py index 91f8ca92..6ae1e7fb 100644 --- a/cast/utils.py +++ b/cast/utils.py @@ -1,10 +1,10 @@ import os from collections.abc import Iterable -from django.core.files.storage import FileSystemStorage +from django.core.files.storage import Storage -def storage_walk_paths(storage: FileSystemStorage, cur_dir: str = "") -> Iterable[str]: +def storage_walk_paths(storage: Storage, cur_dir: str = "") -> Iterable[str]: dirs, files = storage.listdir(cur_dir) for directory in dirs: new_dir = os.path.join(cur_dir, directory) diff --git a/docs/backup.rst b/docs/backup.rst new file mode 100644 index 00000000..f9331850 --- /dev/null +++ b/docs/backup.rst @@ -0,0 +1,9 @@ +###### +Backup +###### + +******************** +Media Backup/Restore +******************** + +Backup and restore are supported by the `media_backup` and `media_restore` commands. diff --git a/docs/index.rst b/docs/index.rst index 8d301419..ceaa3f5f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -16,6 +16,7 @@ fill since Django Cast switched to Wagtail. contributing features tests + backup howto/index release releases/index diff --git a/docs/releases/0.2.18.rst b/docs/releases/0.2.18.rst new file mode 100644 index 00000000..9b26a68c --- /dev/null +++ b/docs/releases/0.2.18.rst @@ -0,0 +1,4 @@ +0.2.18 (2023-08-14) +------------------- + +Improved media backup and restore #99. diff --git a/docs/releases/index.rst b/docs/releases/index.rst index da5ad2f0..a599375d 100644 --- a/docs/releases/index.rst +++ b/docs/releases/index.rst @@ -8,6 +8,7 @@ Versions .. toctree:: :maxdepth: 1 + 0.2.18 0.2.17 0.2.16 0.2.15 diff --git a/docs/settings.rst b/docs/settings.rst index 6abc6832..60dd0d80 100644 --- a/docs/settings.rst +++ b/docs/settings.rst @@ -73,4 +73,33 @@ a name and a display. For instance: The display value is the title displayed in the theme selector within the Wagtail admin panel. The name corresponds to the theme's base directory inside your templates folder. To create a theme named my_theme, make a directory called ``cast/my_theme`` -within your templates folder and place your templates inside. +within your templates folder and place your templates inside. asdf + +******** +Storages +******** + +Configure Backup Storage +======================== + +If you store your media files on S3, you can configure a local backup storage +like this: + +.. code-block:: python + + STORAGES = { + "default": {"BACKEND": "config.settings.local.CustomS3Boto3Storage"}, + "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}, + "production": {"BACKEND": "config.settings.local.CustomS3Boto3Storage"}, + "backup": { + "BACKEND": "django.core.files.storage.FileSystemStorage", + "OPTIONS": { + "location": ROOT_DIR.path("backups").path("media"), + }, + }, + } + + +.. important:: + + This will only work if you are using Django >= 4.2. From ff616ae477f5ee065eab4d909ccd927a3be7a786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Fri, 11 Aug 2023 18:15:19 +0200 Subject: [PATCH 03/25] #99 first test for media_backup management command (print error message if no production / backup storage is defined) --- tests/management_command_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 tests/management_command_test.py diff --git a/tests/management_command_test.py b/tests/management_command_test.py new file mode 100644 index 00000000..d963c044 --- /dev/null +++ b/tests/management_command_test.py @@ -0,0 +1,8 @@ +from django.core.management import call_command + + +def test_media_backup_without_storages(capsys, settings): + settings.STORAGES = {} + call_command("media_backup") + captured = capsys.readouterr() + assert captured.out == "production or backup storage not configured\n" From 98bd3405740910e35cb5097e3fd74a9606260188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Sat, 12 Aug 2023 06:19:53 +0200 Subject: [PATCH 04/25] #99 test for error message on too old django version --- cast/management/commands/media_backup.py | 2 +- tests/management_command_test.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index d5cc0723..0f134130 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -19,7 +19,7 @@ def handle(self, *args, **options): if not DJANGO_VERSION_VALID: # make sure we run at least Django 4.2 print("Django version >= 4.2 is required") - exit(1) + return try: production_storage, backup_storage = storages["production"], storages["backup"] except InvalidStorageError: diff --git a/tests/management_command_test.py b/tests/management_command_test.py index d963c044..b5f0ae2c 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -6,3 +6,11 @@ def test_media_backup_without_storages(capsys, settings): call_command("media_backup") captured = capsys.readouterr() assert captured.out == "production or backup storage not configured\n" + + +def test_media_backup_without_django_version(capsys, settings, mocker): + settings.STORAGES = {} + mocker.patch("cast.management.commands.media_backup.DJANGO_VERSION_VALID", False) + call_command("media_backup") + captured = capsys.readouterr() + assert captured.out == "Django version >= 4.2 is required\n" From ccabbae92bfdd84bae5bceb3541529b936a02f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Sat, 12 Aug 2023 07:20:09 +0200 Subject: [PATCH 05/25] #99 test for new file added to production is backupd --- cast/management/commands/media_backup.py | 2 + tests/management_command_test.py | 53 ++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index 0f134130..21c1efcb 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -26,7 +26,9 @@ def handle(self, *args, **options): print("production or backup storage not configured") return for num, path in enumerate(storage_walk_paths(production_storage)): + print("path: ", path) if not backup_storage.exists(path): + print("not in backup storage") with production_storage.open(path, "rb") as in_f: backup_storage.save(path, in_f) if num % 100 == 0: diff --git a/tests/management_command_test.py b/tests/management_command_test.py index b5f0ae2c..93518216 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -1,3 +1,7 @@ +from io import BytesIO + +import pytest +from django.core.files.storage import storages from django.core.management import call_command @@ -14,3 +18,52 @@ def test_media_backup_without_django_version(capsys, settings, mocker): call_command("media_backup") captured = capsys.readouterr() assert captured.out == "Django version >= 4.2 is required\n" + + +class StubStorage: + _files: list[str] = [] + _exists: list[str] = [] + + def exists(self, path): + return path in self._exists + + def save(self, name, _content): + print("save called: ", name, _content) + self._exists.append(name) + + @staticmethod + def open(name, _mode): + class StubFile: + def __init__(self, file_name): + self.name = file_name + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + return StubFile(name) + + def listdir(self, path): + return [], self._files + + +@pytest.fixture +def stub_storages(settings): + storage_stub = {"BACKEND": "tests.management_command_test.StubStorage"} + settings.STORAGES = {"production": storage_stub, "backup": storage_stub} + return storages + + +def test_media_backup_new_file(capsys, stub_storages): + production, backup = stub_storages["production"], stub_storages["backup"] + + # given there's a new file added to production + production.save("foobar.jpg", BytesIO(b"foobar")) + + # when we run the backup command + call_command("media_backup") + + # then the file should be added to the backup + assert backup.exists("foobar.jpg") From 2ee93e8129edb95c36d3160f2b5caac7a684400a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Sat, 12 Aug 2023 07:21:49 +0200 Subject: [PATCH 06/25] #99 removed obsolete debug output --- cast/management/commands/media_backup.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index 21c1efcb..0f134130 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -26,9 +26,7 @@ def handle(self, *args, **options): print("production or backup storage not configured") return for num, path in enumerate(storage_walk_paths(production_storage)): - print("path: ", path) if not backup_storage.exists(path): - print("not in backup storage") with production_storage.open(path, "rb") as in_f: backup_storage.save(path, in_f) if num % 100 == 0: From 7e1ada308a00720eff8aeb3a7dcd5ff4635aaf73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Sat, 12 Aug 2023 07:36:45 +0200 Subject: [PATCH 07/25] #99 removed pycharm warning --- tests/management_command_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 93518216..bba95ad4 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -45,7 +45,7 @@ def __exit__(self, *args): return StubFile(name) - def listdir(self, path): + def listdir(self, _path): return [], self._files From 469eaece26358ce8d2b643b5256c16fdd41991fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 09:21:09 +0200 Subject: [PATCH 08/25] #99 moved django version logic and check for valid settings into own function because it will be reused in other management commands + modified tests accordingly --- cast/management/commands/media_backup.py | 17 ++-------------- cast/management/commands/storage_backend.py | 22 +++++++++++++++++++++ tests/management_command_test.py | 20 +++++++++---------- 3 files changed, 34 insertions(+), 25 deletions(-) create mode 100644 cast/management/commands/storage_backend.py diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index 0f134130..2d8c203e 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -1,12 +1,7 @@ -try: - from django.core.files.storage import InvalidStorageError, storages - - DJANGO_VERSION_VALID = True -except ImportError: - DJANGO_VERSION_VALID = False from django.core.management.base import BaseCommand from ...utils import storage_walk_paths +from .storage_backend import get_production_and_backup_storage class Command(BaseCommand): @@ -16,15 +11,7 @@ class Command(BaseCommand): ) def handle(self, *args, **options): - if not DJANGO_VERSION_VALID: - # make sure we run at least Django 4.2 - print("Django version >= 4.2 is required") - return - try: - production_storage, backup_storage = storages["production"], storages["backup"] - except InvalidStorageError: - print("production or backup storage not configured") - return + production_storage, backup_storage = get_production_and_backup_storage() for num, path in enumerate(storage_walk_paths(production_storage)): if not backup_storage.exists(path): with production_storage.open(path, "rb") as in_f: diff --git a/cast/management/commands/storage_backend.py b/cast/management/commands/storage_backend.py new file mode 100644 index 00000000..2d3d695d --- /dev/null +++ b/cast/management/commands/storage_backend.py @@ -0,0 +1,22 @@ +from django.core.files.storage import InvalidStorageError, Storage + +try: + from django.core.files.storage import storages # noqa F401 + + DJANGO_VERSION_VALID = True +except ImportError: + DJANGO_VERSION_VALID = False + +from django.core.management.base import CommandError + + +def get_production_and_backup_storage() -> tuple[Storage, Storage]: + if not DJANGO_VERSION_VALID: + # make sure we run at least Django 4.2 + raise CommandError("Django version >= 4.2 is required") + else: + try: + production_storage, backup_storage = storages["production"], storages["backup"] + return production_storage, backup_storage + except InvalidStorageError: + raise CommandError("production or backup storage not configured") diff --git a/tests/management_command_test.py b/tests/management_command_test.py index bba95ad4..93ef8f38 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -3,21 +3,21 @@ import pytest from django.core.files.storage import storages from django.core.management import call_command +from django.core.management.base import CommandError -def test_media_backup_without_storages(capsys, settings): +def test_media_backup_without_storages(settings): settings.STORAGES = {} - call_command("media_backup") - captured = capsys.readouterr() - assert captured.out == "production or backup storage not configured\n" + with pytest.raises(CommandError) as err: + call_command("media_backup") + assert str(err.value) == "production or backup storage not configured" -def test_media_backup_without_django_version(capsys, settings, mocker): - settings.STORAGES = {} - mocker.patch("cast.management.commands.media_backup.DJANGO_VERSION_VALID", False) - call_command("media_backup") - captured = capsys.readouterr() - assert captured.out == "Django version >= 4.2 is required\n" +def test_media_backup_with_wrong_django_version(mocker): + mocker.patch("cast.management.commands.storage_backend.DJANGO_VERSION_VALID", False) + with pytest.raises(CommandError) as err: + call_command("media_backup") + assert str(err.value) == "Django version >= 4.2 is required" class StubStorage: From 8ff245fd455aa012cf75b82e2ccd28c98be82c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 09:56:23 +0200 Subject: [PATCH 09/25] #99 nocov for import error --- cast/management/commands/storage_backend.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cast/management/commands/storage_backend.py b/cast/management/commands/storage_backend.py index 2d3d695d..0ce12c95 100644 --- a/cast/management/commands/storage_backend.py +++ b/cast/management/commands/storage_backend.py @@ -4,7 +4,7 @@ from django.core.files.storage import storages # noqa F401 DJANGO_VERSION_VALID = True -except ImportError: +except ImportError: # pragma: no cover DJANGO_VERSION_VALID = False from django.core.management.base import CommandError From c95bb6df978ffa3f0b7b3e9453ab6400d9b587bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:19:12 +0200 Subject: [PATCH 10/25] #99 if _files is a class variable, save adds to both production and backup -> fixed --- cast/management/commands/media_backup.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index 2d8c203e..ea6f4a39 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -10,11 +10,14 @@ class Command(BaseCommand): "(requires Django >= 4.2 and production and backup storage configured)" ) - def handle(self, *args, **options): - production_storage, backup_storage = get_production_and_backup_storage() + @staticmethod + def backup_media_files(production_storage, backup_storage): for num, path in enumerate(storage_walk_paths(production_storage)): if not backup_storage.exists(path): with production_storage.open(path, "rb") as in_f: backup_storage.save(path, in_f) if num % 100 == 0: print(".", end="", flush=True) + + def handle(self, *args, **options): + self.backup_media_files(*get_production_and_backup_storage()) From b695b0a5fd21482116f189fecac838a3122416a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:19:55 +0200 Subject: [PATCH 11/25] #99 moved actual backup into own method to make it testable by unit tests --- tests/management_command_test.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 93ef8f38..2999436f 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -21,15 +21,14 @@ def test_media_backup_with_wrong_django_version(mocker): class StubStorage: - _files: list[str] = [] - _exists: list[str] = [] + def __init__(self): + self._files = [] def exists(self, path): - return path in self._exists + return path in self._files def save(self, name, _content): - print("save called: ", name, _content) - self._exists.append(name) + self._files.append(name) @staticmethod def open(name, _mode): From d0aff1710e56e7cf1656c21b00a62628775b4f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:36:11 +0200 Subject: [PATCH 12/25] #99 use contextmanager decorator from contextlib instead of verbose class + some annotations --- tests/management_command_test.py | 35 ++++++++++++++------------------ 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 2999436f..217e00f0 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -1,3 +1,5 @@ +from collections.abc import Generator +from contextlib import contextmanager from io import BytesIO import pytest @@ -21,32 +23,25 @@ def test_media_backup_with_wrong_django_version(mocker): class StubStorage: - def __init__(self): - self._files = [] + def __init__(self) -> None: + self._files: dict[str, str] = {} - def exists(self, path): + def exists(self, path: str) -> bool: return path in self._files - def save(self, name, _content): - self._files.append(name) + def save(self, name: str, content: str) -> None: + self._files[name] = content - @staticmethod - def open(name, _mode): - class StubFile: - def __init__(self, file_name): - self.name = file_name - - def __enter__(self): - return self - - def __exit__(self, *args): - pass - - return StubFile(name) - - def listdir(self, _path): + def listdir(self, _path: str) -> tuple[list, dict[str, str]]: return [], self._files + @contextmanager + def open(self, name: str, _mode: str) -> Generator[str, None, None]: + try: + yield self._files[name] + finally: + pass + @pytest.fixture def stub_storages(settings): From fd2219b22f1e5373a1e7c819d4b224b2f0e84d84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:39:25 +0200 Subject: [PATCH 13/25] #99 improved name + removed capsys fixture --- tests/management_command_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 217e00f0..4c4911bb 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -50,7 +50,7 @@ def stub_storages(settings): return storages -def test_media_backup_new_file(capsys, stub_storages): +def test_media_backup_new_file_in_production(stub_storages): production, backup = stub_storages["production"], stub_storages["backup"] # given there's a new file added to production From 920f02315eeb296621c098a8927c5820ed5c8036 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:48:21 +0200 Subject: [PATCH 14/25] #99 make sure the new file is not just existing in backup but was added by the media backup command --- tests/management_command_test.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 4c4911bb..da877a1c 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -1,6 +1,5 @@ from collections.abc import Generator from contextlib import contextmanager -from io import BytesIO import pytest from django.core.files.storage import storages @@ -25,11 +24,19 @@ def test_media_backup_with_wrong_django_version(mocker): class StubStorage: def __init__(self) -> None: self._files: dict[str, str] = {} + self._added: set[str] = set() def exists(self, path: str) -> bool: return path in self._files + def was_added_by_backup(self, name: str) -> bool: + return name in self._added + def save(self, name: str, content: str) -> None: + self.save_without_adding(name, content) + self._added.add(name) + + def save_without_adding(self, name: str, content: str) -> None: self._files[name] = content def listdir(self, _path: str) -> tuple[list, dict[str, str]]: @@ -54,10 +61,10 @@ def test_media_backup_new_file_in_production(stub_storages): production, backup = stub_storages["production"], stub_storages["backup"] # given there's a new file added to production - production.save("foobar.jpg", BytesIO(b"foobar")) + production.save_without_adding("foobar.jpg", "foobar") # type: ignore # when we run the backup command call_command("media_backup") - # then the file should be added to the backup - assert backup.exists("foobar.jpg") + # then the file should have been added by the backup command + assert backup.was_added_by_backup("foobar.jpg") # type: ignore From e5b5d62a934a224c27e170c55aa2e91fa48056e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 10:55:57 +0200 Subject: [PATCH 15/25] #99 new test for already existing file + some type foo + no cover for progress output -> coverage at 100% again --- cast/management/commands/media_backup.py | 2 +- tests/management_command_test.py | 34 +++++++++++++++++++----- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index ea6f4a39..4ce2b3a0 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -16,7 +16,7 @@ def backup_media_files(production_storage, backup_storage): if not backup_storage.exists(path): with production_storage.open(path, "rb") as in_f: backup_storage.save(path, in_f) - if num % 100 == 0: + if num % 100 == 0: # pragma: no cover print(".", end="", flush=True) def handle(self, *args, **options): diff --git a/tests/management_command_test.py b/tests/management_command_test.py index da877a1c..9f7a9769 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -1,11 +1,14 @@ -from collections.abc import Generator +from collections.abc import Iterator from contextlib import contextmanager +from io import BytesIO import pytest from django.core.files.storage import storages from django.core.management import call_command from django.core.management.base import CommandError +from cast.management.commands.media_backup import Command as MediaBackupCommand + def test_media_backup_without_storages(settings): settings.STORAGES = {} @@ -23,7 +26,7 @@ def test_media_backup_with_wrong_django_version(mocker): class StubStorage: def __init__(self) -> None: - self._files: dict[str, str] = {} + self._files: dict[str, BytesIO] = {} self._added: set[str] = set() def exists(self, path: str) -> bool: @@ -32,18 +35,21 @@ def exists(self, path: str) -> bool: def was_added_by_backup(self, name: str) -> bool: return name in self._added - def save(self, name: str, content: str) -> None: + def was_not_added_by_backup(self, name: str) -> bool: + return name not in self._added + + def save(self, name: str, content: BytesIO) -> None: self.save_without_adding(name, content) self._added.add(name) - def save_without_adding(self, name: str, content: str) -> None: + def save_without_adding(self, name: str, content: BytesIO) -> None: self._files[name] = content - def listdir(self, _path: str) -> tuple[list, dict[str, str]]: + def listdir(self, _path: str) -> tuple[list, dict[str, BytesIO]]: return [], self._files @contextmanager - def open(self, name: str, _mode: str) -> Generator[str, None, None]: + def open(self, name: str, _mode: str) -> Iterator[BytesIO]: try: yield self._files[name] finally: @@ -61,10 +67,24 @@ def test_media_backup_new_file_in_production(stub_storages): production, backup = stub_storages["production"], stub_storages["backup"] # given there's a new file added to production - production.save_without_adding("foobar.jpg", "foobar") # type: ignore + production.save_without_adding("foobar.jpg", BytesIO(b"foobar")) # type: ignore # when we run the backup command call_command("media_backup") # then the file should have been added by the backup command assert backup.was_added_by_backup("foobar.jpg") # type: ignore + + +def test_media_backup_existing_file_in_backup(stub_storages): + production, backup = stub_storages["production"], stub_storages["backup"] + + # given there's a file in the backup + production.save_without_adding("foobar.jpg", BytesIO(b"foobar")) # type: ignore + backup.save_without_adding("foobar.jpg", BytesIO(b"foobar")) # type: ignore + + # when we run the backup method + MediaBackupCommand().backup_media_files(production, backup) + + # then the file should not have been added by the backup command + assert backup.was_not_added_by_backup("foobar.jpg") # type: ignore From 70482e51ab7cdd5308adffd4dd4cba050c35a28a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 12:27:21 +0200 Subject: [PATCH 16/25] #99 format --- tests/management_command_test.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 9f7a9769..90ff47a5 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -4,8 +4,7 @@ import pytest from django.core.files.storage import storages -from django.core.management import call_command -from django.core.management.base import CommandError +from django.core.management import CommandError, call_command from cast.management.commands.media_backup import Command as MediaBackupCommand From 04f5cc29c77ba4a6a8422f75de6b5f7b27c0d738 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 12:28:20 +0200 Subject: [PATCH 17/25] #99 show media sizes of production storage backend instead of hard coded s3 --- .../{s3_media_sizes.py => media_sizes.py} | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) rename cast/management/commands/{s3_media_sizes.py => media_sizes.py} (63%) diff --git a/cast/management/commands/s3_media_sizes.py b/cast/management/commands/media_sizes.py similarity index 63% rename from cast/management/commands/s3_media_sizes.py rename to cast/management/commands/media_sizes.py index 92af70be..752a851a 100644 --- a/cast/management/commands/s3_media_sizes.py +++ b/cast/management/commands/media_sizes.py @@ -1,11 +1,15 @@ -from django.core.files.storage import get_storage_class from django.core.management.base import BaseCommand from cast.utils import storage_walk_paths +from .storage_backend import get_production_and_backup_storage + class Command(BaseCommand): - help = "shows size of media on s3" + help = ( + "show size of media files on production storage backend" + "(requires Django >= 4.2 and production and backup storage configured)" + ) @staticmethod def show_usage(paths): @@ -26,11 +30,16 @@ def show_usage(paths): print(f"misc usage: {misc / unit}") print(f"total usage: {sum(paths.values()) / unit}") - def handle(self, *args, **options): - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() + @staticmethod + def get_paths_with_sizes_for(storage_backend): paths = {} - for path in storage_walk_paths(s3): - size = s3.size(path) + for path in storage_walk_paths(storage_backend): + size = storage_backend.size(path) paths[path] = size print(path, size / 2**20) + return paths + + def handle(self, *args, **options): + production, _ = get_production_and_backup_storage() + paths = self.get_paths_with_sizes_for(production) self.show_usage(paths) From 01853778bc181aea182d62a91c150fff70e6e60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 12:33:47 +0200 Subject: [PATCH 18/25] #99 replace media of production storage backend instead of hard coded s3 --- cast/management/commands/media_replace.py | 24 +++++++++++++++++++++++ cast/management/commands/s3_replace.py | 18 ----------------- 2 files changed, 24 insertions(+), 18 deletions(-) create mode 100644 cast/management/commands/media_replace.py delete mode 100644 cast/management/commands/s3_replace.py diff --git a/cast/management/commands/media_replace.py b/cast/management/commands/media_replace.py new file mode 100644 index 00000000..a36190f5 --- /dev/null +++ b/cast/management/commands/media_replace.py @@ -0,0 +1,24 @@ +from django.core.files.storage import FileSystemStorage +from django.core.management.base import BaseCommand + +from .storage_backend import get_production_and_backup_storage + + +class Command(BaseCommand): + help = ( + "replace paths on production storage backend with local versions - useful for compressed videos for example" + "(requires Django >= 4.2 and production and backup storage configured)" + ) + + def add_arguments(self, parser): + parser.add_argument("paths", nargs="+", type=str) + + def handle(self, *args, **options): + production, _ = get_production_and_backup_storage() + fs_storage = FileSystemStorage() + for path in options["paths"]: + if fs_storage.exists(path): + if production.exists(path): + production.delete(path) + with fs_storage.open(path, "rb") as in_f: + production.save(path, in_f) diff --git a/cast/management/commands/s3_replace.py b/cast/management/commands/s3_replace.py deleted file mode 100644 index f9448bae..00000000 --- a/cast/management/commands/s3_replace.py +++ /dev/null @@ -1,18 +0,0 @@ -from django.core.files.storage import default_storage, get_storage_class -from django.core.management.base import BaseCommand - - -class Command(BaseCommand): - help = "replace paths on s3 with local versions - useful for compressed videos for example" - - def add_arguments(self, parser): - parser.add_argument("paths", nargs="+", type=str) - - def handle(self, *args, **options): - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() - for path in options["paths"]: - if default_storage.exists(path): - if s3.exists(path): - s3.delete(path) - with default_storage.open(path, "rb") as in_f: - s3.save(path, in_f) From aa6605733cc98b42b6f6645aeef92148b58a7d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 12:42:56 +0200 Subject: [PATCH 19/25] #99 use the same sync media files function for both backup and restore --- cast/management/commands/media_backup.py | 10 ++-------- cast/management/commands/media_restore.py | 17 +++++++++++++++++ cast/management/commands/s3_restore.py | 16 ---------------- cast/management/commands/storage_backend.py | 11 +++++++++++ 4 files changed, 30 insertions(+), 24 deletions(-) create mode 100644 cast/management/commands/media_restore.py delete mode 100644 cast/management/commands/s3_restore.py diff --git a/cast/management/commands/media_backup.py b/cast/management/commands/media_backup.py index 4ce2b3a0..5e77c4c2 100644 --- a/cast/management/commands/media_backup.py +++ b/cast/management/commands/media_backup.py @@ -1,7 +1,6 @@ from django.core.management.base import BaseCommand -from ...utils import storage_walk_paths -from .storage_backend import get_production_and_backup_storage +from .storage_backend import get_production_and_backup_storage, sync_media_files class Command(BaseCommand): @@ -12,12 +11,7 @@ class Command(BaseCommand): @staticmethod def backup_media_files(production_storage, backup_storage): - for num, path in enumerate(storage_walk_paths(production_storage)): - if not backup_storage.exists(path): - with production_storage.open(path, "rb") as in_f: - backup_storage.save(path, in_f) - if num % 100 == 0: # pragma: no cover - print(".", end="", flush=True) + sync_media_files(production_storage, backup_storage) def handle(self, *args, **options): self.backup_media_files(*get_production_and_backup_storage()) diff --git a/cast/management/commands/media_restore.py b/cast/management/commands/media_restore.py new file mode 100644 index 00000000..3ae72f32 --- /dev/null +++ b/cast/management/commands/media_restore.py @@ -0,0 +1,17 @@ +from django.core.management.base import BaseCommand + +from .storage_backend import get_production_and_backup_storage, sync_media_files + + +class Command(BaseCommand): + help = ( + "restore media files from backup storage backend to production storage backend " + "(requires Django >= 4.2 and production and backup storage configured)" + ) + + @staticmethod + def restore_media_files(production_storage, backup_storage): + sync_media_files(backup_storage, production_storage) + + def handle(self, *args, **options): + self.restore_media_files(*get_production_and_backup_storage()) diff --git a/cast/management/commands/s3_restore.py b/cast/management/commands/s3_restore.py deleted file mode 100644 index 443485e3..00000000 --- a/cast/management/commands/s3_restore.py +++ /dev/null @@ -1,16 +0,0 @@ -from django.core.files.storage import default_storage, get_storage_class -from django.core.management.base import BaseCommand - -from cast.utils import storage_walk_paths - - -class Command(BaseCommand): - help = "restore media files from local media root to s3" - - def handle(self, *args, **options): - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() - for path in storage_walk_paths(default_storage): - if not s3.exists(path): - print(path) - with default_storage.open(path, "rb") as in_f: - s3.save(path, in_f) diff --git a/cast/management/commands/storage_backend.py b/cast/management/commands/storage_backend.py index 0ce12c95..10b88f38 100644 --- a/cast/management/commands/storage_backend.py +++ b/cast/management/commands/storage_backend.py @@ -9,6 +9,17 @@ from django.core.management.base import CommandError +from ...utils import storage_walk_paths + + +def sync_media_files(source_storage, target_storage): + for num, path in enumerate(storage_walk_paths(source_storage)): + if not target_storage.exists(path): + with source_storage.open(path, "rb") as in_f: + target_storage.save(path, in_f) + if num % 100 == 0: # pragma: no cover + print(".", end="", flush=True) + def get_production_and_backup_storage() -> tuple[Storage, Storage]: if not DJANGO_VERSION_VALID: From 9f40bf17fecbc57fa5520da8f344dfafe817c46f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 12:49:53 +0200 Subject: [PATCH 20/25] #99 detect stale media files only for production storage backend... --- .../commands/{s3_stale.py => media_stale.py} | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) rename cast/management/commands/{s3_stale.py => media_stale.py} (61%) diff --git a/cast/management/commands/s3_stale.py b/cast/management/commands/media_stale.py similarity index 61% rename from cast/management/commands/s3_stale.py rename to cast/management/commands/media_stale.py index 4a9b1479..63cd3a07 100644 --- a/cast/management/commands/s3_stale.py +++ b/cast/management/commands/media_stale.py @@ -1,13 +1,16 @@ -from django.core.files.storage import default_storage, get_storage_class from django.core.management.base import BaseCommand from wagtail.images.models import Image from ...models import File, Video from ...utils import storage_walk_paths +from .storage_backend import get_production_and_backup_storage class Command(BaseCommand): - help = "show media files which are in the filesystem (s3, locale), but not in database and optionally delete them" + help = ( + "show media files which are in the production storage backend, but not in database and optionally delete them " + "(requires Django >= 4.2 and production and backup storage configured)" + ) def add_arguments(self, parser): parser.add_argument( @@ -49,29 +52,18 @@ def get_models_paths(self): return paths def handle(self, *args, **options): + production_storage, _ = get_production_and_backup_storage() paths_from_models = self.get_models_paths() - print("stale s3") - s3 = get_storage_class("storages.backends.s3boto3.S3Boto3Storage")() - s3_paths = self.get_paths(s3) - stale_s3 = {} - for path, size in s3_paths.items(): + print("stale production") + production_paths = self.get_paths(production_storage) + stale_production = {} + for path, size in production_paths.items(): if path not in paths_from_models: print(path) - stale_s3[path] = size - print(f"stale s3 size: {sum(stale_s3.values()) / 2 ** 20} Mb") - - print("stale locale") - locale_paths = self.get_paths(default_storage) - stale_locale = {} - for path, size in locale_paths.items(): - if path not in paths_from_models: - print(path) - stale_locale[path] = size - print(f"stale locale size: {sum(stale_locale.values()) / 2 ** 20} Mb") + stale_production[path] = size + print(f"stale production size: {sum(stale_production.values()) / 2 ** 20} Mb") if options["delete"]: - # for path in stale_s3.keys(): - # s3.delete(path) - for path in stale_locale.keys(): - default_storage.delete(path) + for path in stale_production.keys(): + production_storage.delete(path) From af0d107dd4eaae144d8b46f31c884604f7fe6ce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 13:19:40 +0200 Subject: [PATCH 21/25] #99 some more backup/restore documentation --- docs/backup.rst | 21 ++++++++++++++++++++- docs/management-commands.rst | 14 ++++++++------ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/docs/backup.rst b/docs/backup.rst index f9331850..82dd2a7c 100644 --- a/docs/backup.rst +++ b/docs/backup.rst @@ -2,8 +2,27 @@ Backup ###### +*********************** +Database Backup/Restore +*********************** + +There's no unified way to backup and restore a database. For my personal projects +I use ansible playbooks like this: + +* `backup `_ +* `restore `_ + +It would be nice to be able to fetch all the relevant database contents by just +reading from the cast REST-api and recreate the contents by just writing to +another cast REST-api. This would make it possible to backup and restore really +easy. But for now you have to do something database specific. + ******************** Media Backup/Restore ******************** -Backup and restore are supported by the `media_backup` and `media_restore` commands. +Backup and restore are supported by the `media_backup` and `media_restore` +:ref:`management commands `. + +Once we have a unified way to backup and restore the database, we can also +integrate the media backup and restore into a single command. diff --git a/docs/management-commands.rst b/docs/management-commands.rst index d4647c9a..210fadcc 100644 --- a/docs/management-commands.rst +++ b/docs/management-commands.rst @@ -2,15 +2,17 @@ Django-Admin Commands ********************* +.. _cast_management_commands: + There are some management-commands bundled with django-cast. Most of them are dealing with the management of media files. * ``recalc_video_posters``: Recalculate the poster images for all videos. -* ``s3_backup``: Backup media files from S3 to local media root. -* ``s3_media_sizes``: Print the sizes of all media files on S3. -* ``s3_replace``: Replace paths on s3 with versions from local media root. +* ``media_backup``: Backup media files from production to backup storage backend (requires Django >= 4.2). +* ``media_sizes``: Print the sizes of all media files stored in the production storage backend (requires Django >= 4.2). +* ``media_replace``: Replace files on production storage backend with versions from local file system (requires Django >= 4.2). This might be useful for videos for which you now have a better compressed version, but you don't want to generate a new name. -* ``s3_restore``: Restore media files from local media root to S3. -* ``s3_stale``: Print the paths of all media files on S3 that are not - referenced in the database. +* ``media_restore``: Restore media files from backup storage backend to production storage backend (requires Django >= 4.2). +* ``media_stale``: Print the paths of all media files stored in the production storage backend that are not + referenced in the database (requires Django >= 4.2). From 1b63a0c0e373835834ca55292937a14bde597954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 13:47:49 +0200 Subject: [PATCH 22/25] #99 updated release notes --- docs/releases/0.2.18.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/releases/0.2.18.rst b/docs/releases/0.2.18.rst index 9b26a68c..867877b0 100644 --- a/docs/releases/0.2.18.rst +++ b/docs/releases/0.2.18.rst @@ -1,4 +1,4 @@ 0.2.18 (2023-08-14) ------------------- -Improved media backup and restore #99. +Use the STORAGES setting introduced in Django 4.2 to improve media backup and restore #99. From c38eee9b910f19a39ba18ecfaa6a0b80b5765cff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 13:51:25 +0200 Subject: [PATCH 23/25] bump version number to 0.2.18 --- README.md | 2 +- cast/__init__.py | 2 +- docs/conf.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index ce06cfbd..c585368e 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ and [Wagtail](https://wagtail.org). After switching to Wagtail, the documentation has to be updated. Stay tuned 😄. -**Documentation for [current version 0.2.17](https://django-cast.readthedocs.io/en/develop/)** +**Documentation for [current version 0.2.18](https://django-cast.readthedocs.io/en/develop/)** ## Key Features - Responsive images via [wagtail-srcset](https://github.com/ephes/wagtail_srcset) diff --git a/cast/__init__.py b/cast/__init__.py index e0eb6732..321cff8e 100644 --- a/cast/__init__.py +++ b/cast/__init__.py @@ -1,4 +1,4 @@ """ Django and Wagtail based blogging / podcasting package """ -__version__ = "0.2.17" +__version__ = "0.2.18" diff --git a/docs/conf.py b/docs/conf.py index bcf4d630..93a5f964 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -9,7 +9,7 @@ project = "Django Cast" copyright = "2022, Jochen Wersdörfer" author = "Jochen Wersdörfer" -release = "0.2.17" +release = "0.2.18" # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration From 1e1211406159a41e3f93b6529a354076392923af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 14:26:48 +0200 Subject: [PATCH 24/25] new readthedocs config version? --- .readthedocs.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yml b/.readthedocs.yml index c0e3fe66..43d6e94e 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -1,4 +1,13 @@ +version: 2 + build: - image: latest + image: latest + tools: + python: "3.11" + +sphinx: + configuration: docs/conf.py + python: - version: 3.7 + install: + - requirements: docs/requirements.txt From 3f3545f5562cbee949c66145ddd1ac7048d8d7d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jochen=20Wersd=C3=B6rfer?= Date: Mon, 14 Aug 2023 14:38:41 +0200 Subject: [PATCH 25/25] skip management command tests for Django < 4.2 --- tests/management_command_test.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/management_command_test.py b/tests/management_command_test.py index 90ff47a5..2b6188c9 100644 --- a/tests/management_command_test.py +++ b/tests/management_command_test.py @@ -2,11 +2,29 @@ from contextlib import contextmanager from io import BytesIO +import django import pytest -from django.core.files.storage import storages + +try: + from django.core.files.storage import storages +except ImportError: + pass from django.core.management import CommandError, call_command -from cast.management.commands.media_backup import Command as MediaBackupCommand +try: + from cast.management.commands.media_backup import Command as MediaBackupCommand +except ImportError: + pass + + +def get_comparable_django_version(): + return int("".join(django.get_version().split(".")[:2])) + + +pytestmark = pytest.mark.skipif( + get_comparable_django_version() < 42, + reason="Django version >= 4.2 is required", +) def test_media_backup_without_storages(settings):