diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b1d5389..5a942bda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 3.3.2.dev +* Fix: debug logging in rights/from_file +* Add: option [storage] use_cache_subfolder_for_item for storing item cache outside collection-root + ## 3.3.1 * Add: option [auth] type=dovecot diff --git a/DOCUMENTATION.md b/DOCUMENTATION.md index e964cd53..ab28c6ce 100644 --- a/DOCUMENTATION.md +++ b/DOCUMENTATION.md @@ -1005,6 +1005,12 @@ Folder for storing local collections, created if not present. Default: `/var/lib/radicale/collections` +##### use_cache_subfolder_for_item + +Use subfolder `collections-cache' for cache file structure of item instead of inside collection folders, created if not present + +Default: `False` + ##### max_sync_token_age Delete sync-token that are older than the specified time. (seconds) diff --git a/config b/config index d3139133..1522e17a 100644 --- a/config +++ b/config @@ -138,6 +138,9 @@ # Folder for storing local collections, created if not present #filesystem_folder = /var/lib/radicale/collections +# Use subfolder `collections-cache' for item cache file structure instead of inside collection folder +#use_cache_subfolder_for_item = False + # Delete sync token that are older (seconds) #max_sync_token_age = 2592000 diff --git a/radicale/config.py b/radicale/config.py index 2a70a7e7..da7877ae 100644 --- a/radicale/config.py +++ b/radicale/config.py @@ -279,6 +279,10 @@ def json_str(value: Any) -> dict: "value": "/var/lib/radicale/collections", "help": "path where collections are stored", "type": filepath}), + ("use_cache_subfolder_for_item", { + "value": "False", + "help": "use subfolder `collections-cache' for item cache file structure instead of inside collection folder", + "type": bool}), ("max_sync_token_age", { "value": "2592000", # 30 days "help": "delete sync token that are older", diff --git a/radicale/rights/from_file.py b/radicale/rights/from_file.py index 20928a64..6d63c801 100644 --- a/radicale/rights/from_file.py +++ b/radicale/rights/from_file.py @@ -1,6 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2019 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -99,12 +100,9 @@ def authorization(self, user: str, path: str) -> str: user, sane_path, user_pattern, collection_pattern, section, permission) return permission - logger.debug("Rule %r:%r doesn't match %r:%r from section %r", - user, sane_path, user_pattern, collection_pattern, - section) if self._log_rights_rule_doesnt_match_on_debug: logger.debug("Rule %r:%r doesn't match %r:%r from section %r", user, sane_path, user_pattern, collection_pattern, section) - logger.info("Rights: %r:%r doesn't match any section", user, sane_path) + logger.debug("Rights: %r:%r doesn't match any section", user, sane_path) return "" diff --git a/radicale/storage/__init__.py b/radicale/storage/__init__.py index 3a5ef586..58f2a499 100644 --- a/radicale/storage/__init__.py +++ b/radicale/storage/__init__.py @@ -1,7 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2022 Unrud # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/radicale/storage/multifilesystem/__init__.py b/radicale/storage/multifilesystem/__init__.py index 67aa6a52..1b908239 100644 --- a/radicale/storage/multifilesystem/__init__.py +++ b/radicale/storage/multifilesystem/__init__.py @@ -1,7 +1,8 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2019 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +29,7 @@ from typing import ClassVar, Iterator, Optional, Type from radicale import config +from radicale.log import logger from radicale.storage.multifilesystem.base import CollectionBase, StorageBase from radicale.storage.multifilesystem.cache import CollectionPartCache from radicale.storage.multifilesystem.create_collection import \ @@ -89,3 +91,5 @@ class Storage( def __init__(self, configuration: config.Configuration) -> None: super().__init__(configuration) self._makedirs_synced(self._filesystem_folder) + logger.info("storage location: %r", self._filesystem_folder) + logger.info("storage cache subfolder usage for item: %s", self._use_cache_subfolder_for_item) diff --git a/radicale/storage/multifilesystem/base.py b/radicale/storage/multifilesystem/base.py index a7cc0bee..8d9f1940 100644 --- a/radicale/storage/multifilesystem/base.py +++ b/radicale/storage/multifilesystem/base.py @@ -70,6 +70,7 @@ class StorageBase(storage.BaseStorage): _filesystem_folder: str _filesystem_fsync: bool + _use_cache_subfolder_for_item: bool def __init__(self, configuration: config.Configuration) -> None: super().__init__(configuration) @@ -77,10 +78,17 @@ def __init__(self, configuration: config.Configuration) -> None: "storage", "filesystem_folder") self._filesystem_fsync = configuration.get( "storage", "_filesystem_fsync") + self._use_cache_subfolder_for_item = configuration.get( + "storage", "use_cache_subfolder_for_item") def _get_collection_root_folder(self) -> str: return os.path.join(self._filesystem_folder, "collection-root") + def _get_collection_cache_folder(self, path, folder, subfolder) -> str: + if (self._use_cache_subfolder_for_item is True) and (subfolder == "item"): + path = path.replace(os.path.join(self._filesystem_folder, "collection-root"), os.path.join(self._filesystem_folder, "collection-cache")) + return os.path.join(path, folder, subfolder) + def _fsync(self, f: IO[AnyStr]) -> None: if self._filesystem_fsync: try: diff --git a/radicale/storage/multifilesystem/cache.py b/radicale/storage/multifilesystem/cache.py index 31ab4715..bf596eb3 100644 --- a/radicale/storage/multifilesystem/cache.py +++ b/radicale/storage/multifilesystem/cache.py @@ -1,7 +1,8 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -81,8 +82,7 @@ def _store_item_cache(self, href: str, item: radicale_item.Item, if not cache_hash: cache_hash = self._item_cache_hash( item.serialize().encode(self._encoding)) - cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", - "item") + cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item") content = self._item_cache_content(item) self._storage._makedirs_synced(cache_folder) # Race: Other processes might have created and locked the file. @@ -95,8 +95,7 @@ def _store_item_cache(self, href: str, item: radicale_item.Item, def _load_item_cache(self, href: str, cache_hash: str ) -> Optional[CacheContent]: - cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", - "item") + cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item") try: with open(os.path.join(cache_folder, href), "rb") as f: hash_, *remainder = pickle.load(f) @@ -110,8 +109,7 @@ def _load_item_cache(self, href: str, cache_hash: str return None def _clean_item_cache(self) -> None: - cache_folder = os.path.join(self._filesystem_path, ".Radicale.cache", - "item") + cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item") self._clean_cache(cache_folder, ( e.name for e in os.scandir(cache_folder) if not os.path.isfile(os.path.join(self._filesystem_path, e.name)))) diff --git a/radicale/storage/multifilesystem/move.py b/radicale/storage/multifilesystem/move.py index 30995c4f..3518a3b4 100644 --- a/radicale/storage/multifilesystem/move.py +++ b/radicale/storage/multifilesystem/move.py @@ -1,7 +1,8 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2021 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,10 +42,8 @@ def move(self, item: radicale_item.Item, if item.collection._filesystem_path != to_collection._filesystem_path: self._sync_directory(item.collection._filesystem_path) # Move the item cache entry - cache_folder = os.path.join(item.collection._filesystem_path, - ".Radicale.cache", "item") - to_cache_folder = os.path.join(to_collection._filesystem_path, - ".Radicale.cache", "item") + cache_folder = self._get_collection_cache_folder(item.collection._filesystem_path, ".Radicale.cache", "item") + to_cache_folder = self._get_collection_cache_folder(to_collection._filesystem_path, ".Radicale.cache", "item") self._makedirs_synced(to_cache_folder) try: os.replace(os.path.join(cache_folder, item.href), diff --git a/radicale/storage/multifilesystem/upload.py b/radicale/storage/multifilesystem/upload.py index a9fcdc2c..01c52b75 100644 --- a/radicale/storage/multifilesystem/upload.py +++ b/radicale/storage/multifilesystem/upload.py @@ -1,7 +1,8 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2014 Jean-Marc Martins # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2018 Unrud +# Copyright © 2017-2022 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -75,8 +76,7 @@ def get_safe_free_hrefs(uid: str) -> Iterator[str]: yield radicale_item.find_available_uid( lambda href: not is_safe_free_href(href), suffix) - cache_folder = os.path.join(self._filesystem_path, - ".Radicale.cache", "item") + cache_folder = self._storage._get_collection_cache_folder(self._filesystem_path, ".Radicale.cache", "item") self._storage._makedirs_synced(cache_folder) for item in items: uid = item.uid diff --git a/radicale/tests/test_storage.py b/radicale/tests/test_storage.py index 22cc1f4c..5eca0753 100644 --- a/radicale/tests/test_storage.py +++ b/radicale/tests/test_storage.py @@ -1,6 +1,7 @@ # This file is part of Radicale - CalDAV and CardDAV server # Copyright © 2012-2017 Guillaume Ayoub -# Copyright © 2017-2019 Unrud +# Copyright © 2017-2022 Unrud +# Copyright © 2024-2024 Peter Bieringer # # This library is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -99,6 +100,22 @@ def test_item_cache_rebuild(self) -> None: assert answer1 == answer2 assert os.path.exists(os.path.join(cache_folder, "event1.ics")) + def test_item_cache_rebuild_subfolder(self) -> None: + """Delete the item cache and verify that it is rebuild.""" + self.configure({"storage": {"use_cache_subfolder_for_item": "True"}}) + self.mkcalendar("/calendar.ics/") + event = get_file_content("event1.ics") + path = "/calendar.ics/event1.ics" + self.put(path, event) + _, answer1 = self.get(path) + cache_folder = os.path.join(self.colpath, "collection-cache", + "calendar.ics", ".Radicale.cache", "item") + assert os.path.exists(os.path.join(cache_folder, "event1.ics")) + shutil.rmtree(cache_folder) + _, answer2 = self.get(path) + assert answer1 == answer2 + assert os.path.exists(os.path.join(cache_folder, "event1.ics")) + def test_put_whole_calendar_uids_used_as_file_names(self) -> None: """Test if UIDs are used as file names.""" _TestBaseRequests.test_put_whole_calendar(