diff --git a/common/snapshots.py b/common/snapshots.py index f86f66418..debdeaf9c 100644 --- a/common/snapshots.py +++ b/common/snapshots.py @@ -3,16 +3,19 @@ # SPDX-FileCopyrightText: © 2008-2022 Richard Bailey # SPDX-FileCopyrightText: © 2008-2022 Germar Reitze # SPDX-FileCopyrightText: © 2008-2022 Taylor Raack +# SPDX-FileCopyrightText: © 2024 Christian Buhtz # # SPDX-License-Identifier: GPL-2.0-or-later # -# This file is part of the program "Back In time" which is released under GNU +# This file is part of the program "Back In Time" which is released under GNU # General Public License v2 (GPLv2). See file/folder LICENSE or go to # . +from __future__ import annotations import os from pathlib import Path import stat import datetime +import calendar import gettext import bz2 import pwd @@ -924,6 +927,7 @@ def backup(self, force=False): # "continue on errors" is enabled if not ret_error: + # Start auto- and smart-remove self.freeSpace(now) self.setTakeSnapshotMessage( 0, _('Please be patient. Finalizing…')) @@ -1517,67 +1521,80 @@ def takeSnapshot(self, sid, now, include_folders): return [True, has_errors] def smartRemoveKeepAll(self, - snapshots, - min_date, - max_date): + snapshots: list[SID], + min_date: datetime.date, + max_date: datetime.date) -> set[SID]: """ - Return all snapshots between ``min_date`` and ``max_date``. + Return all snapshots in the timedelta beginning with ``min_date`` and + ending before ``max_date``. Args: - snapshots (list): full list of :py:class:`SID` objects - min_date (datetime.date): minimum date for snapshots to keep - max_date (datetime.date): maximum date for snapshots to keep + snapshots (list): Full list of :py:class:`SID` objects. + min_date (datetime.date): Minimum date (included in the range). + max_date (datetime.date): Maximum date (excluded from the range). Returns: - set: set of snapshots that should be kept + set: Set of snapshots that should be kept. """ - min_id = SID(min_date, self.config) - max_id = SID(max_date, self.config) + logger.debug(f'Keep all >= {min_date} < {max_date}', self) - logger.debug("Keep all >= %s and < %s" %(min_id, max_id), self) + result = filter(lambda sid: sid.date.date() >= min_date + and sid.date.date() < max_date, + snapshots) - return set([sid for sid in snapshots if sid >= min_id and sid < max_id]) + return set(result) def smartRemoveKeepFirst(self, snapshots, min_date, max_date, - keep_healthy = False): - """ - Return only the first snapshot between ``min_date`` and ``max_date``. + keep_healthy=False): + """Return the first snapshot between ``min_date`` and ``max_date``. + + The first snapshot in ``snapshots`` that hit the timedetla beginning + with ``min_date`` and ending before ``max_date`` will be returned. + Snapshots outthat that range are also lost. The list is not ordered by + date. Args: - snapshots (list): full list of :py:class:`SID` objects - min_date (datetime.date): minimum date for snapshots to keep - max_date (datetime.date): maximum date for snapshots to keep - keep_healthy (bool): return the first healthy snapshot (not - marked as failed) instead of the first - at all. If all snapshots failed this - will again return the very first - snapshot + snapshots (list): Full list of :py:class:`SID` objects. + min_date (datetime.date): Minimum date (included in the range). + max_date (datetime.date): Maximum date (excluded from the range). + keep_healthy (bool): Return the first healthy snapshot (not marked + as failed) instead of the first at all. If all snapshots failed + this will again return the very first snapshot. Returns: - set: set of snapshots that should be kept + set: Set of one snapshot that should be kept or an empty set. + + TODO: It should compare datest not SIDs because of their tag. """ + # print(f'smartRemoveKeepFirst() :: {min_date=} {max_date=}') # DEBUG min_id = SID(min_date, self.config) max_id = SID(max_date, self.config) - logger.debug("Keep first >= %s and < %s" %(min_id, max_id), self) + logger.debug("Keep first >= %s and < %s" % (min_id, max_id), self) for sid in snapshots: # try to keep the first healthy snapshot if keep_healthy and sid.failed: - logger.debug("Do not keep failed snapshot %s" %sid, self) + logger.debug("Do not keep failed snapshot %s" % sid, self) continue + + # DEBUG + # print(f'smartRemoveKeepFirst() :: for sid ... sid={str(sid)}') + if sid >= min_id and sid < max_id: + # print(f' return {str(sid)}') return set([sid]) + # if all snapshots failed return the first snapshot # no matter if it has errors if keep_healthy: return self.smartRemoveKeepFirst(snapshots, min_date, max_date, - keep_healthy = False) + keep_healthy=False) return set() def incMonth(self, date): @@ -1591,12 +1608,13 @@ def incMonth(self, date): Returns: datetime.date: 1st day of next month """ - y = date.year - m = date.month + 1 - if m > 12: - m = 1 - y = y + 1 - return datetime.date(y, m, 1) + # Last day in current month + last = datetime.date( + year=date.year, + month=date.month, + day=calendar.monthrange(date.year, date.month)[1]) + + return last + datetime.timedelta(days=1) def decMonth(self, date): """ @@ -1610,12 +1628,14 @@ def decMonth(self, date): Returns: datetime.date: 1st day of previous month """ - y = date.year - m = date.month - 1 - if m < 1: - m = 12 - y = y - 1 - return datetime.date(y, m, 1) + # First day of current month + first = datetime.date(year=date.year, month=date.month, day=1) + + # Last day of previous month + prev = first - datetime.timedelta(days=1) + + # First day of previous month + return datetime.date(year=prev.year, month=prev.month, day=1) def smartRemoveList(self, now_full, @@ -1623,9 +1643,7 @@ def smartRemoveList(self, keep_one_per_day, keep_one_per_week, keep_one_per_month): - """ - Get a list of old snapshots that should be removed based on configurable - intervals. + """Get list of backups to be removed based on configurable intervals. Args: now_full (datetime.datetime): date and time when takeSnapshot was @@ -1641,8 +1659,10 @@ def smartRemoveList(self, Returns: list: snapshots that should be removed + """ - snapshots = listSnapshots(self.config) + # Latest/younges backup first, the oldest is last + snapshots = listSnapshots(self.config, reverse=True) logger.debug(f'Considered: {snapshots}', self) if len(snapshots) <= 1: @@ -1654,7 +1674,7 @@ def smartRemoveList(self, now = now_full.date() - # keep the last snapshot + # keep the last/youngest backup keep = set([snapshots[0]]) # keep all for the last keep_all days @@ -1821,28 +1841,28 @@ def smartRemove(self, del_snapshots, log = None): self.remove(sid) def freeSpace(self, now): - """ - Remove old snapshots on based on different rules (only if enabled). - First rule is to remove snapshots older than X years. Next will call - :py:func:`smartRemove` to remove snapshots based on - configurable intervals. Third rule is to remove the oldest snapshot - until there is enough free space. Last rule will remove the oldest - snapshot until there are enough free inodes. + """Remove old backups based on several rules (if enabled). + + Rules are considered in the following order: + 1. Remove snapshots older than X years. + 2. Smart-remove rules with calling :py:func:`smartRemoveList`. See + there for details. + 3. Remove the oldest backup until there is enough free space. + 4. Remove the oldest backup until there are enough free inodes. - 'last_snapshot' symlink will be fixed when done. + The 'last_snapshot' symlink will be fixed when done. Args: - now (datetime.datetime): date and time when takeSnapshot was - started + now (datetime.datetime): Timestamp when takeSnapshot was started. """ - snapshots = listSnapshots(self.config, reverse = False) + snapshots = listSnapshots(self.config, reverse=False) if not snapshots: logger.debug('No snapshots. Skip freeSpace', self) return last_snapshot = snapshots[-1] - #remove old backups + # Remove old backups if self.config.removeOldSnapshotsEnabled(): self.setTakeSnapshotMessage(0, _('Removing old snapshots')) @@ -1877,7 +1897,7 @@ def freeSpace(self, now): keep_one_per_month) self.smartRemove(del_snapshots) - # try to keep min free space + # Try to keep min free space if self.config.minFreeSpaceEnabled(): self.setTakeSnapshotMessage(0, _('Trying to keep min free space')) @@ -1885,7 +1905,7 @@ def freeSpace(self, now): logger.debug("Keep min free disk space: {} MiB".format(minFreeSpace), self) - snapshots = listSnapshots(self.config, reverse = False) + snapshots = listSnapshots(self.config, reverse=False) while True: if len(snapshots) <= 1: @@ -1913,7 +1933,7 @@ def freeSpace(self, now): self.remove(snapshots[0]) del snapshots[0] - #try to keep free inodes + # Try to keep free inodes if self.config.minFreeInodesEnabled(): minFreeInodes = self.config.minFreeInodes() self.setTakeSnapshotMessage( @@ -1934,7 +1954,7 @@ def freeSpace(self, now): try: info = os.statvfs(self.config.snapshotsPath()) free_inodes = info.f_favail - max_inodes = info.f_files + max_inodes = info.f_files except Exception as e: logger.debug('Failed to get free inodes for snapshot path %s: %s' % (self.config.snapshotsPath(), str(e)), @@ -1955,7 +1975,7 @@ def freeSpace(self, now): self.remove(snapshots[0]) del snapshots[0] - #set correct last snapshot again + # Set correct last snapshot again if last_snapshot is not snapshots[-1]: self.createLastSnapshotSymlink(snapshots[-1]) @@ -2419,6 +2439,8 @@ def __init__(self, date, cfg): if isinstance(date, datetime.datetime): self.sid = '-'.join((date.strftime('%Y%m%d-%H%M%S'), self.config.tag(self.profileID))) + # TODO: Don't use "date" as attribute name. Btw: It is not a date + # but a datetime. self.date = date elif isinstance(date, datetime.date): @@ -2434,10 +2456,12 @@ def __init__(self, date, cfg): raise LastSnapshotSymlink() else: - raise ValueError("'date' must be in snapshot ID format (e.g 20151218-173512-123)") + raise ValueError("'date' must be in snapshot ID format " + f"(e.g 20151218-173512-123) but is '{date}'") else: - raise TypeError("'date' must be an instance of str, datetime.date or datetime.datetime") + raise TypeError("'date' must be an instance of str, datetime.date " + f"or datetime.datetime but is '{date}'") def __repr__(self): return self.sid @@ -3061,17 +3085,16 @@ def path(self, *path, use_mode = []): return os.path.join(os.sep, *path) -def iterSnapshots(cfg, includeNewSnapshot = False): - """ - A generator to iterate over snapshots in current snapshot path. +def iterSnapshots(cfg, includeNewSnapshot=False): + """A generator to iterate over snapshots in current snapshot path. Args: - cfg (config.Config): current config - includeNewSnapshot (bool): include a NewSnapshot instance if - 'new_snapshot' folder is available. + cfg (config.Config): Current config instance. + includeNewSnapshot (bool): Include a NewSnapshot instance if + 'new_snapshot' directory is available (default: False). Yields: - SID: snapshot IDs + SID: Snapshot IDs """ path = cfg.snapshotsFullPath() @@ -3103,21 +3126,22 @@ def iterSnapshots(cfg, includeNewSnapshot = False): "'{}' is not a snapshot ID: {}".format(item, str(e))) -def listSnapshots(cfg, includeNewSnapshot = False, reverse = True): +def listSnapshots(cfg, includeNewSnapshot=False, reverse=True): """ List of snapshots in current snapshot path. Args: - cfg (config.Config): current config (config.Config instance) - includeNewSnapshot (bool): include a NewSnapshot instance if - 'new_snapshot' folder is available - reverse (bool): sort reverse + cfg (config.Config): Current config instance. + includeNewSnapshot (bool): Include a NewSnapshot instance if + 'new_snapshot' directory is available (default: False). + reverse (bool): Sort reverse (default: True). Returns: - list: list of :py:class:`SID` objects + list: List of :py:class:`SID` objects. """ ret = list(iterSnapshots(cfg, includeNewSnapshot)) - ret.sort(reverse = reverse) + ret.sort(reverse=reverse) + return ret diff --git a/common/test/test_config_crontab.py b/common/test/test_config_crontab.py index 693a68b72..527ffe210 100644 --- a/common/test/test_config_crontab.py +++ b/common/test/test_config_crontab.py @@ -75,27 +75,27 @@ def setUp(self): def _create_config_file(self, parent_path): """Minimal config file""" - # pylint: disable-next=R0801 + # pylint: disable-next=duplicate-code cfg_content = inspect.cleandoc(''' config.version=6 - profile1.snapshots.include.1.type=0 - profile1.snapshots.include.1.value=rootpath/source - profile1.snapshots.include.size=1 profile1.snapshots.no_on_battery=false profile1.snapshots.notify.enabled=true profile1.snapshots.path=rootpath/destination profile1.snapshots.path.host=test-host profile1.snapshots.path.profile=1 profile1.snapshots.path.user=test-user - profile1.snapshots.preserve_acl=false - profile1.snapshots.preserve_xattr=false profile1.snapshots.remove_old_snapshots.enabled=true profile1.snapshots.remove_old_snapshots.unit=80 profile1.snapshots.remove_old_snapshots.value=10 + profile1.snapshots.include.1.type=0 + profile1.snapshots.include.1.value=rootpath/source + profile1.snapshots.include.size=1 + profile1.snapshots.preserve_acl=false + profile1.snapshots.preserve_xattr=false profile1.snapshots.rsync_options.enabled=false profile1.snapshots.rsync_options.value= profiles.version=1 - ''') + ''') # pylint: disable=R0801 # config file location config_fp = parent_path / 'config_path' / 'config' diff --git a/common/test/test_lint.py b/common/test/test_lint.py index 8794642c4..5c851be72 100644 --- a/common/test/test_lint.py +++ b/common/test/test_lint.py @@ -322,7 +322,10 @@ def test050_pylint_exclusive_ruleset(self): 'W4904', # deprecated-class 'R0202', # no-classmethod-decorator 'R0203', # no-staticmethod-decorator - 'R0801', # duplicate-code + # See PyLint bugs: + # https://github.com/pylint-dev/pylint/issues/214 + # https://github.com/pylint-dev/pylint/issues/7920 + # 'R0801', # duplicate-code # Enable asap. This list is a selection of existing (not all!) # problems currently existing in the BIT code base. Quite easy to diff --git a/common/test/test_sid.py b/common/test/test_sid.py index 84346b94c..bc8d44885 100644 --- a/common/test/test_sid.py +++ b/common/test/test_sid.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation,Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - import os import sys import unittest diff --git a/common/test/test_snapshots.py b/common/test/test_snapshots.py index efe4e9016..2f6f2f586 100644 --- a/common/test/test_snapshots.py +++ b/common/test/test_snapshots.py @@ -19,7 +19,6 @@ import string import unittest from unittest.mock import patch -from datetime import date, datetime from tempfile import TemporaryDirectory from test import generic from test.constants import CURRENTUSER, CURRENTGROUP, CURRENTGID, CURRENTUID @@ -300,146 +299,6 @@ def test_error(self): '[E] Error: rsync: send_files failed to open "/foo/bar": Operation not permitted (1)\n', f.read()) -class SmartRemove(generic.SnapshotsTestCase): - def test_increment_month(self): - self.assertEqual(self.sn.incMonth(date(2016, 4, 21)), date(2016, 5, 1)) - self.assertEqual(self.sn.incMonth(date(2016, 12, 24)), date(2017, 1, 1)) - - def test_decrement_month(self): - self.assertEqual(self.sn.decMonth(date(2016, 4, 21)), date(2016, 3, 1)) - self.assertEqual(self.sn.decMonth(date(2016, 1, 14)), date(2015, 12, 1)) - - def test_keep_all(self): - sid1 = snapshots.SID('20160424-215134-123', self.cfg) - sid2 = snapshots.SID('20160422-030324-123', self.cfg) - sid3 = snapshots.SID('20160422-020324-123', self.cfg) - sid4 = snapshots.SID('20160422-010324-123', self.cfg) - sid5 = snapshots.SID('20160421-013218-123', self.cfg) - sid6 = snapshots.SID('20160410-134327-123', self.cfg) - sids = [sid1, sid2, sid3, sid4, sid5, sid6] - - keep = self.sn.smartRemoveKeepAll(sids, - date(2016, 4, 20), - date(2016, 4, 23)) - self.assertSetEqual(keep, set((sid2, sid3, sid4, sid5))) - - keep = self.sn.smartRemoveKeepAll(sids, - date(2016, 4, 11), - date(2016, 4, 18)) - self.assertSetEqual(keep, set()) - - def test_keep_first(self): - sid1 = snapshots.SID('20160424-215134-123', self.cfg) - sid2 = snapshots.SID('20160422-030324-123', self.cfg) - sid3 = snapshots.SID('20160422-020324-123', self.cfg) - sid4 = snapshots.SID('20160422-010324-123', self.cfg) - sid5 = snapshots.SID('20160421-013218-123', self.cfg) - sid6 = snapshots.SID('20160410-134327-123', self.cfg) - sids = [sid1, sid2, sid3, sid4, sid5, sid6] - - keep = self.sn.smartRemoveKeepFirst(sids, - date(2016, 4, 20), - date(2016, 4, 23)) - self.assertSetEqual(keep, set((sid2,))) - - keep = self.sn.smartRemoveKeepFirst(sids, - date(2016, 4, 11), - date(2016, 4, 18)) - self.assertSetEqual(keep, set()) - - def test_keep_first_no_errors(self): - sid1 = snapshots.SID('20160424-215134-123', self.cfg) - sid2 = snapshots.SID('20160422-030324-123', self.cfg) - sid2.makeDirs() - sid2.failed = True - sid3 = snapshots.SID('20160422-020324-123', self.cfg) - sid4 = snapshots.SID('20160422-010324-123', self.cfg) - sid5 = snapshots.SID('20160421-013218-123', self.cfg) - sid6 = snapshots.SID('20160410-134327-123', self.cfg) - sids = [sid1, sid2, sid3, sid4, sid5, sid6] - - # keep the first healthy snapshot - keep = self.sn.smartRemoveKeepFirst(sids, - date(2016, 4, 20), - date(2016, 4, 23), - keep_healthy = True) - self.assertSetEqual(keep, set((sid3,))) - - # if all snapshots failed, keep the first at all - for sid in (sid3, sid4, sid5): - sid.makeDirs() - sid.failed = True - keep = self.sn.smartRemoveKeepFirst(sids, - date(2016, 4, 20), - date(2016, 4, 23), - keep_healthy = True) - self.assertSetEqual(keep, set((sid2,))) - - def test_smart_remove_list(self): - sid1 = snapshots.SID('20160424-215134-123', self.cfg) - sid2 = snapshots.SID('20160422-030324-123', self.cfg) - sid3 = snapshots.SID('20160422-020324-123', self.cfg) - sid4 = snapshots.SID('20160422-010324-123', self.cfg) - sid5 = snapshots.SID('20160421-033218-123', self.cfg) - sid6 = snapshots.SID('20160421-013218-123', self.cfg) - sid7 = snapshots.SID('20160420-013218-123', self.cfg) - sid8 = snapshots.SID('20160419-013218-123', self.cfg) - sid9 = snapshots.SID('20160419-003218-123', self.cfg) - sid10 = snapshots.SID('20160418-003218-123', self.cfg) - sid11 = snapshots.SID('20160417-033218-123', self.cfg) - sid12 = snapshots.SID('20160417-003218-123', self.cfg) - sid13 = snapshots.SID('20160416-134327-123', self.cfg) - sid14 = snapshots.SID('20160416-114327-123', self.cfg) - sid15 = snapshots.SID('20160415-134327-123', self.cfg) - sid16 = snapshots.SID('20160411-134327-123', self.cfg) - sid17 = snapshots.SID('20160410-134327-123', self.cfg) - sid18 = snapshots.SID('20160409-134327-123', self.cfg) - sid19 = snapshots.SID('20160407-134327-123', self.cfg) - sid20 = snapshots.SID('20160403-134327-123', self.cfg) - sid21 = snapshots.SID('20160402-134327-123', self.cfg) - sid22 = snapshots.SID('20160401-134327-123', self.cfg) - sid23 = snapshots.SID('20160331-134327-123', self.cfg) - sid24 = snapshots.SID('20160330-134327-123', self.cfg) - sid25 = snapshots.SID('20160323-133715-123', self.cfg) - sid26 = snapshots.SID('20160214-134327-123', self.cfg) - sid27 = snapshots.SID('20160205-134327-123', self.cfg) - sid28 = snapshots.SID('20160109-134327-123', self.cfg) - sid29 = snapshots.SID('20151224-134327-123', self.cfg) - sid30 = snapshots.SID('20150904-134327-123', self.cfg) - sid31 = snapshots.SID('20140904-134327-123', self.cfg) - - sids = [ sid1, sid2, sid3, sid4, sid5, sid6, sid7, sid8, sid9, - sid10, sid11, sid12, sid13, sid14, sid15, sid16, sid17, sid18, sid19, - sid20, sid21, sid22, sid23, sid24, sid25, sid26, sid27, sid28, sid29, - sid30, sid31] - for sid in sids: - sid.makeDirs() - now = datetime(2016, 4, 24, 21, 51, 34) - - del_snapshots = self.sn.smartRemoveList(now, - 3, #keep_all - 7, #keep_one_per_day - 5, #keep_one_per_week - 3 #keep_one_per_month - ) - self.assertListEqual(del_snapshots, [sid6, sid9, sid12, sid13, sid14, - sid15, sid16, sid18, sid19, sid21, - sid22, sid24, sid27, sid28, sid30]) - - # test failed snapshots - for sid in (sid5, sid8, sid11, sid12, sid20, sid21, sid22): - sid.failed = True - del_snapshots = self.sn.smartRemoveList(now, - 3, #keep_all - 7, #keep_one_per_day - 5, #keep_one_per_week - 3 #keep_one_per_month - ) - self.assertListEqual(del_snapshots, [sid5, sid8, sid11, sid12, sid14, - sid15, sid16, sid18, sid19, sid20, sid21, - sid22, sid24, sid27, sid28, sid30]) - - class SnapshotWithSID(generic.SnapshotsWithSidTestCase): def test_backup_config(self): self.sn.backupConfig(self.sid) diff --git a/common/test/test_snapshots_autoremove.py b/common/test/test_snapshots_autoremove.py new file mode 100644 index 000000000..00e623dbe --- /dev/null +++ b/common/test/test_snapshots_autoremove.py @@ -0,0 +1,892 @@ +# SPDX-FileCopyrightText: © 2008-2022 Oprea Dan +# SPDX-FileCopyrightText: © 2008-2022 Bart de Koning +# SPDX-FileCopyrightText: © 2008-2022 Richard Bailey +# SPDX-FileCopyrightText: © 2008-2022 Germar Reitze +# SPDX-FileCopyrightText: © 2024 Christian Buhtz +# +# SPDX-License-Identifier: GPL-2.0-or-later +# +# This file is part of the program "Back In Time" which is released under GNU +# General Public License v2 (GPLv2). See LICENSES directory or go to +# . +import os +import sys +import inspect +from unittest import mock +from typing import Union +from datetime import date, time, datetime, timedelta +from pathlib import Path +from tempfile import TemporaryDirectory +from test import generic +import pyfakefs.fake_filesystem_unittest as pyfakefs_ut +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import config # noqa: E402,RUF100 +import snapshots # noqa: E402,RUF100 + + +def dt2sidstr(d: Union[date, datetime], t: time = None, tag: int = 123): + """Create a SID identification string out ouf a date and time infos.""" + if not t: + try: + # If d is datetime + t = d.time() + except AttributeError: + t = time(7, 42, 31) + + return datetime.combine(d, t).strftime(f'%Y%m%d-%H%M%S-{tag}') + + +def dt2str(d: Union[date, datetime]): + return d.strftime('%a %d %b %Y') + + +def sid2str(sid): + """Convert a SID string into human readable date incl. weekday.""" + + if isinstance(sid, snapshots.SID): + sid = str(sid) + + result = datetime.strptime(sid.split('-')[0], '%Y%m%d') \ + .date().strftime('%c').strip() + + if result.endswith(' 00:00:00'): + result = result[:-9] + + return result + + +def create_SIDs(start_date: Union[date, datetime, list[date]], + days: int, + cfg: config.Config): + sids = [] + + if isinstance(start_date, list): + the_dates = start_date + else: + the_dates = [start_date + timedelta(days=x) for x in range(days)] + + for d in the_dates: + sids.append(snapshots.SID(dt2sidstr(d), cfg)) + + return list(reversed(sids)) + + +class KeepFirst(pyfakefs_ut.TestCase): + """Test Snapshot.removeKeepFirst(). + + PyFakeFS is used here because of Config file dependency.""" + + def setUp(self): + """Setup a fake filesystem.""" + self.setUpPyfakefs(allow_root_user=False) + + # cleanup() happens automatically + self._temp_dir = TemporaryDirectory(prefix='bit.') + # Workaround: tempfile and pathlib not compatible yet + self.temp_path = Path(self._temp_dir.name) + + self._config_fp = self._create_config_file(parent_path=self.temp_path) + self.cfg = config.Config(str(self._config_fp)) + + self.sn = snapshots.Snapshots(self.cfg) + + def _create_config_file(self, parent_path): + """Minimal config file""" + # pylint: disable-next=R0801 + cfg_content = inspect.cleandoc(''' + config.version=6 + profile1.snapshots.include.1.type=0 + profile1.snapshots.include.1.value=rootpath/source + profile1.snapshots.include.size=1 + profile1.snapshots.no_on_battery=false + profile1.snapshots.notify.enabled=true + profile1.snapshots.path=rootpath/destination + profile1.snapshots.path.host=test-host + profile1.snapshots.path.profile=1 + profile1.snapshots.path.user=test-user + profile1.snapshots.preserve_acl=false + profile1.snapshots.preserve_xattr=false + profile1.snapshots.remove_old_snapshots.enabled=true + profile1.snapshots.remove_old_snapshots.unit=80 + profile1.snapshots.remove_old_snapshots.value=10 + profile1.snapshots.rsync_options.enabled=false + profile1.snapshots.rsync_options.value= + profiles.version=1 + ''') + + # config file location + config_fp = parent_path / 'config_path' / 'config' + config_fp.parent.mkdir() + config_fp.write_text(cfg_content, 'utf-8') + + return config_fp + + def test_one_but_set(self): + """Return value is always a set with always only one element.""" + # One SID for each of 20 days beginning with 5th March 2022 07:42:31 + sids = create_SIDs( + datetime(2020, 3, 5, 7, 42, 31), 700, self.cfg) + + sut = self.sn.smartRemoveKeepFirst( + sids, date(2021, 8, 5), datetime.now().date()) + + self.assertIsInstance(sut, set) + self.assertTrue(len(sut), 1) + + def test_simple_one(self): + """First element in a range of SIDs""" + sids = create_SIDs( + datetime(2022, 3, 5, 7, 42, 31), 20, self.cfg) + + sut = self.sn.smartRemoveKeepFirst( + sids, date(2022, 3, 5), datetime.now().date()) + + sut = sut.pop() + + self.assertTrue(str(sut).startswith('20220324-074231-')) + + def test_no_date_ordering(self): + """Hit first in the list and ignoring its date ordering. + + The list of snapshots is not ordered anywhere.""" + sids = [] + # April, 2016... + for timestamp_string in ['20160424-215134-123', # …24th + # This SID will hit because it is the first + # in the range specified. + '20160422-030324-123', # …22th + '20160422-020324-123', # …22th + '20160422-010324-123', # …22th + # This might be the earliest/first SID in the + # date range specified but it is not the first + # in the list. So it won't be hit. + '20160421-013218-123', # …21th + '20160410-134327-123']: # …10th + sids.append(snapshots.SID(timestamp_string, self.cfg)) + + sut = self.sn.smartRemoveKeepFirst(sids, + date(2016, 4, 20), + date(2016, 4, 23)) + + self.assertEqual(str(sut.pop()), '20160422-030324-123') + + def test_keep_first_range_outside(self): + sids = [] + # April, 2016... + for timestamp_string in ['20160424-215134-123', # …24th + '20160422-030324-123', # …22th + '20160422-020324-123', # …22th + '20160422-010324-123', # …22th + '20160421-013218-123', # …21th + '20160410-134327-123']: # …10th + sids.append(snapshots.SID(timestamp_string, self.cfg)) + + # Between 11th and 18th April + sut = self.sn.smartRemoveKeepFirst(sids, + date(2016, 4, 11), + date(2016, 4, 18)) + + # None will hit, because no SID in that range. + self.assertEqual(len(sut), 0) + + @mock.patch.object(snapshots.SID, 'failed', new_callable=lambda: True) + def test_all_invalid(self, _mock_failed): + """All SIDS invalid (not healthy)""" + sids = create_SIDs( + datetime(2022, 3, 5, 7, 42, 31), 20, self.cfg) + + # By default healthy/invalid status is irrelevant + sut = self.sn.smartRemoveKeepFirst( + sids, date(2022, 3, 5), datetime.now().date()) + self.assertTrue(len(sut), 1) + + # Now make it relevant + sut = self.sn.smartRemoveKeepFirst( + sids, date(2022, 3, 5), datetime.now().date(), + keep_healthy=True) + self.assertTrue(len(sut), 0) + + @mock.patch.object(snapshots.SID, 'failed', new_callable=mock.PropertyMock) + def test_ignore_unhealthy(self, mock_failed): + # The second call to failed-property returns True + mock_failed.side_effect = [False, True, False, False, False, False] + sids = [] + for timestamp_string in ['20160424-215134-123', + # could be hit, but is NOT healthy + '20160422-030324-123', + # hit this + '20160422-020324-123', + '20160422-010324-123', + '20160421-013218-123', + '20160410-134327-123']: + sids.append(snapshots.SID(timestamp_string, self.cfg)) + + # keep the first healthy snapshot + sut = self.sn.smartRemoveKeepFirst(sids, + date(2016, 4, 20), + date(2016, 4, 23), + keep_healthy=True) + self.assertEqual(str(sut.pop()), '20160422-020324-123') + + +class KeepAll(pyfakefs_ut.TestCase): + """Test Snapshot.removeKeepAll(). + + PyFakeFS is used here because of Config file dependency.""" + + def setUp(self): + """Setup a fake filesystem.""" + self.setUpPyfakefs(allow_root_user=False) + + # cleanup() happens automatically + self._temp_dir = TemporaryDirectory(prefix='bit.') + # Workaround: tempfile and pathlib not compatible yet + self.temp_path = Path(self._temp_dir.name) + + self._config_fp = self._create_config_file(parent_path=self.temp_path) + self.cfg = config.Config(str(self._config_fp)) + + self.sn = snapshots.Snapshots(self.cfg) + + def _create_config_file(self, parent_path): + """Minimal config file""" + # pylint: disable-next=R0801 + cfg_content = inspect.cleandoc(''' + config.version=6 + profile1.snapshots.include.1.type=0 + profile1.snapshots.include.1.value=rootpath/source + profile1.snapshots.include.size=1 + profile1.snapshots.no_on_battery=false + profile1.snapshots.notify.enabled=true + profile1.snapshots.path=rootpath/destination + profile1.snapshots.path.host=test-host + profile1.snapshots.path.profile=1 + profile1.snapshots.path.user=test-user + profile1.snapshots.preserve_acl=false + profile1.snapshots.preserve_xattr=false + profile1.snapshots.remove_old_snapshots.enabled=true + profile1.snapshots.remove_old_snapshots.unit=80 + profile1.snapshots.remove_old_snapshots.value=10 + profile1.snapshots.rsync_options.enabled=false + profile1.snapshots.rsync_options.value= + profiles.version=1 + ''') + + # config file location + config_fp = parent_path / 'config_path' / 'config' + config_fp.parent.mkdir() + config_fp.write_text(cfg_content, 'utf-8') + + return config_fp + + def test_simple(self): + """Simple""" + # 10th to 25th + sids = create_SIDs(datetime(2024, 2, 10), 15, self.cfg) + + # keep... + sut = self.sn.smartRemoveKeepAll( + sids, + # ... from 12th ... + date(2024, 2, 12), + # ... to 19th. + date(2024, 2, 20) + ) + + self.assertEqual(len(sut), 8) + + sut = sorted(sut) + + self.assertEqual(sut[0].date.date(), date(2024, 2, 12)) + self.assertEqual(sut[1].date.date(), date(2024, 2, 13)) + self.assertEqual(sut[2].date.date(), date(2024, 2, 14)) + self.assertEqual(sut[3].date.date(), date(2024, 2, 15)) + self.assertEqual(sut[4].date.date(), date(2024, 2, 16)) + self.assertEqual(sut[5].date.date(), date(2024, 2, 17)) + self.assertEqual(sut[6].date.date(), date(2024, 2, 18)) + self.assertEqual(sut[7].date.date(), date(2024, 2, 19)) + + +# class OnePerWeek(pyfakefs_ut.TestCase): +# """Covering the smart remove setting 'Keep one snapshot per week for the +# last N weeks'. + +# That logic is implemented in 'Snapshots.smartRemoveList()' but not testable +# in isolation. So for a first shot we just duplicate that code in this +# tests (see self._org()). +# """ + +# def setUp(self): +# """Setup a fake filesystem.""" +# self.setUpPyfakefs(allow_root_user=False) + +# # cleanup() happens automatically +# self._temp_dir = TemporaryDirectory(prefix='bit.') +# # Workaround: tempfile and pathlib not compatible yet +# self.temp_path = Path(self._temp_dir.name) + +# self._config_fp = self._create_config_file(parent_path=self.temp_path) +# self.cfg = config.Config(str(self._config_fp)) + +# self.sn = snapshots.Snapshots(self.cfg) + +# def _create_config_file(self, parent_path): +# """Minimal config file""" +# # pylint: disable-next=R0801 +# cfg_content = inspect.cleandoc(''' +# config.version=6 +# profile1.snapshots.include.1.type=0 +# profile1.snapshots.include.1.value=rootpath/source +# profile1.snapshots.include.size=1 +# profile1.snapshots.no_on_battery=false +# profile1.snapshots.notify.enabled=true +# profile1.snapshots.path=rootpath/destination +# profile1.snapshots.path.host=test-host +# profile1.snapshots.path.profile=1 +# profile1.snapshots.path.user=test-user +# profile1.snapshots.preserve_acl=false +# profile1.snapshots.preserve_xattr=false +# profile1.snapshots.remove_old_snapshots.enabled=true +# profile1.snapshots.remove_old_snapshots.unit=80 +# profile1.snapshots.remove_old_snapshots.value=10 +# profile1.snapshots.rsync_options.enabled=false +# profile1.snapshots.rsync_options.value= +# profiles.version=1 +# ''') + +# # config file location +# config_fp = parent_path / 'config_path' / 'config' +# config_fp.parent.mkdir() +# config_fp.write_text(cfg_content, 'utf-8') + +# return config_fp + +# def _org(self, now, n_weeks, snapshots, keep_healthy=True): +# """Keep one per week for the last n_weeks weeks. + +# Copied and slightly refactored from inside +# 'Snapshots.smartRemoveList()'. +# """ +# print(f'\n_org() :: now={dt2str(now)} {n_weeks=}') +# keep = set() + +# # Sunday ??? (Sonntag) of previous week +# idx_date = now - timedelta(days=now.weekday() + 1) + +# print(f' for-loop... idx_date={dt2str(idx_date)}') +# for _ in range(0, n_weeks): + +# min_date = idx_date +# max_date = idx_date + timedelta(days=7) + +# print(f' from {dt2str(min_date)} to/before {dt2str(max_date)}') +# keep |= self.sn.smartRemoveKeepFirst( +# snapshots, +# min_date, +# max_date, +# keep_healthy=keep_healthy) +# print(f' {keep=}') + +# idx_date -= timedelta(days=7) +# print(f' new idx_date={dt2str(idx_date)}') +# print(' ...end loop') + +# return keep + +# def test_foobar(self): +# # start = date(2022, 1, 15) +# now = date(2024, 11, 26) +# # sids = create_SIDs(start, 9*7+3, self.cfg) +# sids = create_SIDs( +# [ +# date(2024, 11, 2), +# date(2024, 11, 9), +# date(2024, 11, 16), +# date(2024, 11, 23), +# # date(2024, 11, 25) +# ], +# None, +# self.cfg +# ) + +# weeks = 3 +# sut = self._org( +# # "Today" is Thursday 28th March +# now=now, +# # Keep the last week +# n_weeks=weeks, +# snapshots=sids) + +# print(f'\noldest snapshot: {sid2str(sids[0])}') +# for s in sorted(sut): +# print(f'keep: {sid2str(s)}') +# print(f'from/now: {dt2str(now)} {weeks=}') +# print(f'latest snapshot: {sid2str(sids[-1])}') + +# def test_sunday_last_week(self): +# """Keep sunday of the last week.""" +# # 9 backups: 18th (Monday) - 26th (Thursday) March 2024 +# sids = create_SIDs(date(2024, 3, 18), 9, self.cfg) + +# sut = self._org( +# # "Today" is Thursday 28th March +# now=date(2024, 3, 28), +# # Keep the last week +# n_weeks=1, +# snapshots=sids) + +# # only one kept +# self.assertTrue(len(sut), 1) +# # Sunday March 24th +# self.assertTrue(str(sut.pop()).startswith('20240324-')) + +# def test_three_weeks(self): +# """Keep sunday of the last 3 weeks and throw away the rest.""" + +# # 6 Weeks of backups (2024-02-18 - 2024-03-30) +# sids = create_SIDs(datetime(2024, 2, 18), 7*6, self.cfg) +# print(f'{str(sids[0])=} {str(sids[-1])=}') + +# sut = self._org( +# # "Today" is Thursday 28th March +# now=date(2024, 3, 28), +# # Keep the last week +# n_weeks=3, +# snapshots=sids) + +# # only one kept +# self.assertTrue(len(sut), 3) +# sut = sorted(sut) +# for s in sut: +# print(s) + + +# class ForLastNDays(pyfakefs_ut.TestCase): +# """Covering the smart remove setting 'Keep one per day for N days.'. + +# That logic is implemented in 'Snapshots.smartRemoveList()' but not testable +# in isolation. So for a first shot we just duplicate that code in this +# tests (see self._org()). +# """ + +# def setUp(self): +# """Setup a fake filesystem.""" +# self.setUpPyfakefs(allow_root_user=False) + +# # cleanup() happens automatically +# self._temp_dir = TemporaryDirectory(prefix='bit.') +# # Workaround: tempfile and pathlib not compatible yet +# self.temp_path = Path(self._temp_dir.name) + +# self._config_fp = self._create_config_file(parent_path=self.temp_path) +# self.cfg = config.Config(str(self._config_fp)) + +# self.sn = snapshots.Snapshots(self.cfg) + +# def _create_config_file(self, parent_path): +# """Minimal config file""" +# # pylint: disable-next=R0801 +# cfg_content = inspect.cleandoc(''' +# config.version=6 +# profile1.snapshots.include.1.type=0 +# profile1.snapshots.include.1.value=rootpath/source +# profile1.snapshots.include.size=1 +# profile1.snapshots.no_on_battery=false +# profile1.snapshots.notify.enabled=true +# profile1.snapshots.path=rootpath/destination +# profile1.snapshots.path.host=test-host +# profile1.snapshots.path.profile=1 +# profile1.snapshots.path.user=test-user +# profile1.snapshots.preserve_acl=false +# profile1.snapshots.preserve_xattr=false +# profile1.snapshots.remove_old_snapshots.enabled=true +# profile1.snapshots.remove_old_snapshots.unit=80 +# profile1.snapshots.remove_old_snapshots.value=10 +# profile1.snapshots.rsync_options.enabled=false +# profile1.snapshots.rsync_options.value= +# profiles.version=1 +# ''') + +# # config file location +# config_fp = parent_path / 'config_path' / 'config' +# config_fp.parent.mkdir() +# config_fp.write_text(cfg_content, 'utf-8') + +# return config_fp + +# def _org(self, now, n_days, snapshots): +# """Copied and slightly refactored from inside +# 'Snapshots.smartRemoveList()'. +# """ +# print(f'\n_org() :: now={dt2str(now)} {n_days=}') + +# keep = self.sn.smartRemoveKeepAll( +# snapshots, +# now - timedelta(days=n_days-1), +# now + timedelta(days=1)) + +# return keep + +# def test_foobar(self): +# sids = create_SIDs(datetime(2024, 2, 18), 10, self.cfg) +# sut = self._org(now=date(2024, 2, 27), +# n_days=3, +# snapshots=sids) + +# self.assertEqual(len(sut), 3) + +# sut = sorted(sut) + +# self.assertEqual(sut[0].date.date(), date(2024, 2, 25)) +# self.assertEqual(sut[1].date.date(), date(2024, 2, 26)) +# self.assertEqual(sut[2].date.date(), date(2024, 2, 27)) + + +# class OnePerMonth(pyfakefs_ut.TestCase): +# """Covering the smart remove setting 'Keep one snapshot per week for the +# last N weeks'. + +# That logic is implemented in 'Snapshots.smartRemoveList()' but not testable +# in isolation. So for a first shot we just duplicate that code in this +# tests (see self._org()). +# """ + +# def setUp(self): +# """Setup a fake filesystem.""" +# self.setUpPyfakefs(allow_root_user=False) + +# # cleanup() happens automatically +# self._temp_dir = TemporaryDirectory(prefix='bit.') +# # Workaround: tempfile and pathlib not compatible yet +# self.temp_path = Path(self._temp_dir.name) + +# self._config_fp = self._create_config_file(parent_path=self.temp_path) +# self.cfg = config.Config(str(self._config_fp)) + +# self.sn = snapshots.Snapshots(self.cfg) + +# def _create_config_file(self, parent_path): +# """Minimal config file""" +# # pylint: disable-next=R0801 +# cfg_content = inspect.cleandoc(''' +# config.version=6 +# profile1.snapshots.include.1.type=0 +# profile1.snapshots.include.1.value=rootpath/source +# profile1.snapshots.include.size=1 +# profile1.snapshots.no_on_battery=false +# profile1.snapshots.notify.enabled=true +# profile1.snapshots.path=rootpath/destination +# profile1.snapshots.path.host=test-host +# profile1.snapshots.path.profile=1 +# profile1.snapshots.path.user=test-user +# profile1.snapshots.preserve_acl=false +# profile1.snapshots.preserve_xattr=false +# profile1.snapshots.remove_old_snapshots.enabled=true +# profile1.snapshots.remove_old_snapshots.unit=80 +# profile1.snapshots.remove_old_snapshots.value=10 +# profile1.snapshots.rsync_options.enabled=false +# profile1.snapshots.rsync_options.value= +# profiles.version=1 +# ''') + +# # config file location +# config_fp = parent_path / 'config_path' / 'config' +# config_fp.parent.mkdir() +# config_fp.write_text(cfg_content, 'utf-8') + +# return config_fp + +# def _org(self, now, n_months, snapshots, keep_healthy=True): +# """Keep one per months for the last n_months weeks. + +# Copied and slightly refactored from inside +# 'Snapshots.smartRemoveList()'. +# """ +# print(f'\n_org() :: now={dt2str(now)} {n_months=}') +# keep = set() + +# d1 = date(now.year, now.month, 1) +# d2 = self.sn.incMonth(d1) + +# # each months +# for i in range(0, n_months): +# print(f'{i=} {d1=} {d2}') +# keep |= self.sn.smartRemoveKeepFirst( +# snapshots, d1, d2, keep_healthy=keep_healthy) +# d2 = d1 +# d1 = self.sn.decMonth(d1) + +# return keep + +# def test_foobarm(self): +# now = date(2024, 12, 16) +# # sids = create_SIDs(start, 9*7+3, self.cfg) +# sids = create_SIDs(date(2023, 10, 26), 500, self.cfg) + +# months = 3 +# sut = self._org( +# now=now, +# # Keep the last week +# n_months=months, +# snapshots=sids) + +# print(f'\noldest snapshot: {sid2str(sids[0])}') +# for s in sorted(sut): +# print(f'keep: {sid2str(s)}') +# print(f'from/now: {dt2str(now)} {months=}') +# print(f'latest snapshot: {sid2str(sids[-1])}') + + +# class OnePerYear(pyfakefs_ut.TestCase): +# """Covering the smart remove setting 'Keep one snapshot per year for all +# years.' + +# That logic is implemented in 'Snapshots.smartRemoveList()' but not testable +# in isolation. So for a first shot we just duplicate that code in this +# tests (see self._org()). +# """ + +# def setUp(self): +# """Setup a fake filesystem.""" +# self.setUpPyfakefs(allow_root_user=False) + +# # cleanup() happens automatically +# self._temp_dir = TemporaryDirectory(prefix='bit.') +# # Workaround: tempfile and pathlib not compatible yet +# self.temp_path = Path(self._temp_dir.name) + +# self._config_fp = self._create_config_file(parent_path=self.temp_path) +# self.cfg = config.Config(str(self._config_fp)) + +# self.sn = snapshots.Snapshots(self.cfg) + +# def _create_config_file(self, parent_path): +# """Minimal config file""" +# # pylint: disable-next=R0801 +# cfg_content = inspect.cleandoc(''' +# config.version=6 +# profile1.snapshots.include.1.type=0 +# profile1.snapshots.include.1.value=rootpath/source +# profile1.snapshots.include.size=1 +# profile1.snapshots.no_on_battery=false +# profile1.snapshots.notify.enabled=true +# profile1.snapshots.path=rootpath/destination +# profile1.snapshots.path.host=test-host +# profile1.snapshots.path.profile=1 +# profile1.snapshots.path.user=test-user +# profile1.snapshots.preserve_acl=false +# profile1.snapshots.preserve_xattr=false +# profile1.snapshots.remove_old_snapshots.enabled=true +# profile1.snapshots.remove_old_snapshots.unit=80 +# profile1.snapshots.remove_old_snapshots.value=10 +# profile1.snapshots.rsync_options.enabled=false +# profile1.snapshots.rsync_options.value= +# profiles.version=1 +# ''') + +# # config file location +# config_fp = parent_path / 'config_path' / 'config' +# config_fp.parent.mkdir() +# config_fp.write_text(cfg_content, 'utf-8') + +# return config_fp + +# def _org(self, now, snapshots, keep_healthy=True): +# """Keep one per year + +# Copied and slightly refactored from inside +# 'Snapshots.smartRemoveList()'. +# """ +# first_year = int(snapshots[-1].sid[:4]) + +# print(f'\n_org() :: now={dt2str(now)} {first_year=}') +# keep = set() + +# for i in range(first_year, now.year+1): +# keep |= self.sn.smartRemoveKeepFirst( +# snapshots, +# date(i, 1, 1), +# date(i+1, 1, 1), +# keep_healthy=keep_healthy) + +# return keep + +# def test_foobary(self): +# now = date(2024, 12, 16) +# # sids = create_SIDs(start, 9*7+3, self.cfg) +# sids = create_SIDs(date(2019, 10, 26), 365*6, self.cfg) + +# sut = self._org( +# now=now, +# snapshots=sids) + +# print(f'\noldest snapshot: {sid2str(sids[0])}') +# for s in sorted(sut): +# print(f'keep: {sid2str(s)}') +# print(f'from/now: {dt2str(now)}') +# print(f'latest snapshot: {sid2str(sids[-1])}') + + +class IncDecMonths(pyfakefs_ut.TestCase): + """PyFakeFS is used here because of Config file dependency.""" + + def setUp(self): + """Setup a fake filesystem.""" + self.setUpPyfakefs(allow_root_user=False) + + # cleanup() happens automatically + self._temp_dir = TemporaryDirectory(prefix='bit.') + # Workaround: tempfile and pathlib not compatible yet + self.temp_path = Path(self._temp_dir.name) + + self._config_fp = self._create_config_file(parent_path=self.temp_path) + self.cfg = config.Config(str(self._config_fp)) + + self.sn = snapshots.Snapshots(self.cfg) + + def _create_config_file(self, parent_path): + """Minimal config file""" + # pylint: disable-next=R0801 + cfg_content = inspect.cleandoc(''' + config.version=6 + profile1.snapshots.include.1.type=0 + profile1.snapshots.include.1.value=rootpath/source + profile1.snapshots.include.size=1 + profile1.snapshots.no_on_battery=false + profile1.snapshots.notify.enabled=true + profile1.snapshots.path=rootpath/destination + profile1.snapshots.path.host=test-host + profile1.snapshots.path.profile=1 + profile1.snapshots.path.user=test-user + profile1.snapshots.preserve_acl=false + profile1.snapshots.preserve_xattr=false + profile1.snapshots.remove_old_snapshots.enabled=true + profile1.snapshots.remove_old_snapshots.unit=80 + profile1.snapshots.remove_old_snapshots.value=10 + profile1.snapshots.rsync_options.enabled=false + profile1.snapshots.rsync_options.value= + profiles.version=1 + ''') + + # config file location + config_fp = parent_path / 'config_path' / 'config' + config_fp.parent.mkdir() + config_fp.write_text(cfg_content, 'utf-8') + + return config_fp + + def test_inc_simple(self): + sut = self.sn.incMonth(date(1982, 8, 6)) + self.assertEqual(sut, date(1982, 9, 1)) + + def test_inc_next_year(self): + sut = self.sn.incMonth(date(1982, 12, 16)) + self.assertEqual(sut, date(1983, 1, 1)) + + def test_inc_leap_year(self): + sut = self.sn.incMonth(date(2020, 12, 16)) + self.assertEqual(sut, date(2021, 1, 1)) + + def test_inc_leap_months(self): + sut = self.sn.incMonth(date(2020, 2, 29)) + self.assertEqual(sut, date(2020, 3, 1)) + + def test_dec_simple(self): + sut = self.sn.decMonth(date(1982, 8, 6)) + self.assertEqual(sut, date(1982, 7, 1)) + + def test_dec_year(self): + sut = self.sn.decMonth(date(1982, 1, 6)) + self.assertEqual(sut, date(1981, 12, 1)) + + def test_dec_leap_months(self): + sut = self.sn.decMonth(date(2020, 2, 29)) + self.assertEqual(sut, date(2020, 1, 1)) + + +class OldOrg_SmartRemove(generic.SnapshotsTestCase): + """This is the old/original test case using real filesystem and to much + dependencies.""" + + def test_keep_all(self): + sid1 = snapshots.SID('20160424-215134-123', self.cfg) + sid2 = snapshots.SID('20160422-030324-123', self.cfg) + sid3 = snapshots.SID('20160422-020324-123', self.cfg) + sid4 = snapshots.SID('20160422-010324-123', self.cfg) + sid5 = snapshots.SID('20160421-013218-123', self.cfg) + sid6 = snapshots.SID('20160410-134327-123', self.cfg) + sids = [sid1, sid2, sid3, sid4, sid5, sid6] + + keep = self.sn.smartRemoveKeepAll(sids, + date(2016, 4, 20), + date(2016, 4, 23)) + self.assertSetEqual(keep, set((sid2, sid3, sid4, sid5))) + + keep = self.sn.smartRemoveKeepAll(sids, + date(2016, 4, 11), + date(2016, 4, 18)) + self.assertSetEqual(keep, set()) + + def test_smart_remove_list(self): + sid1 = snapshots.SID('20160424-215134-123', self.cfg) + sid2 = snapshots.SID('20160422-030324-123', self.cfg) + sid3 = snapshots.SID('20160422-020324-123', self.cfg) + sid4 = snapshots.SID('20160422-010324-123', self.cfg) + sid5 = snapshots.SID('20160421-033218-123', self.cfg) + sid6 = snapshots.SID('20160421-013218-123', self.cfg) + sid7 = snapshots.SID('20160420-013218-123', self.cfg) + sid8 = snapshots.SID('20160419-013218-123', self.cfg) + sid9 = snapshots.SID('20160419-003218-123', self.cfg) + sid10 = snapshots.SID('20160418-003218-123', self.cfg) + sid11 = snapshots.SID('20160417-033218-123', self.cfg) + sid12 = snapshots.SID('20160417-003218-123', self.cfg) + sid13 = snapshots.SID('20160416-134327-123', self.cfg) + sid14 = snapshots.SID('20160416-114327-123', self.cfg) + sid15 = snapshots.SID('20160415-134327-123', self.cfg) + sid16 = snapshots.SID('20160411-134327-123', self.cfg) + sid17 = snapshots.SID('20160410-134327-123', self.cfg) + sid18 = snapshots.SID('20160409-134327-123', self.cfg) + sid19 = snapshots.SID('20160407-134327-123', self.cfg) + sid20 = snapshots.SID('20160403-134327-123', self.cfg) + sid21 = snapshots.SID('20160402-134327-123', self.cfg) + sid22 = snapshots.SID('20160401-134327-123', self.cfg) + sid23 = snapshots.SID('20160331-134327-123', self.cfg) + sid24 = snapshots.SID('20160330-134327-123', self.cfg) + sid25 = snapshots.SID('20160323-133715-123', self.cfg) + sid26 = snapshots.SID('20160214-134327-123', self.cfg) + sid27 = snapshots.SID('20160205-134327-123', self.cfg) + sid28 = snapshots.SID('20160109-134327-123', self.cfg) + sid29 = snapshots.SID('20151224-134327-123', self.cfg) + sid30 = snapshots.SID('20150904-134327-123', self.cfg) + sid31 = snapshots.SID('20140904-134327-123', self.cfg) + + sids = [ sid1, sid2, sid3, sid4, sid5, sid6, sid7, sid8, sid9, + sid10, sid11, sid12, sid13, sid14, sid15, sid16, sid17, sid18, sid19, + sid20, sid21, sid22, sid23, sid24, sid25, sid26, sid27, sid28, sid29, + sid30, sid31] + for sid in sids: + sid.makeDirs() + now = datetime(2016, 4, 24, 21, 51, 34) + + del_snapshots = self.sn.smartRemoveList(now, + 3, #keep_all + 7, #keep_one_per_day + 5, #keep_one_per_week + 3 #keep_one_per_month + ) + self.assertListEqual(del_snapshots, [sid6, sid9, sid12, sid13, sid14, + sid15, sid16, sid18, sid19, sid21, + sid22, sid24, sid27, sid28, sid30]) + + # test failed snapshots + for sid in (sid5, sid8, sid11, sid12, sid20, sid21, sid22): + sid.failed = True + del_snapshots = self.sn.smartRemoveList(now, + 3, #keep_all + 7, #keep_one_per_day + 5, #keep_one_per_week + 3 #keep_one_per_month + ) + self.assertListEqual(del_snapshots, [sid5, sid8, sid11, sid12, sid14, + sid15, sid16, sid18, sid19, sid20, sid21, + sid22, sid24, sid27, sid28, sid30]) diff --git a/doc/maintain/1_doc_howto.md b/doc/maintain/1_doc_howto.md index 2f6cb229f..b5ee560ac 100644 --- a/doc/maintain/1_doc_howto.md +++ b/doc/maintain/1_doc_howto.md @@ -4,7 +4,7 @@ SPDX-FileCopyrightText: © 2024 Christian Buhtz SPDX-License-Identifier: GPL-2.0-or-later This file is part of the program "Back In Time" which is released under GNU -General Public License v2 (GPLv2). See file/folder LICENSE or go to +General Public License v2 (GPLv2). See LICENSES folder or go to --> diff --git a/doc/maintain/5_auto_smart_remove.md b/doc/maintain/5_auto_smart_remove.md new file mode 100644 index 000000000..1e53d6adf --- /dev/null +++ b/doc/maintain/5_auto_smart_remove.md @@ -0,0 +1,144 @@ + +# Auto- & Smart-Remove +## Table of contents +* [Introduction](#introduction) +* [What we know](#what-we-know) +* [How it could be](#how-it-could-be) + +# Introduction +The actual auto- and smart-remove behavior of BIT will be described in this +document. Don't take this as a regular user manual. The document will help to +decide how that feature can be revised. See +[Meta Issue #1945](https://github.com/bit-team/backintime/issues/1945) about +the background story. + +This is how it looks like currently: +![Aut-remove tab](https://translate.codeberg.org/media/screenshots/bit_manage_profiles_autoremove.gif) + +# What we know +## Location in code +* `common/snapshots.py` + * `Snapshots.freeSpace()` is the main entry for the overall logic. + * `Snapshots.smartRemoveList()` is called by `freeSpace()` and is the entry + for _Smart remove_ related rules. + +## Ordering and interference of the rules +1. Remove snapshots older than N years/weeks/days. +2. Smart-remove rules with calling `Snapshots.smartRemoveList`. + 1. Don't if there is only one backup left. + 2. Always keep the latest/youngest backup. + 3. Keep one per day for N days. + 4. Keep one per week for N weeks. + 5. keep one per month for N months. + 6. Keep one per year for all years. +3. Free space: Remove until there is enough. +4. Free inodes: Remove until there are enough. + +## Details +- In `smartRemoveList()` the direction of ordering of the initial snapshots + list is of high relevance. + +### Older than N years +- Happens in `Snapshots.freeSpace()` +- Relevant also `self.config.removeOldSnapshotsDate()` +- Backups removed immediately before executing any other rule. +- Named snapshots ignored and kept. + +### Smart remove: Daily +GUI wording: _Keep all snapshots for the last `N` day(s)._ + +Current behavior of the algorithm: +* Bug was that in some cases `N-1` days are kept. + * Reason was that not dates but snapshotIDS (included their tags, the last 3 + digits) are used for comparison. + * The bug is fixed. + +### Smart remove: Weekly +GUI wording: _Keep one snapshot per week for the last `N` week(s)._ + +Current behavior of the algorithm: +* A "week" is defined based on the weekdays Monday to Sunday. +* The first week BIT is looking into is the current week even if it is not + completed yet. E.g. today is Wednesday the 27th November, BIT will look + for existing backups starting with Sunday the 24th ending and including the + Saturday 30th November. +* If there is no backup in the current week found, that week is "lost" and + there will only be `N-1` backups in the resulting list of weekly backups. +* See + * [#1094](https://github.com/bit-team/backintime/issues/1094) + * [PR #1944](https://github.com/bit-team/backintime/pull/1944) + * [PR #1819](https://github.com/bit-team/backintime/pull/1819) + + + +### Smart remove: Monthly +- GUI wording: _Keep one snapshot per months for the last `N` month(s)._ +- Seems to use the current month, too. +- Keeps the oldest, so it is the 1th of each months. + +### Smart remove: One per year for all years +- s +### Free space +- Remove until enough free disc space (`self.config.minFreeSpaceMib()`). +- Immediately removed before executing any other rule. + +### Free inodes +- Remove until enough free inodes (`self.config.minFreeInodes()`) +- Immediately removed before executing any other rule. + +# How it could be +## Overview +The following does not reflect the real behavior. It is a draft and suggestion +for the auto-/smart-remove related behavior of BIT and how to implement it. + +## General +- Wording: Remove "Smart" and make everything "Auto remove". +- The rules should to be consistent in their behavior. + - Always keep the latest/newest element in the list (Sunday for weeks, 31th + for months, ...). + - Ignore the current running/incomplete time frame. +- Wording: Use the term "backup" instead of "snapshot". See Issue #1929. + +## Mockup +![Mockup](autoremove_mockup.png) + +[autoremove_mockup.drawio](autoremove_mockup.drawio) + +## Rules in details +For new wording see the mockup. + +1. Remove snapshots older than N years. + - No need for modification. +2. Smart-remove rules with calling `Snapshots.smartRemoveList`. + 1. Don't if there is only one backup left. + - No need for modification. + 2. Always keep the latest/youngest backup. + - No need for modification. + 3. Keep one per day for N days. + - Ignore "today", the current day. + - Keep the latest/newest backup per day. + 4. Keep one per week for N weeks. + - Define "week" as calendar element from Monday to Sunday. + - Ignore the current running and incomplete week. + - Keep the latest/newest backup per week. So it would be Sunday in most + cases if available. + 5. keep one per month for N months. + - Ignore the current running and incomplete month. + - Keep the latest/newset backup per months (30th/31th day of the months). + 6. Keep one per year for all years. + - Use the latest day of year. + - That implicit ignores the current running year. +3. Free space: Remove until there is enough. + - No need for modification. +4. Free inodes: Remove until there are enough. + - No need for modification. + +December 2024 diff --git a/doc/maintain/README.md b/doc/maintain/README.md index 84bcda29c..286d510ca 100644 --- a/doc/maintain/README.md +++ b/doc/maintain/README.md @@ -16,6 +16,7 @@ General Public License v2 (GPLv2). See directory LICENSES or go to - [How to setup openssh for unit tests](3_How_to_set_up_openssh_server_for_ssh_unit_tests.md) - [Usage of control files (locks, flocks, logs and others)](4_Control_files_usage_(locks_flocks_logs_and_others).md) - [How to prepare and publish a new BiT release](BiT_release_process.md) +- [Auto- & Smart-Remove](5_auto_smart_remove.md) Sept 2024 diff --git a/doc/maintain/REUSE.toml b/doc/maintain/_images/REUSE.toml similarity index 96% rename from doc/maintain/REUSE.toml rename to doc/maintain/_images/REUSE.toml index 6339bff8d..e8014c771 100644 --- a/doc/maintain/REUSE.toml +++ b/doc/maintain/_images/REUSE.toml @@ -12,6 +12,6 @@ version = 1 [[annotations]] -path = "_images/**" +path = "*" SPDX-License-Identifier = "GPL-2.0-or-later" SPDX-FileCopyrightText = "© 2024 Back In Time Team" diff --git a/doc/maintain/autoremove_mockup.drawio b/doc/maintain/autoremove_mockup.drawio new file mode 100644 index 000000000..53fc491ae --- /dev/null +++ b/doc/maintain/autoremove_mockup.drawio @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/maintain/autoremove_mockup.png b/doc/maintain/autoremove_mockup.png new file mode 100644 index 000000000..c9f3766cd Binary files /dev/null and b/doc/maintain/autoremove_mockup.png differ