diff --git a/Makefile b/Makefile index 17b4679..685de6e 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,26 @@ -.PHONY: test +.PHONY: test coverage dist lint fmt + test: # TODO: figure out why -s option is required for fabric remote connection hatch run test -s -vv -.PHONY: coverage +coverage: hatch run cov -s -vv -.PHONY: dist +lint: + hatch run lint:all + +fmt: + hatch run lint:fmt + +# dist: +# # todo move all this to script in pyproject.toml +# python3 -m pip install --upgrade build +# python3 -m build +# python3 -m pip install --upgrade twine +# echo "CHECK THIS VERSION NUMBER AND CHANGE IF NECESSARY!" +# cat ./src/pisync/__about__.py +# python3 -m twine upload dist/* + dist: - # todo move all this to script in pyproject.toml - python3 -m pip install --upgrade build - python3 -m build - python3 -m pip install --upgrade twine - echo "CHECK THIS VERSION NUMBER AND CHANGE IF NECESSARY!" - cat ./src/pisync/__about__.py - python3 -m twine upload dist/* + hatch publish diff --git a/pyproject.toml b/pyproject.toml index 9cf784c..3e3d0bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,6 +60,8 @@ dependencies = [ "black>=23.1.0", "mypy>=1.0.0", "ruff>=0.0.243", + "pytest", + "pisync" ] [tool.hatch.envs.lint.scripts] typing = "mypy --install-types --non-interactive {args:src/pisync tests}" @@ -77,6 +79,12 @@ all = [ "typing", ] +[[tool.mypy.overrides]] +module = [ + "fabric" +] +ignore_missing_imports = true + [tool.black] target-version = ["py37"] line-length = 120 @@ -121,6 +129,8 @@ ignore = [ "S105", "S106", "S107", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # I do not know how to fix this `subprocess` call: check for execution of untrusted input + "S603" ] unfixable = [ # Don't touch unused imports @@ -137,6 +147,9 @@ ban-relative-imports = "all" # Tests can use magic values, assertions, and relative imports "tests/**/*" = ["PLR2004", "S101", "TID252"] +# allow print statement in example script +"./examples/run_backups.py" = ["T201"] + [tool.coverage.run] source_pkgs = ["pisync", "tests"] branch = true diff --git a/src/pisync/__init__.py b/src/pisync/__init__.py index 576343f..422e8e8 100644 --- a/src/pisync/__init__.py +++ b/src/pisync/__init__.py @@ -1,2 +1,4 @@ from pisync.backup import backup from pisync.config import LocalConfig, RemoteConfig + +__all__ = ("backup", "LocalConfig", "RemoteConfig") diff --git a/src/pisync/backup.py b/src/pisync/backup.py index aac9140..61f3aa6 100644 --- a/src/pisync/backup.py +++ b/src/pisync/backup.py @@ -16,10 +16,10 @@ from pathlib import Path from typing import List -from pisync.config.base_config import _BaseConfig +from pisync.config.base_config import BackupType, BaseConfig -def backup(config: _BaseConfig) -> str: +def backup(config: BaseConfig) -> str: """ Returns the path to the latest backup directory """ @@ -44,12 +44,14 @@ def backup(config: _BaseConfig) -> str: raise Exception(msg) if prev_backup_exists: + backup_method = BackupType.Incremental logging.info(f"Starting incremental backup from {config.resolve(config.link_dir)}") else: + backup_method = BackupType.Complete logging.info(f"No previous backup found at {config.destination_dir}") - logging.info(f"Starting a fresh complete backup from {config.source_dir} " "to {config.destination_dir}") + logging.info(f"Starting a fresh complete backup from {config.source_dir} to {config.destination_dir}") - rsync_command = config.get_rsync_command(latest_backup_path, previous_backup_exists=prev_backup_exists) + rsync_command = config.get_rsync_command(latest_backup_path, backup_method=backup_method) exit_code = run_rsync(rsync_command) @@ -61,21 +63,22 @@ def backup(config: _BaseConfig) -> str: logging.info(f"Symlink created from {latest_backup_path} to {config.link_dir}") return latest_backup_path else: + msg = f"Backup failed. Rsync exit code: {exit_code}" + logging.error(msg) # backup failed, we should delete the most recent backup - logging.error(f"Backup failed. Rsync exit code: {exit_code}") if config.file_exists(latest_backup_path): logging.info(f"Deleting failed backup at {latest_backup_path}") shutil.rmtree(latest_backup_path) - return None + raise Exception(msg) def configure_logging(filename: str): - filename = Path(filename) - if not filename.parent.exists(): - filename.parent.mkdir(parents=True) + _filename = Path(filename) + if not _filename.parent.exists(): + _filename.parent.mkdir(parents=True) logging.basicConfig( - filename=str(filename.resolve()), + filename=str(_filename.resolve()), filemode="a", format="%(levelname)s\t%(asctime)s\t%(message)s", datefmt="%m/%d/%Y %I:%M:%S %p", diff --git a/src/pisync/config/__init__.py b/src/pisync/config/__init__.py index 3efc9f2..9c858d3 100644 --- a/src/pisync/config/__init__.py +++ b/src/pisync/config/__init__.py @@ -1,3 +1,5 @@ -from pisync.config.base_config import InvalidPath +from pisync.config.base_config import BackupType, InvalidPathError from pisync.config.local_config import LocalConfig from pisync.config.remote_config import RemoteConfig + +__all__ = ("InvalidPathError", "BackupType", "LocalConfig", "RemoteConfig") diff --git a/src/pisync/config/base_config.py b/src/pisync/config/base_config.py index 1250d3a..59e505a 100644 --- a/src/pisync/config/base_config.py +++ b/src/pisync/config/base_config.py @@ -1,15 +1,21 @@ from abc import ABC, abstractmethod -from typing import List +from enum import Enum +from typing import List, Optional -class InvalidPath(Exception): +class InvalidPathError(Exception): pass -class _BaseConfig(ABC): +class BackupType(Enum): + Complete = 1 + Incremental = 2 + + +class BaseConfig(ABC): source_dir: str destination_dir: str - exclude_file_patterns: List[str] + exclude_file_patterns: Optional[List[str]] log_file: str link_dir: str @@ -44,11 +50,11 @@ def resolve(self, path: str) -> str: pass @abstractmethod - def ensure_dir_exists(self, path: str): + def ensure_dir_exists(self, path: str) -> None: """ :returns: The Path object from the input path str :raises: - InvalidPath: If path does not exist or is not a directory + InvalidPathError: If path does not exist or is not a directory """ pass @@ -58,10 +64,10 @@ def generate_new_backup_dir_path(self) -> str: :returns: The Path string of the directory where the new backup will be written. :raises: - InvalidPath: If the destination directory already exists + InvalidPathError: If the destination directory already exists """ pass @abstractmethod - def get_rsync_command(self, new_backup_dir: str, previous_backup_exists: bool = False): + def get_rsync_command(self, new_backup_dir: str, backup_method: BackupType): pass diff --git a/src/pisync/config/local_config.py b/src/pisync/config/local_config.py index 55b4ae1..d2113bc 100644 --- a/src/pisync/config/local_config.py +++ b/src/pisync/config/local_config.py @@ -1,24 +1,27 @@ from pathlib import Path from typing import List, Optional -from pisync.config.base_config import InvalidPath, _BaseConfig +from pisync.config.base_config import BackupType, BaseConfig, InvalidPathError from pisync.util import get_time_stamp -class LocalConfig(_BaseConfig): +class LocalConfig(BaseConfig): def __init__( self, source_dir: str, destination_dir: str, exclude_file_patterns: Optional[List[str]] = None, - log_file: str = str(Path.home() / ".local/share/backup/rsync-backups.log"), + log_file: Optional[str] = None, ): self.ensure_dir_exists(source_dir) self.ensure_dir_exists(destination_dir) self.source_dir = source_dir self.destination_dir = destination_dir self.exclude_file_patterns = exclude_file_patterns - self.log_file = log_file + if log_file is None: + self.log_file = str(Path.home() / ".local/share/backup/rsync-backups.log") + else: + self.log_file = log_file self.link_dir = str(Path(self.destination_dir) / "latest") self._optionless_rsync_arguments = [ "--delete", # delete extraneous files from dest dirs @@ -46,30 +49,30 @@ def is_empty_directory(self, path: str) -> bool: return not any(Path(path).iterdir()) def ensure_dir_exists(self, path: str): - path = Path(path) - if not path.exists(): - msg = f"{path} does not exist" - raise InvalidPath(msg) - if not path.is_dir(): - msg = f"{path} is not a directory" - raise InvalidPath(msg) + _path = Path(path) + if not _path.exists(): + msg = f"{_path} does not exist" + raise InvalidPathError(msg) + if not _path.is_dir(): + msg = f"{_path} is not a directory" + raise InvalidPathError(msg) def generate_new_backup_dir_path(self) -> str: time_stamp = get_time_stamp() new_backup_dir = Path(self.destination_dir) / time_stamp if new_backup_dir.exists(): msg = f"{new_backup_dir} already exists and will get overwritten" - raise InvalidPath(msg) + raise InvalidPathError(msg) else: return str(new_backup_dir) - def get_rsync_command(self, new_backup_dir: str, previous_backup_exists: bool = False) -> List[str]: + def get_rsync_command(self, new_backup_dir: str, backup_method: BackupType) -> List[str]: destination = new_backup_dir source = self.source_dir link_dest = self.link_dir option_arguments = [] - if previous_backup_exists: + if backup_method == BackupType.Incremental: option_arguments.append(f"--link-dest={link_dest}") if self.exclude_file_patterns is not None: diff --git a/src/pisync/config/remote_config.py b/src/pisync/config/remote_config.py index 42ec14e..f7641ce 100644 --- a/src/pisync/config/remote_config.py +++ b/src/pisync/config/remote_config.py @@ -3,18 +3,18 @@ from fabric import Connection -from pisync.config.base_config import InvalidPath, _BaseConfig +from pisync.config.base_config import BackupType, BaseConfig, InvalidPathError from pisync.util import get_time_stamp -class RemoteConfig(_BaseConfig): +class RemoteConfig(BaseConfig): def __init__( self, user_at_hostname: str, source_dir: str, destination_dir: str, exclude_file_patterns: Optional[List[str]] = None, - log_file: str = Path.home() / ".local/share/backup/rsync-backups.log", + log_file: Optional[str] = None, ): self.user_at_hostname = user_at_hostname self.connection: Connection = Connection(user_at_hostname) @@ -23,7 +23,10 @@ def __init__( self.source_dir = source_dir self.destination_dir = destination_dir self.exclude_file_patterns = exclude_file_patterns - self.log_file = log_file + if log_file is None: + self.log_file = str(Path.home() / ".local/share/backup/rsync-backups.log") + else: + self.log_file = log_file self.link_dir = f"{self.destination_dir}/latest" self._optionless_rsync_arguments = [ "--delete", # delete extraneous files from dest dirs @@ -36,13 +39,13 @@ def __init__( ] def _ensure_dir_exists_locally(self, path: str): - path = Path(path) - if not path.exists(): - msg = f"{path} does not exist" - raise InvalidPath(msg) - if not path.is_dir(): - msg = f"{path} is not a directory" - raise InvalidPath(msg) + _path = Path(path) + if not _path.exists(): + msg = f"{_path} does not exist" + raise InvalidPathError(msg) + if not _path.is_dir(): + msg = f"{_path} is not a directory" + raise InvalidPathError(msg) def is_symlink(self, path: str) -> bool: """returns true if path is a symbolic link""" @@ -77,13 +80,13 @@ def resolve(self, path: str) -> str: result = self.connection.run(f"realpath {path}", warn=True) return result.stdout.strip() - def ensure_dir_exists(self, path: str) -> str: + def ensure_dir_exists(self, path: str) -> None: result = self.connection.run(f"test -d {path}", warn=True) if not result.ok: msg = f"{path} is not a directory" - raise InvalidPath(msg) + raise InvalidPathError(msg) - def _is_directory(self, path) -> bool: + def _is_directory(self, path) -> BackupType: result = self.connection.run(f"test -d {path}", warn=True) return result.ok @@ -92,24 +95,24 @@ def generate_new_backup_dir_path(self) -> str: :returns: The Path string of the directory where the new backup will be written. :raises: - InvalidPath: If the destination directory already exists + InvalidPathError: If the destination directory already exists """ time_stamp = get_time_stamp() new_backup_dir = f"{self.destination_dir}/{time_stamp}" exists = self._is_directory(new_backup_dir) or self.file_exists(new_backup_dir) if exists: msg = f"{new_backup_dir} already exists and will get overwritten" - raise InvalidPath(msg) + raise InvalidPathError(msg) else: return str(new_backup_dir) - def get_rsync_command(self, new_backup_dir: str, previous_backup_exists: bool = False) -> List[str]: + def get_rsync_command(self, new_backup_dir: str, backup_method: BackupType) -> List[str]: destination = f"{self.user_at_hostname}:{new_backup_dir}" source = self.source_dir link_dest = self.link_dir option_arguments = [] - if previous_backup_exists: + if backup_method == BackupType.Incremental: option_arguments.append(f"--link-dest={link_dest}") if self.exclude_file_patterns is not None: diff --git a/src/pisync/util.py b/src/pisync/util.py index 82bd0f9..c68e6c1 100644 --- a/src/pisync/util.py +++ b/src/pisync/util.py @@ -2,6 +2,6 @@ def get_time_stamp() -> str: - now = datetime.now() + now = datetime.now().astimezone() stamp = now.strftime("%Y-%m-%d-%H-%M-%S") return str(stamp) diff --git a/tests/test_local_config.py b/tests/test_local_config.py index 11efc98..e0e7c2b 100644 --- a/tests/test_local_config.py +++ b/tests/test_local_config.py @@ -3,7 +3,7 @@ import pytest -from pisync.config import InvalidPath, LocalConfig +from pisync.config import BackupType, InvalidPathError, LocalConfig from pisync.util import get_time_stamp @@ -13,8 +13,8 @@ def home() -> str: @pytest.fixture -def tmp() -> str: - return "/tmp" +def tmp(tmp_path) -> str: + return tmp_path @pytest.fixture @@ -26,21 +26,21 @@ class TestInitConfig: def test_valid_params_no_exceptions(self, home_tmp_config, optionless_arguments): home, tmp, config = home_tmp_config assert str(config.source_dir) == home - assert str(config.destination_dir) == tmp + assert str(config.destination_dir) == str(tmp) assert str(config.link_dir) == f"{tmp}/latest" assert config.exclude_file_patterns is None assert config._optionless_rsync_arguments == optionless_arguments - def test_source_does_not_exist_throws_invalid_path(self): + def test_source_does_not_exist_throws_invalid_path(self, tmp): source = "/bad/directory/path/here/does/not/exist" - dest = "/tmp" - with pytest.raises(InvalidPath): + dest = tmp + with pytest.raises(InvalidPathError): _ = LocalConfig(source, dest) def test_destination_does_not_exist_throws_invalid_path(self): source = str(Path("~/").expanduser()) dest = "/bad/directory/path/here/does/not/exist" - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): _ = LocalConfig(source, dest) @@ -48,29 +48,29 @@ class TestGetRsyncCommand: def test_no_previous_backup(self, home_tmp_config, optionless_arguments): home, tmp, config = home_tmp_config new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=False) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Complete) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == ["rsync", *optionless_arguments, home, new_backup_dir] def test_previous_backup_exists(self, home_tmp_config, optionless_arguments): home, tmp, config = home_tmp_config new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=True) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Incremental) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == ["rsync", *optionless_arguments, f"--link-dest={tmp}/latest", home, new_backup_dir] def test_exclude_patterns(self, home, tmp, optionless_arguments): exclude_file_patterns = ["/exclude/path1", "/exclude/path2", "/exclude/path3/**/*.bak"] config = LocalConfig(home, tmp, exclude_file_patterns) new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=False) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Complete) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == [ "rsync", *optionless_arguments, @@ -152,11 +152,11 @@ def test_ensure_dir_exists(self, scratch_file_system, home_tmp_config): # not a dir assert (fs / "file1").exists() - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): config.ensure_dir_exists(fs / "file1") # non existent - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): config.ensure_dir_exists(fs / "dir") @pytest.mark.skip(reason="Time difference between the two statements sometimes causes fail") @@ -176,5 +176,5 @@ def test_generate_new_backup_dir_throws_exception_on_overwrite( time_stamp = get_time_stamp() (tmp_path / time_stamp).touch() - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): _ = config.generate_new_backup_dir_path() diff --git a/tests/test_remote_config.py b/tests/test_remote_config.py index 0670168..9cf4dcd 100644 --- a/tests/test_remote_config.py +++ b/tests/test_remote_config.py @@ -4,7 +4,7 @@ import pytest -from pisync.config import InvalidPath, RemoteConfig +from pisync.config import BackupType, InvalidPathError, RemoteConfig from pisync.util import get_time_stamp @@ -14,8 +14,8 @@ def home() -> str: @pytest.fixture -def tmp() -> str: - return "/tmp" +def tmp(tmp_path) -> str: + return tmp_path @pytest.fixture @@ -32,21 +32,21 @@ class TestInitConfig: def test_valid_params_no_exceptions(self, home_tmp_config, optionless_arguments): home, tmp, config = home_tmp_config assert str(config.source_dir) == home - assert str(config.destination_dir) == tmp + assert str(config.destination_dir) == str(tmp) assert str(config.link_dir) == f"{tmp}/latest" assert config.exclude_file_patterns is None assert config._optionless_rsync_arguments == optionless_arguments - def test_source_does_not_exist_throws_invalid_path(self, user_at_localhost): + def test_source_does_not_exist_throws_invalid_path(self, user_at_localhost, tmp): source = "/bad/directory/path/here/does/not/exist" - dest = "/tmp" - with pytest.raises(InvalidPath): + dest = tmp + with pytest.raises(InvalidPathError): _ = RemoteConfig(user_at_localhost, source, dest) def test_destination_does_not_exist_throws_invalid_path(self, user_at_localhost): source = str(Path("~/").expanduser()) dest = "/bad/directory/path/here/does/not/exist" - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): _ = RemoteConfig(user_at_localhost, source, dest) @@ -54,19 +54,19 @@ class TestGetRsyncCommand: def test_no_previous_backup(self, home_tmp_config, optionless_arguments, user_at_localhost): home, tmp, config = home_tmp_config new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=False) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Complete) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == ["rsync", *optionless_arguments, home, f"{user_at_localhost}:{new_backup_dir}"] def test_previous_backup_exists(self, home_tmp_config, optionless_arguments, user_at_localhost): home, tmp, config = home_tmp_config new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=True) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Incremental) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == [ "rsync", *optionless_arguments, @@ -79,10 +79,10 @@ def test_exclude_patterns(self, home, tmp, optionless_arguments, user_at_localho exclude_file_patterns = ["/exclude/path1", "/exclude/path2", "/exclude/path3/**/*.bak"] config = RemoteConfig(user_at_localhost, home, tmp, exclude_file_patterns) new_backup_dir = config.generate_new_backup_dir_path() - rsync_cmd = config.get_rsync_command(new_backup_dir, previous_backup_exists=False) + rsync_cmd = config.get_rsync_command(new_backup_dir, backup_method=BackupType.Complete) # For example: /tmp/2023-07-14-17-24-23 - assert new_backup_dir.startswith("/tmp/") + assert new_backup_dir.startswith(str(tmp.parts[0])) assert rsync_cmd == [ "rsync", *optionless_arguments, @@ -164,11 +164,11 @@ def test_ensure_dir_exists(self, scratch_file_system, home_tmp_config): # not a dir assert (fs / "file1").exists() - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): config.ensure_dir_exists(fs / "file1") # non existent - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): config.ensure_dir_exists(fs / "dir") @pytest.mark.skip(reason="Time difference between the two statements sometimes causes fail") @@ -186,5 +186,5 @@ def test_generate_new_backup_dir_throws_exception_on_overwrite( time_stamp = get_time_stamp() (tmp_path / time_stamp).touch() - with pytest.raises(InvalidPath): + with pytest.raises(InvalidPathError): _ = config.generate_new_backup_dir_path()