From ebc1e3edb43a74684bdff86a51db13411dd0512d Mon Sep 17 00:00:00 2001 From: Ruslan Kuprieiev Date: Tue, 28 Feb 2023 10:45:15 +0200 Subject: [PATCH] lint: fix formatting --- src/dvc_objects/_tqdm.py | 12 +++------ src/dvc_objects/db.py | 28 ++++++--------------- src/dvc_objects/executors.py | 4 +-- src/dvc_objects/fs/__init__.py | 16 +++--------- src/dvc_objects/fs/base.py | 44 +++++++++------------------------ src/dvc_objects/fs/callbacks.py | 4 +-- src/dvc_objects/fs/generic.py | 8 ++---- src/dvc_objects/fs/local.py | 4 +-- src/dvc_objects/fs/memory.py | 4 +-- src/dvc_objects/fs/path.py | 8 ++---- src/dvc_objects/fs/utils.py | 11 +-------- src/dvc_objects/obj.py | 6 +---- tests/fs/test_callbacks.py | 4 +-- tests/fs/test_localfs.py | 15 +++-------- tests/fs/test_system.py | 8 +----- tests/test_odb.py | 8 ++---- tests/test_reflink.py | 4 +-- 17 files changed, 44 insertions(+), 144 deletions(-) diff --git a/src/dvc_objects/_tqdm.py b/src/dvc_objects/_tqdm.py index ad34a3c..5cf2c35 100644 --- a/src/dvc_objects/_tqdm.py +++ b/src/dvc_objects/_tqdm.py @@ -37,9 +37,7 @@ class Tqdm(tqdm): "{postfix[info]}{n_fmt}/{total_fmt}" " [{elapsed}<{remaining}, {rate_fmt:>11}]" ) - BAR_FMT_NOTOTAL = ( - "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" - ) + BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]" BYTES_DEFAULTS = { "unit": "B", "unit_scale": True, @@ -104,9 +102,7 @@ def __init__( if bar_format is None: if self.__len__(): self.bar_format = ( - self.BAR_FMT_DEFAULT_NESTED - if self.pos - else self.BAR_FMT_DEFAULT + self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT ) else: self.bar_format = self.BAR_FMT_NOTOTAL @@ -148,9 +144,7 @@ def format_dict(self): d = super().format_dict ncols = d["ncols"] or 80 # assumes `bar_format` has max one of ("ncols_desc" & "ncols_info") - ncols_left = ( - ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1 - ) + ncols_left = ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1 ncols_left = max(ncols_left, 0) if ncols_left: d["ncols_desc"] = d["ncols_info"] = ncols_left diff --git a/src/dvc_objects/db.py b/src/dvc_objects/db.py index 19eec37..7932ab8 100644 --- a/src/dvc_objects/db.py +++ b/src/dvc_objects/db.py @@ -203,9 +203,7 @@ def _list_prefixes( jobs: Optional[int] = None, ) -> Iterator[str]: if prefixes: - paths: Union[str, List[str]] = list( - map(self.oid_to_path, prefixes) - ) + paths: Union[str, List[str]] = list(map(self.oid_to_path, prefixes)) if len(paths) == 1: paths = paths[0] prefix = True @@ -236,9 +234,7 @@ def _list_oids( try: yield self.path_to_oid(path) except ValueError: - logger.debug( - "'%s' doesn't look like a cache file, skipping", path - ) + logger.debug("'%s' doesn't look like a cache file, skipping", path) def _oids_with_limit( self, @@ -287,9 +283,7 @@ def iter_with_pbar(oids): yield oid if max_oids: - oids = self._oids_with_limit( - max_oids / total_prefixes, prefixes=[prefix] - ) + oids = self._oids_with_limit(max_oids / total_prefixes, prefixes=[prefix]) else: oids = self._list_oids(prefixes=[prefix]) @@ -403,9 +397,7 @@ def oids_exist(self, oids, jobs=None, progress=noop): always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False) oids = set(oids) - if ( - len(oids) == 1 or not self.fs.CAN_TRAVERSE - ) and not always_traverse: + if (len(oids) == 1 or not self.fs.CAN_TRAVERSE) and not always_traverse: remote_oids = self.list_oids_exists(oids, jobs=jobs) callback = partial(progress, "querying", len(oids)) return list(wrap_iter(remote_oids, callback)) @@ -423,9 +415,7 @@ def oids_exist(self, oids, jobs=None, progress=noop): # From testing with S3, for remotes with 1M+ files, object_exists is # faster until len(oids) is at least 10k~100k if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE: - traverse_weight = ( - traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER - ) + traverse_weight = traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER else: traverse_weight = traverse_pages if len(oids) < traverse_weight and not always_traverse: @@ -439,15 +429,11 @@ def oids_exist(self, oids, jobs=None, progress=noop): ret = list(oids & remote_oids) callback = partial(progress, "querying", len(remaining_oids)) ret.extend( - wrap_iter( - self.list_oids_exists(remaining_oids, jobs=jobs), callback - ) + wrap_iter(self.list_oids_exists(remaining_oids, jobs=jobs), callback) ) return ret logger.debug(f"Querying '{len(oids)}' oids via traverse") - remote_oids = self._list_oids_traverse( - remote_size, remote_oids, jobs=jobs - ) + remote_oids = self._list_oids_traverse(remote_size, remote_oids, jobs=jobs) callback = partial(progress, "querying", remote_size) return list(oids & set(wrap_iter(remote_oids, callback))) diff --git a/src/dvc_objects/executors.py b/src/dvc_objects/executors.py index fb6a9c1..07f2d1f 100644 --- a/src/dvc_objects/executors.py +++ b/src/dvc_objects/executors.py @@ -51,9 +51,7 @@ def create_taskset(n: int) -> Set[futures.Future]: it = zip(*iterables) tasks = create_taskset(self.max_workers * 5) while tasks: - done, tasks = futures.wait( - tasks, return_when=futures.FIRST_COMPLETED - ) + done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED) for fut in done: yield fut.result() tasks.update(create_taskset(len(done))) diff --git a/src/dvc_objects/fs/__init__.py b/src/dvc_objects/fs/__init__.py index 9bef9d2..dcdc899 100644 --- a/src/dvc_objects/fs/__init__.py +++ b/src/dvc_objects/fs/__init__.py @@ -22,9 +22,7 @@ }, Schemes.GDRIVE: { "class": "dvc_gdrive.GDriveFileSystem", - "err": ( - "gdrive is supported, but requires 'dvc-gdrive' to be installed" - ), + "err": ("gdrive is supported, but requires 'dvc-gdrive' to be installed"), }, Schemes.GS: { "class": "dvc_gs.GSFileSystem", @@ -56,21 +54,15 @@ }, Schemes.WEBDAV: { "class": "dvc_webdav.WebDAVFileSystem", - "err": ( - "webdav is supported, but requires 'dvc-webdav' to be installed" - ), + "err": ("webdav is supported, but requires 'dvc-webdav' to be installed"), }, Schemes.WEBDAVS: { "class": "dvc_webdav.WebDAVSFileSystem", - "err": ( - "webdavs is supported, but requires 'dvc-webdav' to be installed" - ), + "err": ("webdavs is supported, but requires 'dvc-webdav' to be installed"), }, Schemes.WEBHDFS: { "class": "dvc_webhdfs.WebHDFSFileSystem", - "err": ( - "webhdfs is supported, but requires 'dvc-webhdfs' to be installed" - ), + "err": ("webhdfs is supported, but requires 'dvc-webhdfs' to be installed"), }, } diff --git a/src/dvc_objects/fs/base.py b/src/dvc_objects/fs/base.py index 484118b..49f2f98 100644 --- a/src/dvc_objects/fs/base.py +++ b/src/dvc_objects/fs/base.py @@ -209,9 +209,7 @@ def cat( on_error: Literal["raise", "omit", "return"] = "raise", **kwargs: Any, ) -> Union[bytes, Dict[AnyFSPath, bytes]]: - return self.fs.cat( - path, recursive=recursive, on_error=on_error, **kwargs - ) + return self.fs.cat(path, recursive=recursive, on_error=on_error, **kwargs) def cat_ranges( self, @@ -221,9 +219,7 @@ def cat_ranges( max_gap: int = None, **kwargs, ) -> List[bytes]: - return self.fs.cat_ranges( - paths, starts, ends, max_gap=max_gap, **kwargs - ) + return self.fs.cat_ranges(paths, starts, ends, max_gap=max_gap, **kwargs) def cat_file( self, @@ -284,9 +280,7 @@ def pipe( ) -> None: return self.fs.pipe(path, value=value, **kwargs) - def touch( - self, path: AnyFSPath, truncate: bool = True, **kwargs: Any - ) -> None: + def touch(self, path: AnyFSPath, truncate: bool = True, **kwargs: Any) -> None: return self.fs.touch(path, truncate=truncate, **kwargs) def checksum(self, path: AnyFSPath) -> str: @@ -296,9 +290,7 @@ def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None: self.makedirs(self.path.parent(to_info)) self.fs.copy(from_info, to_info) - def cp_file( - self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any - ) -> None: + def cp_file(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None: self.fs.cp_file(from_info, to_info, **kwargs) @overload @@ -388,15 +380,11 @@ def iscopy(self, path: AnyFSPath) -> bool: return not (self.is_symlink(path) or self.is_hardlink(path)) @overload - def ls( - self, path: AnyFSPath, detail: Literal[True], **kwargs - ) -> "Iterator[Entry]": + def ls(self, path: AnyFSPath, detail: Literal[True], **kwargs) -> "Iterator[Entry]": ... @overload - def ls( - self, path: AnyFSPath, detail: Literal[False], **kwargs - ) -> Iterator[str]: + def ls(self, path: AnyFSPath, detail: Literal[False], **kwargs) -> Iterator[str]: ... def ls(self, path, detail=False, **kwargs): @@ -434,9 +422,7 @@ def find( for result in executor.imap_unordered(find, path): yield from result - def mv( - self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any - ) -> None: + def mv(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None: self.fs.mv(from_info, to_info) move = mv @@ -486,9 +472,7 @@ def info(self, path, callback=DEFAULT_CALLBACK, batch_size=None, **kwargs): fut = asyncio.run_coroutine_threadsafe( batch_coros( [ - self.fs._info( # pylint: disable=protected-access - p, **kwargs - ) + self.fs._info(p, **kwargs) # pylint: disable=protected-access for p in path ], batch_size=jobs, @@ -525,9 +509,7 @@ def put_file( self.upload_fobj(stream, to_info, size=size) else: assert isinstance(from_file, str) - self.fs.put_file( - os.fspath(from_file), to_info, callback=callback, **kwargs - ) + self.fs.put_file(os.fspath(from_file), to_info, callback=callback, **kwargs) self.fs.invalidate_cache(self.path.parent(to_info)) def get_file( @@ -631,9 +613,7 @@ def get_file(rpath, lpath, **kwargs): return localfs.makedirs(to_info, exist_ok=True) to_infos = [ - localfs.path.join( - to_info, *self.path.relparts(info, from_info) - ) + localfs.path.join(to_info, *self.path.relparts(info, from_info)) for info in from_infos ] @@ -660,9 +640,7 @@ def created(self, path: AnyFSPath) -> datetime.datetime: def modified(self, path: AnyFSPath) -> datetime.datetime: return self.fs.modified(path) - def sign( - self, path: AnyFSPath, expiration: int = 100, **kwargs: Any - ) -> str: + def sign(self, path: AnyFSPath, expiration: int = 100, **kwargs: Any) -> str: return self.fs.sign(path, expiration=expiration, **kwargs) diff --git a/src/dvc_objects/fs/callbacks.py b/src/dvc_objects/fs/callbacks.py index ea1f2de..a38b7dc 100644 --- a/src/dvc_objects/fs/callbacks.py +++ b/src/dvc_objects/fs/callbacks.py @@ -103,9 +103,7 @@ def absolute_update(self, value: int) -> None: return super().absolute_update(value) @classmethod - def as_callback( - cls, maybe_callback: Optional["Callback"] = None - ) -> "Callback": + def as_callback(cls, maybe_callback: Optional["Callback"] = None) -> "Callback": if maybe_callback is None: return DEFAULT_CALLBACK return maybe_callback diff --git a/src/dvc_objects/fs/generic.py b/src/dvc_objects/fs/generic.py index 9d749b3..f0dc2f9 100644 --- a/src/dvc_objects/fs/generic.py +++ b/src/dvc_objects/fs/generic.py @@ -20,9 +20,7 @@ logger = logging.getLogger(__name__) -TransferErrorHandler = Callable[ - ["AnyFSPath", "AnyFSPath", BaseException], None -] +TransferErrorHandler = Callable[["AnyFSPath", "AnyFSPath", BaseException], None] def log_exceptions(func: Callable) -> Callable: @@ -264,9 +262,7 @@ def _try_links( del links[0] - raise OSError( - errno.ENOTSUP, "no more link types left to try out" - ) from error + raise OSError(errno.ENOTSUP, "no more link types left to try out") from error def transfer( diff --git a/src/dvc_objects/fs/local.py b/src/dvc_objects/fs/local.py index 2c4b884..ff8c4be 100644 --- a/src/dvc_objects/fs/local.py +++ b/src/dvc_objects/fs/local.py @@ -61,9 +61,7 @@ def walk(self, path, maxdepth=None, topdown=True, detail=False, **kwargs): topdown=topdown, ): if detail: - dirs_dict = { - name: self.info(os.path.join(root, name)) for name in dirs - } + dirs_dict = {name: self.info(os.path.join(root, name)) for name in dirs} files_dict = { name: self.info(os.path.join(root, name)) for name in files } diff --git a/src/dvc_objects/fs/memory.py b/src/dvc_objects/fs/memory.py index c04b526..05ff98e 100644 --- a/src/dvc_objects/fs/memory.py +++ b/src/dvc_objects/fs/memory.py @@ -16,8 +16,6 @@ def __init__(self, global_store=True, fs=None, **kwargs): self.fs.pseudo_dirs = [""] def __eq__(self, other): - return ( - isinstance(other, type(self)) and self.fs.store is other.fs.store - ) + return isinstance(other, type(self)) and self.fs.store is other.fs.store __hash__ = FileSystem.__hash__ diff --git a/src/dvc_objects/fs/path.py b/src/dvc_objects/fs/path.py index d0a8d6e..93bbf31 100644 --- a/src/dvc_objects/fs/path.py +++ b/src/dvc_objects/fs/path.py @@ -133,13 +133,9 @@ def overlaps(self, left: str, right: str) -> bool: def relpath(self, path: str, start: Optional[str] = None) -> str: if start is None: start = "." - return self.flavour.relpath( - self.abspath(path), start=self.abspath(start) - ) + return self.flavour.relpath(self.abspath(path), start=self.abspath(start)) - def relparts( - self, path: str, start: Optional[str] = None - ) -> Tuple[str, ...]: + def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]: return self.parts(self.relpath(path, start=start)) def as_posix(self, path: str) -> str: diff --git a/src/dvc_objects/fs/utils.py b/src/dvc_objects/fs/utils.py index 2105ed9..cda06a9 100644 --- a/src/dvc_objects/fs/utils.py +++ b/src/dvc_objects/fs/utils.py @@ -7,16 +7,7 @@ import threading from concurrent import futures from contextlib import contextmanager, suppress -from typing import ( - TYPE_CHECKING, - Any, - Collection, - Dict, - Iterator, - Optional, - Set, - Union, -) +from typing import TYPE_CHECKING, Any, Collection, Dict, Iterator, Optional, Set, Union from ..executors import ThreadPoolExecutor from . import system diff --git a/src/dvc_objects/obj.py b/src/dvc_objects/obj.py index 98c47df..53f2ad4 100644 --- a/src/dvc_objects/obj.py +++ b/src/dvc_objects/obj.py @@ -29,11 +29,7 @@ def __bool__(self): def __eq__(self, other): if not isinstance(other, type(self)): return False - return ( - self.path == other.path - and self.fs == other.fs - and self.oid == other.oid - ) + return self.path == other.path and self.fs == other.fs and self.oid == other.oid def __hash__(self): return hash( diff --git a/tests/fs/test_callbacks.py b/tests/fs/test_callbacks.py index 3bac41a..89a8e6d 100644 --- a/tests/fs/test_callbacks.py +++ b/tests/fs/test_callbacks.py @@ -3,9 +3,7 @@ from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback -@pytest.mark.parametrize( - "api", ["set_size", "relative_update", "absolute_update"] -) +@pytest.mark.parametrize("api", ["set_size", "relative_update", "absolute_update"]) @pytest.mark.parametrize( "callback_factory, kwargs", [ diff --git a/tests/fs/test_localfs.py b/tests/fs/test_localfs.py index 2f11f71..ae9cfd4 100644 --- a/tests/fs/test_localfs.py +++ b/tests/fs/test_localfs.py @@ -6,9 +6,7 @@ from dvc_objects.fs.local import LocalFileSystem -@pytest.mark.parametrize( - "path, contents", [("file", "foo"), ("тест", "проверка")] -) +@pytest.mark.parametrize("path, contents", [("file", "foo"), ("тест", "проверка")]) def test_local_fs_open(tmp_path, path, contents): file = tmp_path / path file.write_text(contents, encoding="utf8") @@ -61,9 +59,7 @@ def test_local_fs_rm(tmp_path): def convert_to_sets(walk_results): - return [ - (root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results - ] + return [(root, set(dirs), set(nondirs)) for root, dirs, nondirs in walk_results] @pytest.fixture @@ -74,16 +70,13 @@ def dir_path(tmp_path): ("тест", "проверка"), ( "code.py", - "import sys\nimport shutil\n" - "shutil.copyfile(sys.argv[1], sys.argv[2])", + "import sys\nimport shutil\n" "shutil.copyfile(sys.argv[1], sys.argv[2])", ), ]: (tmp_path / file).write_text(contents, encoding="utf8") (tmp_path / "data" / "sub").mkdir(parents=True) (tmp_path / "data" / "file").write_text("file", encoding="utf8") - (tmp_path / "data" / "sub" / "file").write_text( - "sub_file", encoding="utf8" - ) + (tmp_path / "data" / "sub" / "file").write_text("sub_file", encoding="utf8") return tmp_path diff --git a/tests/fs/test_system.py b/tests/fs/test_system.py index 8b8bd0f..299d93a 100644 --- a/tests/fs/test_system.py +++ b/tests/fs/test_system.py @@ -1,13 +1,7 @@ import os from os import fspath -from dvc_objects.fs.system import ( - hardlink, - inode, - is_hardlink, - is_symlink, - symlink, -) +from dvc_objects.fs.system import hardlink, inode, is_hardlink, is_symlink, symlink def test_inode(tmp_path): diff --git a/tests/test_odb.py b/tests/test_odb.py index 7541b24..e4527ef 100644 --- a/tests/test_odb.py +++ b/tests/test_odb.py @@ -206,9 +206,7 @@ def test_oids_exist_less_oids_larger_fs(mocker): odb._max_estimation_size(oids) / pow(16, odb.fs.TRAVERSE_PREFIX_LEN) ) assert max_oids < 2048 - object_exists.assert_called_with( - frozenset(range(max_oids, 1000)), jobs=None - ) + object_exists.assert_called_with(frozenset(range(max_oids, 1000)), jobs=None) traverse.assert_not_called() @@ -264,9 +262,7 @@ def test_list_oids_traverse(mocker, prefix_len, extra_prefixes): odb = ObjectDB(FileSystem(), "/odb") list_oids = mocker.patch.object(odb, "_list_oids", return_value=[]) - mocker.patch.object( - odb, "path_to_oid", side_effect=lambda x: x - ) # pragma: no cover + mocker.patch.object(odb, "path_to_oid", side_effect=lambda x: x) # pragma: no cover mocker.patch.object(odb.fs, "TRAVERSE_PREFIX_LEN", prefix_len) # parallel traverse diff --git a/tests/test_reflink.py b/tests/test_reflink.py index 91377a6..b8d736f 100644 --- a/tests/test_reflink.py +++ b/tests/test_reflink.py @@ -33,9 +33,7 @@ def test_reflink(test_dir): assert dest.is_file() assert dest.read_bytes() == b"content" - stat_mode = src.stat().st_mode & ( - stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO - ) + stat_mode = src.stat().st_mode & (stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) assert stat_mode == (0o666 & ~umask(0))