Skip to content

Commit

Permalink
lint: fix formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
efiop committed Feb 28, 2023
1 parent 9163781 commit ebc1e3e
Show file tree
Hide file tree
Showing 17 changed files with 44 additions and 144 deletions.
12 changes: 3 additions & 9 deletions src/dvc_objects/_tqdm.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,7 @@ class Tqdm(tqdm):
"{postfix[info]}{n_fmt}/{total_fmt}"
" [{elapsed}<{remaining}, {rate_fmt:>11}]"
)
BAR_FMT_NOTOTAL = (
"{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
)
BAR_FMT_NOTOTAL = "{desc}{bar:b}|{postfix[info]}{n_fmt} [{elapsed}, {rate_fmt:>11}]"
BYTES_DEFAULTS = {
"unit": "B",
"unit_scale": True,
Expand Down Expand Up @@ -104,9 +102,7 @@ def __init__(
if bar_format is None:
if self.__len__():
self.bar_format = (
self.BAR_FMT_DEFAULT_NESTED
if self.pos
else self.BAR_FMT_DEFAULT
self.BAR_FMT_DEFAULT_NESTED if self.pos else self.BAR_FMT_DEFAULT
)
else:
self.bar_format = self.BAR_FMT_NOTOTAL
Expand Down Expand Up @@ -148,9 +144,7 @@ def format_dict(self):
d = super().format_dict
ncols = d["ncols"] or 80
# assumes `bar_format` has max one of ("ncols_desc" & "ncols_info")
ncols_left = (
ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
)
ncols_left = ncols - len(self.format_meter(ncols_desc=1, ncols_info=1, **d)) + 1
ncols_left = max(ncols_left, 0)
if ncols_left:
d["ncols_desc"] = d["ncols_info"] = ncols_left
Expand Down
28 changes: 7 additions & 21 deletions src/dvc_objects/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,9 +203,7 @@ def _list_prefixes(
jobs: Optional[int] = None,
) -> Iterator[str]:
if prefixes:
paths: Union[str, List[str]] = list(
map(self.oid_to_path, prefixes)
)
paths: Union[str, List[str]] = list(map(self.oid_to_path, prefixes))
if len(paths) == 1:
paths = paths[0]
prefix = True
Expand Down Expand Up @@ -236,9 +234,7 @@ def _list_oids(
try:
yield self.path_to_oid(path)
except ValueError:
logger.debug(
"'%s' doesn't look like a cache file, skipping", path
)
logger.debug("'%s' doesn't look like a cache file, skipping", path)

def _oids_with_limit(
self,
Expand Down Expand Up @@ -287,9 +283,7 @@ def iter_with_pbar(oids):
yield oid

if max_oids:
oids = self._oids_with_limit(
max_oids / total_prefixes, prefixes=[prefix]
)
oids = self._oids_with_limit(max_oids / total_prefixes, prefixes=[prefix])
else:
oids = self._list_oids(prefixes=[prefix])

Expand Down Expand Up @@ -403,9 +397,7 @@ def oids_exist(self, oids, jobs=None, progress=noop):
always_traverse = getattr(self.fs, "_ALWAYS_TRAVERSE", False)

oids = set(oids)
if (
len(oids) == 1 or not self.fs.CAN_TRAVERSE
) and not always_traverse:
if (len(oids) == 1 or not self.fs.CAN_TRAVERSE) and not always_traverse:
remote_oids = self.list_oids_exists(oids, jobs=jobs)
callback = partial(progress, "querying", len(oids))
return list(wrap_iter(remote_oids, callback))
Expand All @@ -423,9 +415,7 @@ def oids_exist(self, oids, jobs=None, progress=noop):
# From testing with S3, for remotes with 1M+ files, object_exists is
# faster until len(oids) is at least 10k~100k
if remote_size > self.fs.TRAVERSE_THRESHOLD_SIZE:
traverse_weight = (
traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
)
traverse_weight = traverse_pages * self.fs.TRAVERSE_WEIGHT_MULTIPLIER
else:
traverse_weight = traverse_pages
if len(oids) < traverse_weight and not always_traverse:
Expand All @@ -439,15 +429,11 @@ def oids_exist(self, oids, jobs=None, progress=noop):
ret = list(oids & remote_oids)
callback = partial(progress, "querying", len(remaining_oids))
ret.extend(
wrap_iter(
self.list_oids_exists(remaining_oids, jobs=jobs), callback
)
wrap_iter(self.list_oids_exists(remaining_oids, jobs=jobs), callback)
)
return ret

logger.debug(f"Querying '{len(oids)}' oids via traverse")
remote_oids = self._list_oids_traverse(
remote_size, remote_oids, jobs=jobs
)
remote_oids = self._list_oids_traverse(remote_size, remote_oids, jobs=jobs)
callback = partial(progress, "querying", remote_size)
return list(oids & set(wrap_iter(remote_oids, callback)))
4 changes: 1 addition & 3 deletions src/dvc_objects/executors.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,9 +51,7 @@ def create_taskset(n: int) -> Set[futures.Future]:
it = zip(*iterables)
tasks = create_taskset(self.max_workers * 5)
while tasks:
done, tasks = futures.wait(
tasks, return_when=futures.FIRST_COMPLETED
)
done, tasks = futures.wait(tasks, return_when=futures.FIRST_COMPLETED)
for fut in done:
yield fut.result()
tasks.update(create_taskset(len(done)))
Expand Down
16 changes: 4 additions & 12 deletions src/dvc_objects/fs/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,7 @@
},
Schemes.GDRIVE: {
"class": "dvc_gdrive.GDriveFileSystem",
"err": (
"gdrive is supported, but requires 'dvc-gdrive' to be installed"
),
"err": ("gdrive is supported, but requires 'dvc-gdrive' to be installed"),
},
Schemes.GS: {
"class": "dvc_gs.GSFileSystem",
Expand Down Expand Up @@ -56,21 +54,15 @@
},
Schemes.WEBDAV: {
"class": "dvc_webdav.WebDAVFileSystem",
"err": (
"webdav is supported, but requires 'dvc-webdav' to be installed"
),
"err": ("webdav is supported, but requires 'dvc-webdav' to be installed"),
},
Schemes.WEBDAVS: {
"class": "dvc_webdav.WebDAVSFileSystem",
"err": (
"webdavs is supported, but requires 'dvc-webdav' to be installed"
),
"err": ("webdavs is supported, but requires 'dvc-webdav' to be installed"),
},
Schemes.WEBHDFS: {
"class": "dvc_webhdfs.WebHDFSFileSystem",
"err": (
"webhdfs is supported, but requires 'dvc-webhdfs' to be installed"
),
"err": ("webhdfs is supported, but requires 'dvc-webhdfs' to be installed"),
},
}

Expand Down
44 changes: 11 additions & 33 deletions src/dvc_objects/fs/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,9 +209,7 @@ def cat(
on_error: Literal["raise", "omit", "return"] = "raise",
**kwargs: Any,
) -> Union[bytes, Dict[AnyFSPath, bytes]]:
return self.fs.cat(
path, recursive=recursive, on_error=on_error, **kwargs
)
return self.fs.cat(path, recursive=recursive, on_error=on_error, **kwargs)

def cat_ranges(
self,
Expand All @@ -221,9 +219,7 @@ def cat_ranges(
max_gap: int = None,
**kwargs,
) -> List[bytes]:
return self.fs.cat_ranges(
paths, starts, ends, max_gap=max_gap, **kwargs
)
return self.fs.cat_ranges(paths, starts, ends, max_gap=max_gap, **kwargs)

def cat_file(
self,
Expand Down Expand Up @@ -284,9 +280,7 @@ def pipe(
) -> None:
return self.fs.pipe(path, value=value, **kwargs)

def touch(
self, path: AnyFSPath, truncate: bool = True, **kwargs: Any
) -> None:
def touch(self, path: AnyFSPath, truncate: bool = True, **kwargs: Any) -> None:
return self.fs.touch(path, truncate=truncate, **kwargs)

def checksum(self, path: AnyFSPath) -> str:
Expand All @@ -296,9 +290,7 @@ def copy(self, from_info: AnyFSPath, to_info: AnyFSPath) -> None:
self.makedirs(self.path.parent(to_info))
self.fs.copy(from_info, to_info)

def cp_file(
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
) -> None:
def cp_file(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None:
self.fs.cp_file(from_info, to_info, **kwargs)

@overload
Expand Down Expand Up @@ -388,15 +380,11 @@ def iscopy(self, path: AnyFSPath) -> bool:
return not (self.is_symlink(path) or self.is_hardlink(path))

@overload
def ls(
self, path: AnyFSPath, detail: Literal[True], **kwargs
) -> "Iterator[Entry]":
def ls(self, path: AnyFSPath, detail: Literal[True], **kwargs) -> "Iterator[Entry]":
...

@overload
def ls(
self, path: AnyFSPath, detail: Literal[False], **kwargs
) -> Iterator[str]:
def ls(self, path: AnyFSPath, detail: Literal[False], **kwargs) -> Iterator[str]:
...

def ls(self, path, detail=False, **kwargs):
Expand Down Expand Up @@ -434,9 +422,7 @@ def find(
for result in executor.imap_unordered(find, path):
yield from result

def mv(
self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any
) -> None:
def mv(self, from_info: AnyFSPath, to_info: AnyFSPath, **kwargs: Any) -> None:
self.fs.mv(from_info, to_info)

move = mv
Expand Down Expand Up @@ -486,9 +472,7 @@ def info(self, path, callback=DEFAULT_CALLBACK, batch_size=None, **kwargs):
fut = asyncio.run_coroutine_threadsafe(
batch_coros(
[
self.fs._info( # pylint: disable=protected-access
p, **kwargs
)
self.fs._info(p, **kwargs) # pylint: disable=protected-access
for p in path
],
batch_size=jobs,
Expand Down Expand Up @@ -525,9 +509,7 @@ def put_file(
self.upload_fobj(stream, to_info, size=size)
else:
assert isinstance(from_file, str)
self.fs.put_file(
os.fspath(from_file), to_info, callback=callback, **kwargs
)
self.fs.put_file(os.fspath(from_file), to_info, callback=callback, **kwargs)
self.fs.invalidate_cache(self.path.parent(to_info))

def get_file(
Expand Down Expand Up @@ -631,9 +613,7 @@ def get_file(rpath, lpath, **kwargs):
return localfs.makedirs(to_info, exist_ok=True)

to_infos = [
localfs.path.join(
to_info, *self.path.relparts(info, from_info)
)
localfs.path.join(to_info, *self.path.relparts(info, from_info))
for info in from_infos
]

Expand All @@ -660,9 +640,7 @@ def created(self, path: AnyFSPath) -> datetime.datetime:
def modified(self, path: AnyFSPath) -> datetime.datetime:
return self.fs.modified(path)

def sign(
self, path: AnyFSPath, expiration: int = 100, **kwargs: Any
) -> str:
def sign(self, path: AnyFSPath, expiration: int = 100, **kwargs: Any) -> str:
return self.fs.sign(path, expiration=expiration, **kwargs)


Expand Down
4 changes: 1 addition & 3 deletions src/dvc_objects/fs/callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -103,9 +103,7 @@ def absolute_update(self, value: int) -> None:
return super().absolute_update(value)

@classmethod
def as_callback(
cls, maybe_callback: Optional["Callback"] = None
) -> "Callback":
def as_callback(cls, maybe_callback: Optional["Callback"] = None) -> "Callback":
if maybe_callback is None:
return DEFAULT_CALLBACK
return maybe_callback
Expand Down
8 changes: 2 additions & 6 deletions src/dvc_objects/fs/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,7 @@
logger = logging.getLogger(__name__)


TransferErrorHandler = Callable[
["AnyFSPath", "AnyFSPath", BaseException], None
]
TransferErrorHandler = Callable[["AnyFSPath", "AnyFSPath", BaseException], None]


def log_exceptions(func: Callable) -> Callable:
Expand Down Expand Up @@ -264,9 +262,7 @@ def _try_links(

del links[0]

raise OSError(
errno.ENOTSUP, "no more link types left to try out"
) from error
raise OSError(errno.ENOTSUP, "no more link types left to try out") from error


def transfer(
Expand Down
4 changes: 1 addition & 3 deletions src/dvc_objects/fs/local.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,7 @@ def walk(self, path, maxdepth=None, topdown=True, detail=False, **kwargs):
topdown=topdown,
):
if detail:
dirs_dict = {
name: self.info(os.path.join(root, name)) for name in dirs
}
dirs_dict = {name: self.info(os.path.join(root, name)) for name in dirs}
files_dict = {
name: self.info(os.path.join(root, name)) for name in files
}
Expand Down
4 changes: 1 addition & 3 deletions src/dvc_objects/fs/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,6 @@ def __init__(self, global_store=True, fs=None, **kwargs):
self.fs.pseudo_dirs = [""]

def __eq__(self, other):
return (
isinstance(other, type(self)) and self.fs.store is other.fs.store
)
return isinstance(other, type(self)) and self.fs.store is other.fs.store

__hash__ = FileSystem.__hash__
8 changes: 2 additions & 6 deletions src/dvc_objects/fs/path.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,13 +133,9 @@ def overlaps(self, left: str, right: str) -> bool:
def relpath(self, path: str, start: Optional[str] = None) -> str:
if start is None:
start = "."
return self.flavour.relpath(
self.abspath(path), start=self.abspath(start)
)
return self.flavour.relpath(self.abspath(path), start=self.abspath(start))

def relparts(
self, path: str, start: Optional[str] = None
) -> Tuple[str, ...]:
def relparts(self, path: str, start: Optional[str] = None) -> Tuple[str, ...]:
return self.parts(self.relpath(path, start=start))

def as_posix(self, path: str) -> str:
Expand Down
11 changes: 1 addition & 10 deletions src/dvc_objects/fs/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,16 +7,7 @@
import threading
from concurrent import futures
from contextlib import contextmanager, suppress
from typing import (
TYPE_CHECKING,
Any,
Collection,
Dict,
Iterator,
Optional,
Set,
Union,
)
from typing import TYPE_CHECKING, Any, Collection, Dict, Iterator, Optional, Set, Union

from ..executors import ThreadPoolExecutor
from . import system
Expand Down
6 changes: 1 addition & 5 deletions src/dvc_objects/obj.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,7 @@ def __bool__(self):
def __eq__(self, other):
if not isinstance(other, type(self)):
return False
return (
self.path == other.path
and self.fs == other.fs
and self.oid == other.oid
)
return self.path == other.path and self.fs == other.fs and self.oid == other.oid

def __hash__(self):
return hash(
Expand Down
4 changes: 1 addition & 3 deletions tests/fs/test_callbacks.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,7 @@
from dvc_objects.fs.callbacks import DEFAULT_CALLBACK, Callback


@pytest.mark.parametrize(
"api", ["set_size", "relative_update", "absolute_update"]
)
@pytest.mark.parametrize("api", ["set_size", "relative_update", "absolute_update"])
@pytest.mark.parametrize(
"callback_factory, kwargs",
[
Expand Down
Loading

0 comments on commit ebc1e3e

Please sign in to comment.