From a11cd04fd80561f029826b6fb27d8d8a5ed43935 Mon Sep 17 00:00:00 2001 From: adrianloy Date: Mon, 7 Oct 2024 12:14:10 +0200 Subject: [PATCH] Enable fsspec file caching (#215) * Pass details=False to ls to support filecache/simplecache * Bump bug version --- pyproject.toml | 2 +- squirrel/iterstream/source.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 87a9c17..3618e0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "squirrel-core" -version = "0.20.1" +version = "0.20.2" description = "Squirrel is a Python library that enables ML teams to share, load, and transform data in a collaborative, flexible and efficient way." authors = ["Merantix Momentum"] license = "Apache 2.0" diff --git a/squirrel/iterstream/source.py b/squirrel/iterstream/source.py index 7d73e0d..ea39cbe 100644 --- a/squirrel/iterstream/source.py +++ b/squirrel/iterstream/source.py @@ -79,7 +79,7 @@ def __init__( def __iter__(self) -> t.Iterator[str]: """Iterator that does ls and yield filepaths under the given url""" self.fs = get_fs_from_url(self.url, **self.storage_options) - urls = self.fs.ls(self.url) if self.fs.exists(self.url) else [] + urls = self.fs.ls(self.url, detail=False) if self.fs.exists(self.url) else [] urls.sort() if self.nested: dirs = []