From 68e19a29f3d74458e968e7e15afb0e8430c8c8ab Mon Sep 17 00:00:00 2001 From: Ian Carroll Date: Mon, 23 Sep 2024 17:11:57 -0400 Subject: [PATCH] Fix `earthaccess.EarthAccessFile` method lookup (#620) --- CHANGELOG.md | 3 ++ docs/user-reference/store/earthaccessfile.md | 7 +++ earthaccess/api.py | 8 ++-- earthaccess/store.py | 45 +++++++++++--------- mkdocs.yml | 1 + tests/unit/test_store.py | 10 +++++ 6 files changed, 49 insertions(+), 25 deletions(-) create mode 100644 docs/user-reference/store/earthaccessfile.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 02b1c6b6..5d768ae8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,9 @@ - Removed Broken Link "Introduction to NASA earthaccess" ([#779](https://github.com/nsidc/earthaccess/issues/779)) ([**@Sherwin-14**](https://github.com/Sherwin-14)) +* Remove the base class on `EarthAccessFile` to fix method resolution + ([#610](https://github.com/nsidc/earthaccess/issues/610)) + ([**@itcarroll**](https://github.com/itcarroll)) ### Removed diff --git a/docs/user-reference/store/earthaccessfile.md b/docs/user-reference/store/earthaccessfile.md new file mode 100644 index 00000000..31b19c93 --- /dev/null +++ b/docs/user-reference/store/earthaccessfile.md @@ -0,0 +1,7 @@ +# Documentation for `EarthAccessFile` + +::: earthaccess.store.EarthAccessFile + options: + inherited_members: true + show_root_heading: true + show_source: false diff --git a/earthaccess/api.py b/earthaccess/api.py index 91a9f55b..4b4b1698 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -11,7 +11,7 @@ from .auth import Auth from .results import DataCollection, DataGranule from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery -from .store import Store +from .store import EarthAccessFile, Store from .system import PROD, System from .utils import _validation as validate @@ -242,8 +242,8 @@ def download( def open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, -) -> List[AbstractFileSystem]: - """Returns a list of fsspec file-like objects that can be used to access files +) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: @@ -252,7 +252,7 @@ def open( provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - a list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ provider = _normalize_location(provider) results = earthaccess.__store__.open(granules=granules, provider=provider) diff --git a/earthaccess/store.py b/earthaccess/store.py index a97f8188..61437542 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -26,8 +26,22 @@ logger = logging.getLogger(__name__) -class EarthAccessFile(fsspec.spec.AbstractBufferedFile): - def __init__(self, f: fsspec.AbstractFileSystem, granule: DataGranule) -> None: +class EarthAccessFile: + """Handle for a file-like object pointing to an on-prem or Earthdata Cloud granule.""" + + def __init__( + self, f: fsspec.spec.AbstractBufferedFile, granule: DataGranule + ) -> None: + """EarthAccessFile connects an Earthdata search result with an open file-like object. + + No methods exist on the class, which passes all attribute and method calls + directly to the file-like object given during initialization. An instance of + this class can be treated like that file-like object itself. + + Parameters: + f: a file-like object + granule: a granule search result + """ self.f = f self.granule = granule @@ -43,14 +57,14 @@ def __reduce__(self) -> Any: ) def __repr__(self) -> str: - return str(self.f) + return repr(self.f) def _open_files( url_mapping: Mapping[str, Union[DataGranule, None]], fs: fsspec.AbstractFileSystem, threads: Optional[int] = 8, -) -> List[fsspec.AbstractFileSystem]: +) -> List[EarthAccessFile]: def multi_thread_open(data: tuple) -> EarthAccessFile: urls, granule = data return EarthAccessFile(fs.open(urls), granule) @@ -322,17 +336,17 @@ def open( self, granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, - ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files + ) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option + granules: a list of granule instances **or** list of URLs, e.g. `s3://some-granule`. + If a list of URLs is passed, we need to specify the data provider. + provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - A list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ if len(granules): return self._open(granules, provider) @@ -344,17 +358,6 @@ def _open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files - hosted on S3 or HTTPS by third party libraries like xarray. - - Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option - - Returns: - A list of s3fs "file pointers" to s3 files. - """ raise NotImplementedError("granules should be a list of DataGranule or URLs") @_open.register diff --git a/mkdocs.yml b/mkdocs.yml index 2dd3e761..8a62d86d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -100,6 +100,7 @@ nav: - "Granule Queries": "user-reference/granules/granules-query.md" - "Granule Results": "user-reference/granules/granules.md" - Store: + - "EarthAccessFile": "user-reference/store/earthaccessfile.md" - "Store": "user-reference/store/store.md" - Auth: - "Auth": "user-reference/auth/auth.md" diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index 1397358c..9b12c267 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -7,6 +7,7 @@ import responses import s3fs from earthaccess import Auth, Store +from earthaccess.store import EarthAccessFile class TestStoreSessions(unittest.TestCase): @@ -126,3 +127,12 @@ def test_store_can_create_s3_fsspec_session(self): store.get_s3_filesystem() return None + + +def test_earthaccess_file_getattr(): + fs = fsspec.filesystem("memory") + with fs.open("/foo", "wb") as f: + earthaccess_file = EarthAccessFile(f, granule="foo") + assert f.tell() == earthaccess_file.tell() + # cleanup + fs.store.clear()