diff --git a/CHANGELOG.md b/CHANGELOG.md index 007996ee..0a397ba3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,8 +31,6 @@ [**@Sherwin-14**](https://github.com/betolink), [**@mfisher87**](https://github.com/mfisher87) - - ### Fixed - Removed Broken Link "Introduction to NASA earthaccess" @@ -41,6 +39,16 @@ - Restore automation for tidying notebooks used in documentation ([#788](https://github.com/nsidc/earthaccess/issues/788)) ([**@itcarroll**](https://github.com/itcarroll)) +- Remove the base class on `EarthAccessFile` to fix method resolution + ([#610](https://github.com/nsidc/earthaccess/issues/610)) + ([**@itcarroll**](https://github.com/itcarroll)) + +### Removed + +- Remove `binder/` directory, as we no longer need a special [binder](https://mybinder.org) + environment with the top-level `environment.yml` introduced in + [#733](https://github.com/nsidc/earthaccess/issues/733) + ([@jhkennedy](https://github.com/jhkennedy)) ## [0.10.0] 2024-07-19 diff --git a/binder/environment.yml b/binder/environment.yml deleted file mode 100644 index 95c0496b..00000000 --- a/binder/environment.yml +++ /dev/null @@ -1,24 +0,0 @@ -name: earthaccess -channels: - - conda-forge -dependencies: - - python=3.9 - - xarray>=0.19 - - dask>=2022.1 - - rioxarray>=0.3 - - matplotlib-base>=3.3 - - cartopy>=0.18.0 - - ipyleaflet>=0.15 - - netcdf4>=1.5 - - h5netcdf>=0.11 - - h5py>=3.2 - - geopandas>=0.9 - - zarr>=2.9.5 - - jupyterlab>=3 - - hvplot - - holoviews - - panel - - - pip - - pip: - - "." diff --git a/docs/user-reference/store/earthaccessfile.md b/docs/user-reference/store/earthaccessfile.md new file mode 100644 index 00000000..31b19c93 --- /dev/null +++ b/docs/user-reference/store/earthaccessfile.md @@ -0,0 +1,7 @@ +# Documentation for `EarthAccessFile` + +::: earthaccess.store.EarthAccessFile + options: + inherited_members: true + show_root_heading: true + show_source: false diff --git a/earthaccess/api.py b/earthaccess/api.py index 91a9f55b..4b4b1698 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -11,7 +11,7 @@ from .auth import Auth from .results import DataCollection, DataGranule from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery -from .store import Store +from .store import EarthAccessFile, Store from .system import PROD, System from .utils import _validation as validate @@ -242,8 +242,8 @@ def download( def open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, -) -> List[AbstractFileSystem]: - """Returns a list of fsspec file-like objects that can be used to access files +) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: @@ -252,7 +252,7 @@ def open( provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - a list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ provider = _normalize_location(provider) results = earthaccess.__store__.open(granules=granules, provider=provider) diff --git a/earthaccess/store.py b/earthaccess/store.py index a97f8188..61437542 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -26,8 +26,22 @@ logger = logging.getLogger(__name__) -class EarthAccessFile(fsspec.spec.AbstractBufferedFile): - def __init__(self, f: fsspec.AbstractFileSystem, granule: DataGranule) -> None: +class EarthAccessFile: + """Handle for a file-like object pointing to an on-prem or Earthdata Cloud granule.""" + + def __init__( + self, f: fsspec.spec.AbstractBufferedFile, granule: DataGranule + ) -> None: + """EarthAccessFile connects an Earthdata search result with an open file-like object. + + No methods exist on the class, which passes all attribute and method calls + directly to the file-like object given during initialization. An instance of + this class can be treated like that file-like object itself. + + Parameters: + f: a file-like object + granule: a granule search result + """ self.f = f self.granule = granule @@ -43,14 +57,14 @@ def __reduce__(self) -> Any: ) def __repr__(self) -> str: - return str(self.f) + return repr(self.f) def _open_files( url_mapping: Mapping[str, Union[DataGranule, None]], fs: fsspec.AbstractFileSystem, threads: Optional[int] = 8, -) -> List[fsspec.AbstractFileSystem]: +) -> List[EarthAccessFile]: def multi_thread_open(data: tuple) -> EarthAccessFile: urls, granule = data return EarthAccessFile(fs.open(urls), granule) @@ -322,17 +336,17 @@ def open( self, granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, - ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files + ) -> List[EarthAccessFile]: + """Returns a list of file-like objects that can be used to access files hosted on S3 or HTTPS by third party libraries like xarray. Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option + granules: a list of granule instances **or** list of URLs, e.g. `s3://some-granule`. + If a list of URLs is passed, we need to specify the data provider. + provider: e.g. POCLOUD, NSIDC_CPRD, etc. Returns: - A list of s3fs "file pointers" to s3 files. + A list of "file pointers" to remote (i.e. s3 or https) files. """ if len(granules): return self._open(granules, provider) @@ -344,17 +358,6 @@ def _open( granules: Union[List[str], List[DataGranule]], provider: Optional[str] = None, ) -> List[Any]: - """Returns a list of fsspec file-like objects that can be used to access files - hosted on S3 or HTTPS by third party libraries like xarray. - - Parameters: - granules: a list of granules(DataGranule) instances or list of URLs, - e.g. s3://some-granule - provider: an option - - Returns: - A list of s3fs "file pointers" to s3 files. - """ raise NotImplementedError("granules should be a list of DataGranule or URLs") @_open.register diff --git a/environment.yml b/environment.yml index e0eb49b3..fa8a3362 100644 --- a/environment.yml +++ b/environment.yml @@ -1,14 +1,13 @@ -name: earthaccess-dev +name: earthaccess channels: - conda-forge dependencies: # This environment bootstraps pip, the actual dev environment # is installed and managed with pip - - python=3.10 - + - python - pip - pip: - - --editable .[dev,test,docs] + - --editable ".[dev,test,docs]" variables: # Allow pip installs when conda environment is active PIP_REQUIRE_VENV: 0 diff --git a/mkdocs.yml b/mkdocs.yml index 2dd3e761..8a62d86d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -100,6 +100,7 @@ nav: - "Granule Queries": "user-reference/granules/granules-query.md" - "Granule Results": "user-reference/granules/granules.md" - Store: + - "EarthAccessFile": "user-reference/store/earthaccessfile.md" - "Store": "user-reference/store/store.md" - Auth: - "Auth": "user-reference/auth/auth.md" diff --git a/tests/unit/test_store.py b/tests/unit/test_store.py index 1397358c..9b12c267 100644 --- a/tests/unit/test_store.py +++ b/tests/unit/test_store.py @@ -7,6 +7,7 @@ import responses import s3fs from earthaccess import Auth, Store +from earthaccess.store import EarthAccessFile class TestStoreSessions(unittest.TestCase): @@ -126,3 +127,12 @@ def test_store_can_create_s3_fsspec_session(self): store.get_s3_filesystem() return None + + +def test_earthaccess_file_getattr(): + fs = fsspec.filesystem("memory") + with fs.open("/foo", "wb") as f: + earthaccess_file = EarthAccessFile(f, granule="foo") + assert f.tell() == earthaccess_file.tell() + # cleanup + fs.store.clear()