Skip to content

Commit

Permalink
Avoid auth error when serializing files across machines
Browse files Browse the repository at this point in the history
  • Loading branch information
jrbourbeau committed Aug 7, 2023
1 parent 95253c0 commit 4387ab6
Show file tree
Hide file tree
Showing 2 changed files with 7 additions and 4 deletions.
4 changes: 3 additions & 1 deletion earthaccess/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,6 +176,7 @@ def download(
def open(
granules: Union[List[str], List[earthaccess.results.DataGranule]],
provider: Optional[str] = None,
store: Optional[Store] = None,
) -> List[AbstractFileSystem]:
"""Returns a list of fsspec file-like objects that can be used to access files
hosted on S3 or HTTPS by third party libraries like xarray.
Expand All @@ -186,7 +187,8 @@ def open(
Returns:
a list of s3fs "file pointers" to s3 files.
"""
results = earthaccess.__store__.open(granules=granules, provider=provider)
store = store or earthaccess.__store__
results = store.open(granules=granules, provider=provider)
return results


Expand Down
7 changes: 4 additions & 3 deletions earthaccess/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ def __reduce__(self) -> Any:
return make_instance, (
type(self.f),
self.f,
earthaccess.__store__,
self.f.__reduce__(),
)

Expand Down Expand Up @@ -62,12 +63,12 @@ def multi_thread_open(data: tuple) -> EarthAccessFile:
return fileset


def make_instance(cls: Any, granule: DataGranule, _reduce: Any) -> EarthAccessFile:
if earthaccess.__store__.running_in_aws and cls is not s3fs.S3File:
def make_instance(cls: Any, granule: DataGranule, store, _reduce: Any) -> EarthAccessFile:
if store.running_in_aws and cls is not s3fs.S3File:
# On AWS but not using a S3File. Reopen the file in this case for direct S3 access.
# NOTE: This uses the first data_link listed in the granule. That's not
# guaranteed to be the right one.
return EarthAccessFile(earthaccess.open([granule])[0], granule)
return EarthAccessFile(earthaccess.open([granule], store=store)[0], granule)
else:
func = _reduce[0]
args = _reduce[1]
Expand Down

0 comments on commit 4387ab6

Please sign in to comment.