From 73b9af688acffca24581984c8d5613a171b5fb88 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Thu, 14 Sep 2023 16:58:44 -0500 Subject: [PATCH 1/6] Support automatic authentication --- earthaccess/__init__.py | 30 +++++++++++++++++++++++++++--- earthaccess/auth.py | 33 ++++++++++++++++++--------------- 2 files changed, 45 insertions(+), 18 deletions(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index ef4fdfb8..ee3b79cb 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -1,3 +1,4 @@ +import threading from importlib.metadata import version from typing import Any @@ -36,7 +37,30 @@ "Store", ] -__auth__ = Auth() -__store__: Any = None - __version__ = version("earthaccess") + +_auth = Auth() +_store = None +_lock = threading.Lock() + + +def __getattr__(name): + global _auth, _store + + if name == "__auth__" or name == "__store__": + with _lock: + if not _auth.authenticated: + for strategy in ["environment", "netrc"]: + try: + _auth.login(strategy=strategy) + except Exception: + continue + else: + if not _auth.authenticated: + continue + else: + _store = Store(_auth) + break + return _auth if name == "__auth__" else _store + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/earthaccess/auth.py b/earthaccess/auth.py index cf0cd971..b202e0eb 100644 --- a/earthaccess/auth.py +++ b/earthaccess/auth.py @@ -1,4 +1,5 @@ import getpass +import logging import os from netrc import NetrcParseError from pathlib import Path @@ -10,6 +11,8 @@ from .daac import DAACS +logger = logging.getLogger(__name__) + class SessionWithHeaderRedirection(requests.Session): """ @@ -76,7 +79,7 @@ def login(self, strategy: str = "netrc", persist: bool = False) -> Any: an instance of Auth. """ if self.authenticated: - print("We are already authenticated with NASA EDL") + logger.debug("We are already authenticated with NASA EDL") return self if strategy == "interactive": self._interactive(persist) @@ -98,7 +101,7 @@ def refresh_tokens(self) -> bool: if resp_tokens.ok: self.token = resp_tokens.json() self.tokens = [self.token] - print( + logger.debug( f"earthaccess generated a token for CMR with expiration on: {self.token['expiration_date']}" ) return True @@ -111,7 +114,7 @@ def refresh_tokens(self) -> bool: if resp_tokens.ok: self.token = resp_tokens.json() self.tokens.extend(self.token) - print( + logger.debug( f"earthaccess generated a token for CMR with expiration on: {self.token['expiration_date']}" ) return True @@ -127,7 +130,7 @@ def refresh_tokens(self) -> bool: if resp_tokens.ok: self.token = resp_tokens.json() self.tokens[0] = self.token - print( + logger.debug( f"earthaccess generated a token for CMR with expiration on: {self.token['expiration_date']}" ) return True @@ -221,7 +224,7 @@ def _interactive(self, presist_credentials: bool = False) -> bool: password = getpass.getpass(prompt="Enter your Earthdata password: ") authenticated = self._get_credentials(username, password) if authenticated: - print("Using user provided credentials for EDL") + logger.debug("Using user provided credentials for EDL") if presist_credentials: print("Persisting credentials to .netrc") self._persist_user_credentials(username, password) @@ -231,11 +234,11 @@ def _netrc(self) -> bool: try: my_netrc = Netrc() except FileNotFoundError as err: - print(f"No .netrc found in {os.path.expanduser('~')}") - raise err + raise FileNotFoundError( + f"No .netrc found in {os.path.expanduser('~')}" + ) from err except NetrcParseError as err: - print("Unable to parse .netrc") - raise err + raise NetrcParseError("Unable to parse .netrc") from err if my_netrc["urs.earthdata.nasa.gov"] is not None: username = my_netrc["urs.earthdata.nasa.gov"]["login"] password = my_netrc["urs.earthdata.nasa.gov"]["password"] @@ -243,7 +246,7 @@ def _netrc(self) -> bool: return False authenticated = self._get_credentials(username, password) if authenticated: - print("Using .netrc file for EDL") + logger.debug("Using .netrc file for EDL") return authenticated def _environment(self) -> bool: @@ -251,9 +254,9 @@ def _environment(self) -> bool: password = os.getenv("EARTHDATA_PASSWORD") authenticated = self._get_credentials(username, password) if authenticated: - print("Using environment variables for EDL") + logger.debug("Using environment variables for EDL") else: - print( + logger.debug( "EARTHDATA_USERNAME and EARTHDATA_PASSWORD are not set in the current environment, try " "setting them or use a different strategy (netrc, interactive)" ) @@ -270,7 +273,7 @@ def _get_credentials( f"Authentication with Earthdata Login failed with:\n{token_resp.text}" ) return False - print("You're now authenticated with NASA Earthdata Login") + logger.debug("You're now authenticated with NASA Earthdata Login") self.username = username self.password = password @@ -279,13 +282,13 @@ def _get_credentials( if len(self.tokens) == 0: self.refresh_tokens() - print( + logger.debug( f"earthaccess generated a token for CMR with expiration on: {self.token['expiration_date']}" ) self.token = self.tokens[0] elif len(self.tokens) > 0: self.token = self.tokens[0] - print( + logger.debug( f"Using token with expiration date: {self.token['expiration_date']}" ) profile = self.get_user_profile() From 641eafcf1ae7807cd15323502eb7dab47133ee11 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 18 Sep 2023 13:55:41 -0500 Subject: [PATCH 2/6] Lint --- earthaccess/__init__.py | 2 +- earthaccess/api.py | 3 +-- earthaccess/store.py | 7 +++---- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index ee3b79cb..94d90546 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -44,7 +44,7 @@ _lock = threading.Lock() -def __getattr__(name): +def __getattr__(name): # type: ignore global _auth, _store if name == "__auth__" or name == "__store__": diff --git a/earthaccess/api.py b/earthaccess/api.py index ccb62200..0dc6961d 100644 --- a/earthaccess/api.py +++ b/earthaccess/api.py @@ -1,11 +1,10 @@ from typing import Any, Dict, List, Optional, Type, Union +import earthaccess import requests import s3fs from fsspec import AbstractFileSystem -import earthaccess - from .auth import Auth from .search import CollectionQuery, DataCollections, DataGranules, GranuleQuery from .store import Store diff --git a/earthaccess/store.py b/earthaccess/store.py index dd0f1369..e455d169 100644 --- a/earthaccess/store.py +++ b/earthaccess/store.py @@ -6,21 +6,20 @@ from functools import lru_cache from itertools import chain from pathlib import Path -from typing import Any, Callable, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Union from uuid import uuid4 +import earthaccess import fsspec import requests import s3fs from multimethod import multimethod as singledispatchmethod from pqdm.threads import pqdm -import earthaccess - +from .auth import Auth from .daac import DAAC_TEST_URLS, find_provider from .results import DataGranule from .search import DataCollections -from .auth import Auth class EarthAccessFile(fsspec.spec.AbstractBufferedFile): From 01fd340b9c94fcbe6e1a88d4e1c5351bf3887d06 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 13 Oct 2023 11:16:45 -0500 Subject: [PATCH 3/6] Docstring --- earthaccess/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 94d90546..71e1bd30 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -45,6 +45,12 @@ def __getattr__(name): # type: ignore + """ + Module-level getattr to handle automatic authentication when accessing + `earthaccess.__auth__` and `earthaccess.__store__`. + + Other unhandled attributes raise as `AttributeError` as expected. + """ global _auth, _store if name == "__auth__" or name == "__store__": From 335ae891731d8c1b570f37bf1c9e2ea17a34c12f Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 13 Oct 2023 11:43:45 -0500 Subject: [PATCH 4/6] More logging --- earthaccess/__init__.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 71e1bd30..2f8c55d0 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -1,6 +1,7 @@ import threading from importlib.metadata import version from typing import Any +import logging from .api import ( collection_query, @@ -19,6 +20,8 @@ from .search import DataCollections, DataGranules from .store import Store +logger = logging.getLogger(__name__) + __all__ = [ "login", "search_datasets", @@ -59,13 +62,15 @@ def __getattr__(name): # type: ignore for strategy in ["environment", "netrc"]: try: _auth.login(strategy=strategy) - except Exception: + except Exception as e: + logger.debug(f"An error occurred during automatic authentication with {strategy=}: {str(e)}") continue else: if not _auth.authenticated: continue else: _store = Store(_auth) + logger.debug(f"Automatic authentication with {strategy=} was successful") break return _auth if name == "__auth__" else _store else: From 1a20f87457e2bb58c0ffcb062d251853b98ca269 Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 13 Oct 2023 11:47:10 -0500 Subject: [PATCH 5/6] Lint --- earthaccess/__init__.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/earthaccess/__init__.py b/earthaccess/__init__.py index 2f8c55d0..221b8cd1 100644 --- a/earthaccess/__init__.py +++ b/earthaccess/__init__.py @@ -1,7 +1,7 @@ +import logging import threading from importlib.metadata import version from typing import Any -import logging from .api import ( collection_query, @@ -63,14 +63,18 @@ def __getattr__(name): # type: ignore try: _auth.login(strategy=strategy) except Exception as e: - logger.debug(f"An error occurred during automatic authentication with {strategy=}: {str(e)}") + logger.debug( + f"An error occurred during automatic authentication with {strategy=}: {str(e)}" + ) continue else: if not _auth.authenticated: continue else: _store = Store(_auth) - logger.debug(f"Automatic authentication with {strategy=} was successful") + logger.debug( + f"Automatic authentication with {strategy=} was successful" + ) break return _auth if name == "__auth__" else _store else: From 1cba8173614b0149f31fddf7d66e02123988800b Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Fri, 13 Oct 2023 12:08:37 -0500 Subject: [PATCH 6/6] Documentation updates --- README.md | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 8e40833d..3140c605 100644 --- a/README.md +++ b/README.md @@ -70,22 +70,24 @@ The only requirement to use this library is to open a free account with NASA [ED ### **Authentication** -Once you have an EDL account, you can authenticate using one of the following three methods: +By default, `earthaccess` with automatically look for your EDL account credentials in two locations: -1. Using a `.netrc` file - * Can use *earthaccess* to read your EDL credentials (username and password) from a `.netrc` file -2. Reading your EDL credentials from environment variables - * if available you can use environment variables **EARTHDATA_USERNAME** and **EARTHDATA_PASSWORD** -3. Interactively entering your EDL credentials - * You can be prompted for these credentials and save them to a `.netrc` file +1. A `~/.netrc` file +2. `EARTHDATA_USERNAME` and `EARTHDATA_PASSWORD` environment variables + +If neither of these options are configured, you can authenticate by calling the `earthaccess.login()` method +and manually entering your EDL account credentials. ```python import earthaccess -auth = earthaccess.login() - +earthaccess.login() ``` +Note you can pass `persist=True` to `earthaccess.login()` to have the EDL account credentials you enter +automatically saved to a `~/.netrc` file for future use. + + Once you are authenticated with NASA EDL you can: * Get a file from a DAAC using a `fsspec` session.