From f1834556846d63089dc61361a766809c63564e93 Mon Sep 17 00:00:00 2001 From: dfguerrerom Date: Wed, 8 May 2024 16:04:23 +0200 Subject: [PATCH] refactor: update gee auth process --- .github/workflows/unit.yml | 4 +- docs/source/tutorials/decorator.rst | 3 +- sepal_ui/aoi/aoi_model.py | 4 +- sepal_ui/message/en/locale.json | 3 +- sepal_ui/scripts/decorator.py | 50 +++++++++++++----- sepal_ui/scripts/gee.py | 4 +- sepal_ui/scripts/utils.py | 50 +++++++++++++----- sepal_ui/sepalwidgets/inputs.py | 22 +++++++- tests/conftest.py | 13 +++-- tests/test_scripts/test_decorator.py | 58 +++++++++++++++++++-- tests/test_scripts/test_utils.py | 58 +++++++++++++++++++-- tests/test_sepalwidgets/test_AssetSelect.py | 11 +++- 12 files changed, 231 insertions(+), 49 deletions(-) diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index d7e9d8cf..5b3a0b12 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -9,7 +9,9 @@ on: env: PLANET_API_CREDENTIALS: ${{ secrets.PLANET_API_CREDENTIALS }} PLANET_API_KEY: ${{ secrets.PLANET_API_KEY }} - EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_TOKEN }} + EARTHENGINE_TOKEN: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }} + EARTHENGINE_SERVICE_ACCOUNT: ${{ secrets.EARTHENGINE_SERVICE_ACCOUNT }} + EARTHENGINE_PROJECT: ${{ secrets.EARTHENGINE_PROJECT }} jobs: lint: diff --git a/docs/source/tutorials/decorator.rst b/docs/source/tutorials/decorator.rst index 3ac23b31..552d921d 100644 --- a/docs/source/tutorials/decorator.rst +++ b/docs/source/tutorials/decorator.rst @@ -78,6 +78,7 @@ Let's import the required modules. All the decorators are stored in the utils mo .. code:: python + import ee from time import sleep import ipyvuetify as v import sepal_ui.sepalwidgets as sw @@ -142,7 +143,7 @@ It's time to use the decorators in the class methods. For this example, we will def request_items(self): """Connect to gee and request the root assets id's""" - folder = ee.data.getAssetRoots()[0]["id"] + folder = f"projects/{ee.data._cloud_api_user_project}/assets" return [ asset["id"] for asset diff --git a/sepal_ui/aoi/aoi_model.py b/sepal_ui/aoi/aoi_model.py index 2a230797..9f513810 100644 --- a/sepal_ui/aoi/aoi_model.py +++ b/sepal_ui/aoi/aoi_model.py @@ -154,7 +154,9 @@ def __init__( self.gee = gee if gee: su.init_ee() - self.folder = str(folder) or ee.data.getAssetRoots()[0]["id"] + self.folder = ( + str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/" + ) # set default values self.set_default(vector, admin, asset) diff --git a/sepal_ui/message/en/locale.json b/sepal_ui/message/en/locale.json index 4b08e551..f03349a2 100644 --- a/sepal_ui/message/en/locale.json +++ b/sepal_ui/message/en/locale.json @@ -42,7 +42,8 @@ "custom": "Custom", "no_access": "It seems like you do not have access to the input asset or it does not exist.", "wrong_type": "The type of the selected asset ({}) does not match authorized asset type ({}).", - "placeholder": "users/custom_user/custom_asset" + "placeholder": "projects/{project}/assets/asset_name", + "no_assets": "No user assets found in: '{}'" }, "load_table": { "too_small": "The provided file have less than 3 columns. Please provide a complete point file with at least ID, lattitude and longitude columns." diff --git a/sepal_ui/scripts/decorator.py b/sepal_ui/scripts/decorator.py index 4dfcf9c8..98de3b84 100644 --- a/sepal_ui/scripts/decorator.py +++ b/sepal_ui/scripts/decorator.py @@ -8,6 +8,7 @@ ... """ +import json import os import warnings from functools import wraps @@ -17,7 +18,6 @@ from warnings import warn import ee -import httplib2 import ipyvuetify as v from deprecated.sphinx import versionadded @@ -34,28 +34,52 @@ def init_ee() -> None: - """Initialize earth engine according to the environment. + r"""Initialize earth engine according using a token. - It will use the creddential file if the EARTHENGINE_TOKEN env variable exist. - Otherwise it use the simple Initialize command (asking the user to register if necessary). + THe environment used to run the tests need to have a EARTHENGINE_TOKEN variable. + The content of this variable must be the copy of a personal credential file that you can find on your local computer if you already run the earth engine command line tool. See the usage question for a github action example. + + - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials`` + - Linux: ``/home/USERNAME/.config/earthengine/credentials`` + - MacOS: ``/Users/USERNAME/.config/earthengine/credentials`` + + Note: + As all init method of pytest-gee, this method will fallback to a regular ``ee.Initialize()`` if the environment variable is not found e.g. on your local computer. """ - # only do the initialization if the credential are missing if not ee.data._credentials: - # if the credentials token is asved in the environment use it - if "EARTHENGINE_TOKEN" in os.environ: + credential_folder_path = Path.home() / ".config" / "earthengine" + credential_file_path = credential_folder_path / "credentials" + + if "EARTHENGINE_TOKEN" in os.environ and not credential_file_path.exists(): + # write the token to the appropriate folder ee_token = os.environ["EARTHENGINE_TOKEN"] - credential_folder_path = Path.home() / ".config" / "earthengine" credential_folder_path.mkdir(parents=True, exist_ok=True) - credential_file_path = credential_folder_path / "credentials" credential_file_path.write_text(ee_token) + # Extract the project name from credentials + _credentials = json.loads(credential_file_path.read_text()) + project_id = _credentials.get("project_id", _credentials.get("project", None)) + + if not project_id: + raise NameError( + "The project name cannot be detected. " + "Please set it using `earthengine set_project project_name`." + ) + + # Check if we are using a google service account + if _credentials.get("type") == "service_account": + ee_user = _credentials.get("client_email") + credentials = ee.ServiceAccountCredentials( + ee_user, str(credential_file_path) + ) + ee.Initialize(credentials=credentials) + ee.data._cloud_api_user_project = project_id + return + # if the user is in local development the authentication should # already be available - ee.Initialize(http_transport=httplib2.Http()) - assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root - - return + ee.Initialize(project=project_id) ################################################################################ diff --git a/sepal_ui/scripts/gee.py b/sepal_ui/scripts/gee.py index 175a33e7..b3807e2f 100644 --- a/sepal_ui/scripts/gee.py +++ b/sepal_ui/scripts/gee.py @@ -95,7 +95,7 @@ def get_assets(folder: Union[str, Path] = "") -> List[dict]: """ # set the folder and init the list asset_list = [] - folder = str(folder) or ee.data.getAssetRoots()[0]["id"] + folder = str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/" def _recursive_get(folder, asset_list): @@ -122,7 +122,7 @@ def is_asset(asset_name: str, folder: Union[str, Path] = "") -> bool: true if already in folder """ # get the folder - folder = str(folder) or ee.data.getAssetRoots()[0]["id"] + folder = str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/" # get all the assets asset_list = get_assets(folder) diff --git a/sepal_ui/scripts/utils.py b/sepal_ui/scripts/utils.py index de2f128b..d59e62f4 100644 --- a/sepal_ui/scripts/utils.py +++ b/sepal_ui/scripts/utils.py @@ -1,6 +1,7 @@ """All the helper function of sepal-ui.""" import configparser +import json import math import os import random @@ -12,7 +13,6 @@ from urllib.parse import urlparse import ee -import httplib2 import ipyvuetify as v import requests import tomli @@ -127,28 +127,52 @@ def get_file_size(filename: Pathlike) -> str: def init_ee() -> None: - """Initialize earth engine according to the environment. + r"""Initialize earth engine according using a token. - It will use the creddential file if the EARTHENGINE_TOKEN env variable exist. - Otherwise it use the simple Initialize command (asking the user to register if necessary). + THe environment used to run the tests need to have a EARTHENGINE_TOKEN variable. + The content of this variable must be the copy of a personal credential file that you can find on your local computer if you already run the earth engine command line tool. See the usage question for a github action example. + + - Windows: ``C:\Users\USERNAME\\.config\\earthengine\\credentials`` + - Linux: ``/home/USERNAME/.config/earthengine/credentials`` + - MacOS: ``/Users/USERNAME/.config/earthengine/credentials`` + + Note: + As all init method of pytest-gee, this method will fallback to a regular ``ee.Initialize()`` if the environment variable is not found e.g. on your local computer. """ - # only do the initialization if the credential are missing if not ee.data._credentials: - # if the credentials token is asved in the environment use it - if "EARTHENGINE_TOKEN" in os.environ: + credential_folder_path = Path.home() / ".config" / "earthengine" + credential_file_path = credential_folder_path / "credentials" + + if "EARTHENGINE_TOKEN" in os.environ and not credential_file_path.exists(): + # write the token to the appropriate folder ee_token = os.environ["EARTHENGINE_TOKEN"] - credential_folder_path = Path.home() / ".config" / "earthengine" credential_folder_path.mkdir(parents=True, exist_ok=True) - credential_file_path = credential_folder_path / "credentials" credential_file_path.write_text(ee_token) + # Extract the project name from credentials + _credentials = json.loads(credential_file_path.read_text()) + project_id = _credentials.get("project_id", _credentials.get("project", None)) + + if not project_id: + raise NameError( + "The project name cannot be detected. " + "Please set it using `earthengine set_project project_name`." + ) + + # Check if we are using a google service account + if _credentials.get("type") == "service_account": + ee_user = _credentials.get("client_email") + credentials = ee.ServiceAccountCredentials( + ee_user, str(credential_file_path) + ) + ee.Initialize(credentials=credentials) + ee.data._cloud_api_user_project = project_id + return + # if the user is in local development the authentication should # already be available - ee.Initialize(http_transport=httplib2.Http()) - assert len(ee.data.getAssetRoots()) > 0, ms.utils.ee.no_asset_root - - return + ee.Initialize(project=project_id) def normalize_str(msg: str, folder: bool = True) -> str: diff --git a/sepal_ui/sepalwidgets/inputs.py b/sepal_ui/sepalwidgets/inputs.py index 2c0de078..5062c698 100644 --- a/sepal_ui/sepalwidgets/inputs.py +++ b/sepal_ui/sepalwidgets/inputs.py @@ -690,7 +690,9 @@ def __init__( self.asset_info = None # if folder is not set use the root one - self.folder = str(folder) or ee.data.getAssetRoots()[0]["id"] + self.folder = ( + str(folder) or f"projects/{ee.data._cloud_api_user_project}/assets/" + ) self.types = types # load the default assets @@ -699,6 +701,8 @@ def __init__( # Validate the input as soon as the object is instantiated self.observe(self._validate, "v_model") + self.observe(self._fill_no_data, "items") + # set the default parameters kwargs.setdefault("v_model", None) kwargs.setdefault("clearable", True) @@ -714,10 +718,26 @@ def __init__( # load the assets in the combobox self._get_items() + self._fill_no_data({}) + # add js behaviours self.on_event("click:prepend", self._get_items) self.observe(self._get_items, "default_asset") + def _fill_no_data(self, _: dict) -> None: + """Fill the items with a no data message if the items are empty.""" + # Done in this way because v_slots are not working + if not self.items: + self.v_model = None + self.items = [ + { + "text": ms.widgets.asset_select.no_assets.format(self.folder), + "disabled": True, + } + ] + + return + @sd.switch("loading") def _validate(self, change: dict) -> None: """Validate the selected asset. Throw an error message if is not accessible or not in the type list.""" diff --git a/tests/conftest.py b/tests/conftest.py index 80a99a5c..e20ce789 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,8 +21,9 @@ try: su.init_ee() -except Exception: - pass # try to init earthengine. use ee.data._credentials to skip +except Exception as e: + raise e + # pass # try to init earthengine. use ee.data._credentials to skip # -- a component to fake the display in Ipython -------------------------------- @@ -123,7 +124,7 @@ def gee_dir(_hash: str) -> Optional[Path]: pytest.skip("Eathengine is not connected") # create a test folder with a hash name - root = ee.data.getAssetRoots()[0]["id"] + root = f"projects/{ee.data._cloud_api_user_project}/assets/" gee_dir = Path(root) / f"sepal-ui-{_hash}" ee.data.createAsset({"type": "FOLDER"}, str(gee_dir)) @@ -197,7 +198,7 @@ def fake_asset(gee_dir: Path) -> Path: @pytest.fixture(scope="session") def gee_user_dir(gee_dir: Path) -> Path: - """Return the path to the gee_dir assets without the project elements. + """Return the path to the gee_dir assets. Args: gee_dir: the path to the session defined GEE directory @@ -205,9 +206,7 @@ def gee_user_dir(gee_dir: Path) -> Path: Returns: the path to gee_dir """ - legacy_project = Path("projects/earthengine-legacy/assets") - - return gee_dir.relative_to(legacy_project) + return gee_dir @pytest.fixture(scope="session") diff --git a/tests/test_scripts/test_decorator.py b/tests/test_scripts/test_decorator.py index c32c04b6..1ca0f432 100644 --- a/tests/test_scripts/test_decorator.py +++ b/tests/test_scripts/test_decorator.py @@ -1,6 +1,9 @@ """Test the custom decorators.""" +import json +import os import warnings +from pathlib import Path import ee import ipyvuetify as v @@ -11,11 +14,58 @@ from sepal_ui.scripts.warning import SepalWarning -@pytest.mark.skipif(not ee.data._credentials, reason="GEE is not set") def test_init_ee() -> None: - """Check that ee can be initialized from sepal_ui.""" - # check that no error is raised - sd.init_ee() + """Test the init_ee_from_token function.""" + credentials_filepath = Path(ee.oauth.get_credentials_path()) + existing = False + + try: + # Reset credentials to force the initialization + # It can be initiated from different imports + ee.data._credentials = None + + # Get the credentials path + + # Remove the credentials file if it exists + if credentials_filepath.exists(): + existing = True + credentials_filepath.rename(credentials_filepath.with_suffix(".json.bak")) + + # Act: Earthengine token should be created + sd.init_ee() + + assert credentials_filepath.exists() + + # read the back up and remove the "project_id" key + credentials = json.loads( + credentials_filepath.with_suffix(".json.bak").read_text() + ) + + ## 2. Assert when there's no a project associated + # remove the project_id key if it exists + ee.data._credentials = None + credentials.pop("project_id", None) + credentials.pop("project", None) + if "EARTHENGINE_PROJECT" in os.environ: + del os.environ["EARTHENGINE_PROJECT"] + + # write the new credentials + credentials_filepath.write_text(json.dumps(credentials)) + + with pytest.raises(NameError) as e: + sd.init_ee() + + # Access the exception message via `e.value` + error_message = str(e.value) + assert "The project name cannot be detected" in error_message + + finally: + # restore the file + if existing: + credentials_filepath.with_suffix(".json.bak").rename(credentials_filepath) + + # check that no error is raised + sd.init_ee() return diff --git a/tests/test_scripts/test_utils.py b/tests/test_scripts/test_utils.py index 090899ab..33cdd646 100644 --- a/tests/test_scripts/test_utils.py +++ b/tests/test_scripts/test_utils.py @@ -1,7 +1,10 @@ """Test the helper methods contained in utils file.""" +import json +import os import random from configparser import ConfigParser +from pathlib import Path from unittest.mock import patch import ee @@ -105,11 +108,58 @@ def test_get_file_size() -> None: return -@pytest.mark.skipif(not ee.data._credentials, reason="GEE is not set") def test_init_ee() -> None: - """Check we can init EE.""" - # check that no error is raised - su.init_ee() + """Test the init_ee_from_token function.""" + credentials_filepath = Path(ee.oauth.get_credentials_path()) + existing = False + + try: + # Reset credentials to force the initialization + # It can be initiated from different imports + ee.data._credentials = None + + # Get the credentials path + + # Remove the credentials file if it exists + if credentials_filepath.exists(): + existing = True + credentials_filepath.rename(credentials_filepath.with_suffix(".json.bak")) + + # Act: Earthengine token should be created + su.init_ee() + + assert credentials_filepath.exists() + + # read the back up and remove the "project_id" key + credentials = json.loads( + credentials_filepath.with_suffix(".json.bak").read_text() + ) + + ## 2. Assert when there's no a project associated + # remove the project_id key if it exists + ee.data._credentials = None + credentials.pop("project_id", None) + credentials.pop("project", None) + if "EARTHENGINE_PROJECT" in os.environ: + del os.environ["EARTHENGINE_PROJECT"] + + # write the new credentials + credentials_filepath.write_text(json.dumps(credentials)) + + with pytest.raises(NameError) as e: + su.init_ee() + + # Access the exception message via `e.value` + error_message = str(e.value) + assert "The project name cannot be detected" in error_message + + finally: + # restore the file + if existing: + credentials_filepath.with_suffix(".json.bak").rename(credentials_filepath) + + # check that no error is raised + su.init_ee() return diff --git a/tests/test_sepalwidgets/test_AssetSelect.py b/tests/test_sepalwidgets/test_AssetSelect.py index 0c1c3776..2862c78d 100644 --- a/tests/test_sepalwidgets/test_AssetSelect.py +++ b/tests/test_sepalwidgets/test_AssetSelect.py @@ -7,6 +7,7 @@ import pytest from sepal_ui import sepalwidgets as sw +from sepal_ui.message import ms @pytest.mark.skipif(not ee.data._credentials, reason="GEE is not set") @@ -24,7 +25,15 @@ def test_init(gee_dir: Path, gee_user_dir: Path) -> None: # create an asset select with an undefined type asset_select = sw.AssetSelect(folder=str(gee_dir), types=["toto"]) - assert asset_select.items == [] + + # zero assets are represented by a disabled item + no_asset_item = [ + { + "text": ms.widgets.asset_select.no_assets.format(str(gee_dir)), + "disabled": True, + } + ] + assert asset_select.items == no_asset_item return