feature(*): add geosyspy logger + fix pandas version

update readme for logs update requirements
earthdaily · Dec 20, 2023 · 9fc99ab · 9fc99ab
1 parent 779adaa
commit 9fc99ab
Show file tree

Hide file tree

Showing 19 changed files with 144 additions and 55 deletions.
diff --git a/README.md b/README.md
@@ -226,6 +226,16 @@ year_ago = dt.date.today() + relativedelta(months=-12)
 dataframe = client.get_time_series(polygon, year_ago, today, collection=SatelliteImageryCollection.MODIS, indicators=["NDVI"])
 ```
 
+Use Geosyspy logger
+
+
+````python
+import logging
+
+geosys_logger = logging.getLogger('geosyspy')
+geosys_logger.setLevel(logging.DEBUG)
+````
+
 See the Jupyter notebook [Examples](examples.ipynb) for a working example.
 
 <p align="right">(<a href="#top">back to top</a>)</p>

diff --git a/examples.ipynb b/examples.ipynb
@@ -14,8 +14,8 @@
     "import logging\n",
     "from geosyspy.utils.constants import *\n",
     "\n",
-    "logger = logging.getLogger()\n",
-    "logger.setLevel(logging.INFO)\n",
+    "logger = logging.getLogger('geosyspy')\n",
+    "logger.setLevel(logging.DEBUG)\n",
     "\n",
     "# read .env file\n",
     "load_dotenv()\n",
@@ -106,7 +106,7 @@
    "cell_type": "code",
    "execution_count": null,
    "metadata": {
-    "scrolled": true
+    "scrolled": false
    },
    "outputs": [],
    "source": [
@@ -123,10 +123,53 @@
    },
    "outputs": [],
    "source": [
+    "\n",
+    "import xarray as xr\n",
     "# Get satellite image time series for LANDSAT_8 and SENTINEL_2 NDVI \n",
     "# list of available indicators: Reflectance, NDVI, EVI, GNDVI, NDWI, CVI, CVIn, LAI\n",
     "ndvi_time_series_xarr = client.get_satellite_image_time_series(polygon, year_ago, today, collections=[SatelliteImageryCollection.SENTINEL_2, SatelliteImageryCollection.LANDSAT_8], indicators=[\"ndvi\"])\n",
-    "ndvi_time_series_xarr"
+    "\n",
+    "ndvi_time_series_xarr\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [],
+   "source": [
+    "lai_time_series_xarr = client.get_satellite_image_time_series(polygon, year_ago, today, collections=[SatelliteImageryCollection.SENTINEL_2, SatelliteImageryCollection.LANDSAT_8], indicators=[\"NDWI\"])\n",
+    "lai_time_series_xarr"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pip install zarr\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if ndvi_time_series_xarr.coords == lai_time_series_xarr.coords:\n",
+    "    print(\"Les dimensions sont identiques.\")\n",
+    "else:\n",
+    "    print(\"Les dimensions ne sont pas identiques.\")\n",
+    "    print(ndvi_time_series_xarr.coords)\n",
+    "    print(lai_time_series_xarr.coords)\n",
+    "\n",
+    "\n",
+    "datacube = xr.merge([ndvi_time_series_xarr, lai_time_series_xarr])\n",
+    "\n",
+    "datacube.to_zarr(\"result.zarr\")"
    ]
   },
   {
@@ -354,8 +397,8 @@
     "# for a specific weather indicator\n",
     "aq_weather_data_frame = client.get_agriquest_weather_block_data(start_date=\"2022-06-01\",\n",
     "                                                  end_date=\"2023-06-10\",\n",
-    "                                                  block_code=AgriquestBlocks.FRA_DEPARTEMENTS,\n",
-    "                                                  weather_type=AgriquestWeatherType.SOIL_MOISTURE)\n",
+    "                                                  block_code=AgriquestBlocks.AMU_NORTH_AMERICA,\n",
+    "                                                  weather_type=AgriquestWeatherType.AVERAGE_TEMPERATURE)\n",
     "\n",
     "aq_weather_data_frame"
    ]
@@ -609,9 +652,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "geosyspy_aq",
+   "display_name": "geosyspy_0.2.0",
    "language": "python",
-   "name": "geosyspy_aq"
+   "name": "geosyspy_0.2.0"
   },
   "language_info": {
    "codemirror_mode": {
@@ -623,7 +666,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.4"
   },
   "vscode": {
    "interpreter": {

diff --git a/geosyspy/__init__.py b/geosyspy/__init__.py
@@ -1 +1,4 @@
 from .geosys import Geosys
+from .utils.logging_config import configure_logging
+
+configure_logging()
diff --git a/geosyspy/geosys.py b/geosyspy/geosys.py
@@ -38,6 +38,7 @@ def __init__(self, client_id: str,
                  enum_region: Region,
                  priority_queue: str = "realtime",
                  ):
+        self.logger = logging.getLogger(__name__)
         self.region: str = enum_region.value
         self.env: str = enum_env.value
         self.base_url: str = GEOSYS_API_URLS[enum_region.value][enum_env.value]
@@ -185,7 +186,7 @@ def download_image(self, image_reference,
         if path == "":
             path = Path.cwd() / f"image_{image_reference.image_id}_tiff.zip"
         with open(path, "wb") as f:
-            logging.info(f"writing to {path}")
+            self.logger.info(f"writing to {path}")
             f.write(response_zipped_tiff.content)
 
     def __get_images_as_dataset(self, polygon: str,
@@ -291,7 +292,7 @@ def get_coordinates_by_pixel(raster):
                                     f"The highest resolution's image grid size is {(len_x, len_y)}"
                                 )
                             else:
-                                logging.info(
+                                self.logger.info(
                                     f"interpolating {img_id} to {first_img_id}'s grid"
                                 )
                                 xarr = xarr.interp(
@@ -569,7 +570,7 @@ def get_harvest_analytics(self,
             harvest_type=harvest_type
         )
 
-        logging.info(f"Task Id: {task_id}")
+        self.logger.info(f"Task Id: {task_id}")
 
         # check the task status to continue or not the process
         self.__analytics_processor_service.wait_and_check_task_status(task_id)

diff --git a/geosyspy/services/agriquest_service.py b/geosyspy/services/agriquest_service.py
@@ -12,6 +12,7 @@ class AgriquestService:
     def __init__(self, base_url: str, http_client: HttpClient):
         self.base_url: str = base_url
         self.http_client: HttpClient = http_client
+        self.logger = logging.getLogger(__name__)
 
     def weather_indicators_builder(self, start_date, end_date, isFrance):
         """build weather indicators list from 2 dates
@@ -107,7 +108,7 @@ def get_agriquest_block_weather_data(self,
             df = df.rename(columns={"Name": "AMU"})
             return df
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
 
 
     def get_agriquest_block_ndvi_data(self,
@@ -155,4 +156,4 @@ def get_agriquest_block_ndvi_data(self,
             df = df.rename(columns={"Name": "AMU", "Value": "NDVI"})
             return df
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
diff --git a/geosyspy/services/analytics_fabric_service.py b/geosyspy/services/analytics_fabric_service.py
@@ -12,6 +12,8 @@ class AnalyticsFabricService:
     def __init__(self, base_url: str, http_client: HttpClient):
         self.base_url: str = base_url
         self.http_client: HttpClient = http_client
+        self.logger = logging.getLogger(__name__)
+
 
     @staticmethod
     def build_timestamp_query_parameters(start_date: Optional[datetime] = None,
@@ -69,9 +71,9 @@ def create_schema_id(self, schema_id: str,
         if response.status_code == 201 :
             return response.content
         elif response.status_code == 400 and "This schema already exists." in str(dict_response["Errors"]["Body"]["Id"]):
-            logging.info(f"The schema {schema_id} already exists.")
+            self.logger.info(f"The schema {schema_id} already exists.")
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
 
     def get_metrics(self, season_field_id: str,
                     schema_id: str,
@@ -95,7 +97,7 @@ def get_metrics(self, season_field_id: str,
             df : A Pandas DataFrame containing several columns with metrics
 
         """
-        logging.info("Calling APIs for metrics")
+        self.logger.info("Calling APIs for metrics")
         if start_date is not None:
             start_date: str = start_date.strftime("%Y-%m-%d")
         if end_date is not None:
@@ -122,7 +124,7 @@ def get_metrics(self, season_field_id: str,
                     date_msg =f"<= {end_date} "
                 elif start_date is not None and end_date is None:
                     date_msg = f">= {start_date} "
-                logging.info(f"No metrics found in Analytic Fabric with "
+                self.logger.info(f"No metrics found in Analytic Fabric with "
                              f"SchemaId: {schema_id}, "
                              f"SeasonField:{season_field_id} "
                              f"{date_msg} ")
@@ -136,7 +138,7 @@ def get_metrics(self, season_field_id: str,
             df.set_index("date", inplace=True)
             return df
         else:
-            logging.error("Issue in get_metrics. Status Code: "+response.status_code)
+            self.logger.error("Issue in get_metrics. Status Code: "+response.status_code)
 
     def get_lastest_metrics(self, season_field_id: str,
                             schema_id: str):
@@ -150,7 +152,7 @@ def get_lastest_metrics(self, season_field_id: str,
             df : A Pandas DataFrame containing several columns with metrics
 
         """
-        logging.info("Calling APIs for Latest metrics")
+        self.logger.info("Calling APIs for Latest metrics")
 
         parameters: str = f'?%24filter=Entity.TypedId==\'SeasonField:{season_field_id}\'' \
                           f'&Schema.Id={schema_id}' \
@@ -166,7 +168,7 @@ def get_lastest_metrics(self, season_field_id: str,
         if response.status_code == 200:
             df = pd.json_normalize(response.json())
             if df.empty:
-                logging.info(f"No Latest metrics found in Analytic Fabric with "
+                self.logger.info(f"No Latest metrics found in Analytic Fabric with "
                              f"SchemaId: {schema_id}, "
                              f"SeasonField:{season_field_id} ")
                 return df
@@ -179,7 +181,7 @@ def get_lastest_metrics(self, season_field_id: str,
             df.set_index("date", inplace=True)
             return df
         else:
-            logging.error("Issue in get_latests_metrics. Status Code: "+response.status_code)
+            self.logger.error("Issue in get_latests_metrics. Status Code: "+response.status_code)
 
     def push_metrics(self, season_field_id: str,
                      schema_id: str,
@@ -213,4 +215,4 @@ def push_metrics(self, season_field_id: str,
         if response.status_code == 200:
             return response.status_code
         else:
-            logging.error("Issue in push_metrics. Status Code: "+response.status_code)
+            self.logger.error("Issue in push_metrics. Status Code: "+response.status_code)
diff --git a/geosyspy/services/analytics_processor_service.py b/geosyspy/services/analytics_processor_service.py
@@ -13,6 +13,7 @@ class AnalyticsProcessorService:
     def __init__(self, base_url: str, http_client: HttpClient):
         self.base_url: str = base_url
         self.http_client: HttpClient = http_client
+        self.logger = logging.getLogger(__name__)
 
     @retrying.retry(wait_exponential_multiplier=1000, wait_exponential_max=10000, stop_max_attempt_number=50, retry_on_exception=lambda exc: isinstance(exc, KeyError))
     def wait_and_check_task_status(self, task_id: str):
@@ -35,13 +36,13 @@ def wait_and_check_task_status(self, task_id: str):
                 dict_resp = json.loads(response.content)
                 task_status = dict_resp["status"]
             else:
-                logging.info(response.status_code)
+                self.logger.info(response.status_code)
                 return "Failed"
 
             if task_status == "Ended":
                 break  # the task is completed.
             elif task_status == "Running":
-                logging.info("Retry -- Task still running")
+                self.logger.info("Retry -- Task still running")
                 raise KeyError("Task still running")  # raise exception to retry
             else:
                 raise Exception(f"Task Status: {task_status}, Content: {response.content}" )
@@ -71,7 +72,7 @@ def get_s3_path_from_task_and_processor(self, task_id: str,
             task_id = dict_resp["taskId"]
             return f"s3://geosys-{customer_code}/{user_id}/{processor_name}/{task_id}"
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_mr_time_series_processor(self, polygon,
@@ -135,7 +136,7 @@ def launch_mr_time_series_processor(self, polygon,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_planted_area_processor(self,
@@ -174,7 +175,7 @@ def launch_planted_area_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_harvest_processor(self,
@@ -230,7 +231,7 @@ def launch_harvest_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_emergence_processor(self,
@@ -289,7 +290,7 @@ def launch_emergence_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_potential_score_processor(self,
@@ -349,7 +350,7 @@ def launch_potential_score_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_brazil_in_season_crop_id_processor(self,
@@ -398,7 +399,7 @@ def launch_brazil_in_season_crop_id_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_greenness_processor(self,
@@ -449,7 +450,7 @@ def launch_greenness_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
 
@@ -501,7 +502,7 @@ def launch_harvest_readiness_processor(self,
             task_id = json.loads(response.content)["taskId"]
             return task_id
         else:
-            logging.info(response.status_code)
+            self.logger.info(response.status_code)
             raise ValueError(response.content)
 
     def launch_zarc_processor(self,
@@ -557,6 +558,6 @@ def launch_zarc_processor(self,
                 task_id = json.loads(response.content)["taskId"]
                 return task_id
             else:
-                logging.info(response.status_code)
+                self.logger.info(response.status_code)
                 raise ValueError(response.content)