0.3.0

.gitignore

@@ -4,6 +4,8 @@ __pycache__/
 *$py.class
 *.tif
 *.xml
+*.gpkg
+*.csv
 
 # C extensions
 *.so

eo_validation/async_write.py

@@ -2,7 +2,7 @@
 
 
 # Inheriting the base class 'Thread'
-class AsyncWriteGDF(threading.Thread): 
+class AsyncWriteGDF(threading.Thread):
 
     def __init__(self):
 

eo_validation/validation_dashboard.py

@@ -266,7 +266,7 @@ def __init__(self, **kwargs):
         self._current_marker_id = None
         self._current_time = None
         self._seconds_per_point = None
-        
+
         self.async_writer = AsyncWriteGDF()
 
         # Adding default Google Basemap
@@ -631,7 +631,7 @@ def add_polygon_markers(
             #    self.output_dir, f"{Path(in_filename).stem}.gpkg")
             self.output_filename = os.path.join(
                 self.output_dir, f"{Path(in_filename).stem}.gpkg")
-            
+
         # Case #1: student is already working on the points
         if os.path.isfile(self.output_filename):
             validation_points = self.load_gpkg(self.output_filename)
@@ -730,13 +730,9 @@ def create_property_widgets(self, properties):
         )
         point_id_widget._property_key = 'ID'
 
-
-        # print("PRE UPDATE MARKER", self._current_marker_id)
         self._current_marker_id = property_items['ID']
-        # print("POST UPDATE MARKER", self._current_marker_id)
 
         # (y, x) as (lat, lon)
-
         point_coords_widget_y = widgets.Text(
             value=str(property_items['y']),
             description='Lat:',
@@ -763,14 +759,16 @@ def changed_checked_widget(b):
             self._seconds_per_point = round(
                     time.time() - self._current_time, 4)
             self._feature['properties']['date'] = str(pd.Timestamp.now())
-            self._feature['properties']['seconds_taken'] = self._seconds_per_point
+            self._feature['properties']['seconds_taken'] = \
+                self._seconds_per_point
             self._feature['properties']['verified'] = True
 
             # updating the information with new data
             self.geo_data_layer.geo_dataframe.loc[
                 self.geo_data_layer.geo_dataframe['ID']
                 == self._feature['properties']['ID'],
-                self._feature['properties'].keys()] = self._feature['properties'].values()
+                self._feature['properties'].keys()] = \
+                self._feature['properties'].values()
 
         checked_widget.observe(changed_checked_widget)
 
@@ -787,15 +785,16 @@ def changed_checked_widget(b):
         return popup
 
     def on_click_polygon_object(self, event, feature, **kwargs):
-        
+
         # get current time
-        self._current_time = time.time()        
+        self._current_time = time.time()
         self._feature = feature
 
         self._feature['properties'] = self.geo_data_layer.geo_dataframe.loc[
                 self.geo_data_layer.geo_dataframe['ID']
-                == self._feature['properties']['ID']].to_dict(orient='records')[0]
-
+                == self._feature['properties']['ID']
+        ].to_dict(orient='records')[0]
+
         # Dynamically create input widgets for each property
         self.property_widgets = self.create_property_widgets(
             self._feature['properties'])
@@ -832,7 +831,8 @@ def save_changes(_):
             original_feature = copy.deepcopy(self._feature)
             # Update the properties with the new values
             for widget in self.property_widgets:
-                self._feature['properties'][widget._property_key] = widget.value
+                self._feature['properties'][widget._property_key] = \
+                    widget.value
 
             for i, f in enumerate(original_data['features']):
                 if f == original_feature:
@@ -847,7 +847,8 @@ def save_changes(_):
             self.geo_data_layer.geo_dataframe.loc[
                 self.geo_data_layer.geo_dataframe['ID']
                 == self._feature['properties']['ID'],
-                self._feature['properties'].keys()] = self._feature['properties'].values()
+                self._feature['properties'].keys()] = \
+                self._feature['properties'].values()
 
             # saving output
             self.geo_data_layer.geo_dataframe.to_file(
@@ -893,15 +894,13 @@ def load_gpkg(self, input_filename):
                 'verified'].tolist()
         else:
             verified_list = gdf['verified'].tolist()
-            
+
         self._marker_counter = [
             i for i, x in enumerate(verified_list) if not x][0] - 1
 
         if self._marker_counter is None:
             self._marker_counter = -1
 
-        print("MARKER COUNTER", self._marker_counter)
-
         return gdf
 
     def _main_toolbar(self):

notebooks/examples/bin_peng/ValidationDatabaseReport-Peng.ipynb

@@ -0,0 +1,185 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "9f315b6b-73ad-467e-88af-70d09251eacd",
+   "metadata": {},
+   "source": [
+    "# Earth Observation Validation Report Notebook\n",
+    "\n",
+    "The following notebook provides a report on the progress of validations when performed\n",
+    "using the Teams option. The second portion of this notebook generates the overall aggregated database."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4c9353bc-4fb2-4b93-b1f5-2523bd87247a",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "import csv\n",
+    "import pandas as pd\n",
+    "import geopandas as gpd\n",
+    "\n",
+    "from glob import glob\n",
+    "from pathlib import Path\n",
+    "from datetime import date\n",
+    "from tabulate import tabulate"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "5f890f1e-5375-4b1e-af84-1d8cd29c1b1b",
+   "metadata": {},
+   "source": [
+    "## Validation Report"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ecd35cbe-1f58-400e-b523-db93e23eaf49",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# specify the data_dir option used in the ValidationDashboard notebook\n",
+    "data_dir = '/home/jovyan/efs/BinPeng_Colombia/validation'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "b2ea9a44-8504-4cad-9fd4-afbaf6fb8fc8",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report_list = []\n",
+    "fields = [\"Number of Points\", \"Verified Points\", \"Percentage\", \"Seconds Per Point (Mean)\", \"Username-Filename\"]\n",
+    "for username in os.listdir(data_dir):\n",
+    "    \n",
+    "    if username in [\".ipynb_checkpoints\", \"original_points\"]:\n",
+    "        continue\n",
+    "    \n",
+    "    filenames = glob(os.path.join(data_dir, username, '*.gpkg'))\n",
+    "\n",
+    "    for filename in filenames:\n",
+    "        \n",
+    "        try:\n",
+    "            gdf = gpd.read_file(filename)\n",
+    "            report_list.append(\n",
+    "                [\n",
+    "                    gdf.shape[0], # total points\n",
+    "                    gdf['verified'].sum(), # verified points\n",
+    "                    round((gdf['verified'].sum() / gdf.shape[0]) * 100, 2), # percentage done\n",
+    "                    pd.to_numeric(gdf['seconds_taken'], errors='coerce').mean(), # seconds per point\n",
+    "                    username\n",
+    "                ])\n",
+    "        except:\n",
+    "            report_list.append([\"broken file\", 0, Path(filename).stem])\n",
+    "\n",
+    "print (tabulate(report_list, headers=fields))\n",
+    "\n",
+    "with open(f'validation-database-report-{date.today()}.csv', 'w') as f:\n",
+    "     \n",
+    "    write = csv.writer(f)\n",
+    "    write.writerow(fields)\n",
+    "    write.writerows(report_list)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "54add659-a329-4d29-b238-ea66f768c11e",
+   "metadata": {},
+   "source": [
+    "# Validation Database Generation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "782acd67-1ba4-4477-9089-9739f8d2400c",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# specify the database_filename option used to name the Validation Database, ends with .gpkg\n",
+    "database_filename = f'validation-database-{date.today()}.gpkg'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4961d929-d929-4a2b-9cbc-8213db16058f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "report_list = []\n",
+    "database_list = []\n",
+    "for username in os.listdir(data_dir):\n",
+    "    \n",
+    "    if username in [\".ipynb_checkpoints\", \"original_points\"]:\n",
+    "        continue\n",
+    "    \n",
+    "    filenames = glob(os.path.join(data_dir, username, '*.gpkg'))\n",
+    "    \n",
+    "    for filename in filenames:\n",
+    "        \n",
+    "        try:\n",
+    "            gdf = gpd.read_file(filename)\n",
+    "\n",
+    "            report_list.append(\n",
+    "                [\n",
+    "                    gdf.shape[0], # total points\n",
+    "                    gdf['verified'].sum(), # verified points\n",
+    "                    round((gdf['verified'].sum() / gdf.shape[0]) * 100, 2), # percentage done\n",
+    "                    pd.to_numeric(gdf['seconds_taken'], errors='coerce').mean(), # seconds per point\n",
+    "                    username\n",
+    "                ])\n",
+    "\n",
+    "            gdf['username'] = username\n",
+    "            gdf['short_filename'] = Path(filename).stem\n",
+    "            database_list.append(gdf)\n",
+    "\n",
+    "        except:\n",
+    "            report_list.append([\"broken file\", 0, Path(filename).stem])\n",
+    "\n",
+    "print (tabulate(report_list, headers=[\n",
+    "    \"Number of Points\", \"Verified Points\", \"Percentage\", \"Seconds Per Point (Mean)\", \"Username-Filename\"]))\n",
+    "\n",
+    "full_database = pd.concat(database_list)\n",
+    "full_database.to_file(database_filename, driver='GPKG', layer='validation') \n",
+    "full_database.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "badd0b03-160c-403e-b650-6ea0fc1d42c1",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}