Skip to content

Commit

Permalink
feat: Initialize EarthEngine with the project ID in heet_export
Browse files Browse the repository at this point in the history
- Fixes authentication issues when trying to export to Google Drive.
- Makes the project folder a required parameter in CLI.
  • Loading branch information
tomjanus committed Sep 26, 2024
1 parent e0f8398 commit bd20e58
Showing 1 changed file with 22 additions and 26 deletions.
48 changes: 22 additions & 26 deletions geocaret_export_cli.py
Original file line number Diff line number Diff line change
@@ -1,28 +1,19 @@
import os
import argparse
import logging
import ee
import logging
import polling2
from tqdm import tqdm
from yaspin import yaspin
from yaspin.spinners import Spinners

ee.Initialize() # Needs initialising before loading from geocaret

from geocaret import config as cfg
from geocaret import log as lg
from geocaret import task
from geocaret import monitor as mtr

lg.log_file_name = "geocaret_export.log"

parser = argparse.ArgumentParser(
usage="python geocaret_export_cli.py --results-path <path-to-results-folder-in-GEE> --drive-folder <folder-in-GDrive> --project <GEE-project-name>"
usage="python heet_export_cli.py --results-path <path-to-results-folder-in-GEE> --drive-folder <folder-in-GDrive> --project <GEE-project-name>"
)
parser.add_argument(
"--results-path",
required=True,
help="Full path to the geocaret results folder on Earth Engine (must start from projects/<project-name>/...)"
help="Full path to the heet results folder on Earth Engine (must start from projects/<project-name>/...)"
)

# EE Export to Drive functionality is not well documented
Expand All @@ -42,7 +33,7 @@
parser.add_argument(
"--project",
type=str,
default = "",
required=True,
help="Name of Earth Engine cloud project to use",
)

Expand All @@ -51,10 +42,15 @@
drive_folder = args.drive_folder
project = args.project

if project:
ee.Initialize(project=project)
else:
ee.Initialize()
ee.Initialize(project=project)

# Importing from delineator needs to be done after ee.Initialize
from delineator import heet_config as cfg
from delineator import heet_log as lg
from delineator import heet_task
from delineator import heet_monitor as mtr

lg.log_file_name = "heet_export.log"

# Warning - overwriting ipmorted config data.
cfg.output_drive_folder = drive_folder
Expand Down Expand Up @@ -97,18 +93,18 @@ def update_sp_fail(sp):

def update_sp_err_fatal(sp):
sp.write("")
sp.write(" [ERROR] GeoCARET EXPORTER encountered a fatal error and will exit")
sp.write(" [ERROR] HEET EXPORTER encountered a fatal error and will exit")
sp.write("")
sp.write("Thank you for using GeoCARET EXPORTER!")
sp.write("Thank you for using HEET EXPORTER!")
return sp


def update_sp_warn_skip(sp):
sp.write("")
sp.write(
" [WARNING] GeoCARET EXPORTER encountered an problem and will skip Export step"
" [WARNING] HEET EXPORTER encountered an problem and will skip Export step"
)
sp.write(" [WARNING] Check geocaret_export.log for further details.")
sp.write(" [WARNING] Check heet_export.log for further details.")
sp.write("")
sp.write("")
return sp
Expand All @@ -117,7 +113,7 @@ def update_sp_warn_skip(sp):
def update_sp_err_time(sp):
current_active_analyses = ",".join([str(i) for i in mtr.active_analyses])
sp.write(
" [WARNING] Analysis wait time limit exceeded. Cancelling all unfinished GeoCARET EXPORTER"
" [WARNING] Analysis wait time limit exceeded. Cancelling all unfinished HEET EXPORTER"
)
sp.write(f" tasks (active analysis ids: {current_active_analyses})")
sp.write("")
Expand Down Expand Up @@ -151,7 +147,7 @@ def update_sp_inf_service(sp):
if "CI_ROBOT_USER" in os.environ:
sp = update_sp_inf_service(sp)
else:
task.export_to_drive(export_from_path=results_path)
heet_task.export_to_drive(export_from_path=results_path)
except Exception as error:
# Handles any issue, including connectivity
print("Raised exception")
Expand All @@ -168,7 +164,7 @@ def update_sp_inf_service(sp):
keep_going = 1
while (len(mtr.active_exports) > 0) and (keep_going == 1):
try:
task.wait_until_exports()
heet_task.wait_until_exports()
active_count = len(mtr.active_exports)
new_exports = remaining_export_size - active_count
remaining_export_size = active_count
Expand All @@ -178,10 +174,10 @@ def update_sp_inf_service(sp):
sp = update_sp_fail(sp)
sp = update_sp_err_time(sp)
logger.info(
"(Waiting) Analysis wait time limit exceeded. Cancelling all unfinished GeoCARET tasks."
"(Waiting) Analysis wait time limit exceeded. Cancelling all unfinished HEET tasks."
)
try:
task.kill_all_heet_tasks()
heet_task.kill_all_heet_tasks()
except Exception:
sp = update_sp_err_fatal(sp)
sys.exit()
Expand Down

0 comments on commit bd20e58

Please sign in to comment.