From c6f35a558f51fb7cd076a81d7131830dac73506c Mon Sep 17 00:00:00 2001 From: Tyler Erickson Date: Mon, 25 Mar 2024 06:49:40 -0700 Subject: [PATCH] Add ability to filter out expired jobs --- 00_core.ipynb | 53 +++++++- 01_hyp3.ipynb => 01_asf_hyp3.ipynb | 208 ++++++++++++++++++----------- sar_asf_to_gee/asf_hyp3.py | 191 ++++++++++++++++++++++++++ sar_asf_to_gee/core.py | 39 +++++- 4 files changed, 412 insertions(+), 79 deletions(-) rename 01_hyp3.ipynb => 01_asf_hyp3.ipynb (53%) create mode 100644 sar_asf_to_gee/asf_hyp3.py diff --git a/00_core.ipynb b/00_core.ipynb index b5089b0..3fe6319 100644 --- a/00_core.ipynb +++ b/00_core.ipynb @@ -35,8 +35,11 @@ "outputs": [], "source": [ "#| export\n", - "import ee\n", - "import logging" + "import datetime\n", + "from dateutil import parser\n", + "import logging\n", + "\n", + "import ee" ] }, { @@ -63,6 +66,45 @@ " raise(e)" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "#| export\n", + "def filter_jobs(\n", + " # hyp3_batch,\n", + " jobs,\n", + " expired=None,\n", + " status_code=None,\n", + "):\n", + " \"Filter ASF batch jobs by specified criteria.\"\n", + " # jobs = hyp3_batch.jobs\n", + " # Filter by expiration status.\n", + " if expired is False:\n", + " jobs = [\n", + " job for job in jobs\n", + " if parser.parse(job.to_dict()['expiration_time']) > datetime.datetime.now(datetime.timezone.utc)\n", + " ]\n", + " elif expired is True:\n", + " jobs = [\n", + " job for job in jobs\n", + " if parser.parse(job.to_dict()['expiration_time']) <= datetime.datetime.now(datetime.timezone.utc)\n", + " ]\n", + " # Filter by status code.\n", + " if isinstance(status_code, str):\n", + " print('Status code is a string')\n", + " status_code = [status_code]\n", + " if isinstance(status_code, list):\n", + " print('Status code is a list')\n", + " jobs = [\n", + " job for job in jobs\n", + " if job.to_dict()['status_code'] in status_code\n", + " ]\n", + " return jobs" + ] + }, { "cell_type": "code", "execution_count": null, @@ -72,6 +114,13 @@ "#| hide\n", "import nbdev; nbdev.nbdev_export()" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/01_hyp3.ipynb b/01_asf_hyp3.ipynb similarity index 53% rename from 01_hyp3.ipynb rename to 01_asf_hyp3.ipynb index 54dbdb7..68dfff9 100644 --- a/01_hyp3.ipynb +++ b/01_asf_hyp3.ipynb @@ -17,7 +17,7 @@ "metadata": {}, "outputs": [], "source": [ - "#| default_exp hyp3" + "#| default_exp asf_hyp3" ] }, { @@ -145,12 +145,12 @@ "metadata": {}, "outputs": [], "source": [ - "# job_type = 'INSAR_GAMMA'\n", - "# job_name = 'INSAR_GAMMA_processing_example'\n", - "# granule1 = 'S1A_IW_SLC__1SDV_20240311T185926_20240311T185953_052938_066872_3CAD'\n", - "# granule2 = 'S1A_IW_SLC__1SDV_20231206T185929_20231206T185956_051538_0638B3_78A8'\n", - "# def submit_job():\n", - "# return hyp3.submit_insar_job(granule1, granule2, job_name)" + "job_type = 'INSAR_GAMMA'\n", + "job_name = 'INSAR_GAMMA_processing_example'\n", + "granule1 = 'S1A_IW_SLC__1SDV_20240311T185926_20240311T185953_052938_066872_3CAD'\n", + "granule2 = 'S1A_IW_SLC__1SDV_20231206T185929_20231206T185956_051538_0638B3_78A8'\n", + "def submit_job():\n", + " return hyp3.submit_insar_job(granule1, granule2, job_name)" ] }, { @@ -160,12 +160,12 @@ "metadata": {}, "outputs": [], "source": [ - "job_type = 'INSAR_ISCE_BURST'\n", - "job_name = 'INSAR_ISCE_BURST_processing_example'\n", - "burst1 = 'S1_184906_IW1_20240104T154111_VV_3C7F-BURST'\n", - "burst2 = 'S1_184906_IW1_20240116T154110_VV_D1E5-BURST'\n", - "def submit_job():\n", - " return hyp3.submit_insar_isce_burst_job(burst1, burst2, job_name)" + "# job_type = 'INSAR_ISCE_BURST'\n", + "# job_name = 'INSAR_ISCE_BURST_processing_example'\n", + "# burst1 = 'S1_184906_IW1_20240104T154111_VV_3C7F-BURST'\n", + "# burst2 = 'S1_184906_IW1_20240116T154110_VV_D1E5-BURST'\n", + "# def submit_job():\n", + "# return hyp3.submit_insar_isce_burst_job(burst1, burst2, job_name)" ] }, { @@ -178,7 +178,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Number of INSAR_ISCE_BURST jobs = 1\n" + "Number of INSAR_GAMMA jobs = 1\n" ] } ], @@ -200,19 +200,22 @@ "name": "stdout", "output_type": "stream", "text": [ - "Number of active INSAR_ISCE_BURST jobs = 1\n" + "Number of active INSAR_GAMMA jobs = 1\n" ] } ], "source": [ - "from dateutil import parser\n", + "from sar_asf_to_gee import core\n", + "\n", + "# # Remove failed jobs\n", + "# batch_active = [\n", + "# job for job in batch_completed.jobs\n", + "# if parser.parse(job.to_dict()['expiration_time']) > datetime.datetime.now(datetime.timezone.utc)\n", + "# and job.to_dict()['status_code'] != 'FAILED'\n", + "# ]\n", + "\n", + "batch_active = core.filter_jobs(batch_completed.jobs, expired=False)\n", "\n", - "# Remove failed jobs\n", - "batch_active = [\n", - " job for job in batch_completed.jobs\n", - " if parser.parse(job.to_dict()['expiration_time']) > datetime.datetime.now(datetime.timezone.utc)\n", - " and job.to_dict()['status_code'] != 'FAILED'\n", - "]\n", "print(f'Number of active {job_type} jobs = {len(batch_active)}')" ] }, @@ -250,7 +253,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Number of successful INSAR_ISCE_BURST jobs = 1\n", + "Number of successful INSAR_GAMMA jobs = 1\n", "Selecting the latest successful job.\n" ] } @@ -276,40 +279,49 @@ "data": { "application/json": { "browse_images": [ - "https://d3gm2hf49xd6jj.cloudfront.net/bbec3d98-3a70-42e8-838b-9e74b4f907fc/S1_184906_IW1_20240104_20240116_VV_INT80_E33E_unw_phase.png" + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_color_phase.png", + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.png" ], "credit_cost": 1, "expiration_time": "2024-03-28T00:00:00+00:00", "files": [ { - "filename": "S1_184906_IW1_20240104_20240116_VV_INT80_E33E.zip", + "filename": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip", "s3": { "bucket": "hyp3-edc-prod-contentbucket-1fv14ed36ifj6", - "key": "bbec3d98-3a70-42e8-838b-9e74b4f907fc/S1_184906_IW1_20240104_20240116_VV_INT80_E33E.zip" + "key": "4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip" }, - "size": 14091784, - "url": "https://d3gm2hf49xd6jj.cloudfront.net/bbec3d98-3a70-42e8-838b-9e74b4f907fc/S1_184906_IW1_20240104_20240116_VV_INT80_E33E.zip" + "size": 93984120, + "url": "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip" } ], - "job_id": "bbec3d98-3a70-42e8-838b-9e74b4f907fc", + "job_id": "4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b", "job_parameters": { "apply_water_mask": false, "granules": [ - "S1_184906_IW1_20240104T154111_VV_3C7F-BURST", - "S1_184906_IW1_20240116T154110_VV_D1E5-BURST" + "S1A_IW_SLC__1SDV_20240311T185926_20240311T185953_052938_066872_3CAD", + "S1A_IW_SLC__1SDV_20231206T185929_20231206T185956_051538_0638B3_78A8" ], - "looks": "20x4" + "include_dem": false, + "include_displacement_maps": false, + "include_inc_map": false, + "include_look_vectors": false, + "include_los_displacement": false, + "include_wrapped_phase": false, + "looks": "20x4", + "phase_filter_parameter": 0.6 }, - "job_type": "INSAR_ISCE_BURST", + "job_type": "INSAR_GAMMA", "logs": [], - "name": "INSAR_ISCE_BURST_processing_example", + "name": "INSAR_GAMMA_processing_example", "processing_times": [ - 456.229 + 2425.944 ], - "request_time": "2024-03-13T20:36:12+00:00", + "request_time": "2024-03-13T20:56:36+00:00", "status_code": "SUCCEEDED", "thumbnail_images": [ - "https://d3gm2hf49xd6jj.cloudfront.net/bbec3d98-3a70-42e8-838b-9e74b4f907fc/S1_184906_IW1_20240104_20240116_VV_INT80_E33E_unw_phase_thumb.png" + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_color_phase_thumb.png", + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase_thumb.png" ], "user_id": "tylere" }, @@ -464,22 +476,34 @@ "output_type": "stream", "text": [ "INFO:root:Starting hpy3_results_to_local()\n", - "INFO:root:Processing S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.zip\n", - "/Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/asf_search/download/download.py:65: UserWarning: File already exists, skipping download: temp_downloads/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.zip\n", + "INFO:root:Processing S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip\n", + "/Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/asf_search/download/download.py:65: UserWarning: File already exists, skipping download: temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip\n", " warnings.warn(f'File already exists, skipping download: {os.path.join(path, filename)}')\n", "INFO:root: Unzipping the file\n", - "INFO:root: Converting to a Cloud Optimized GeoTIFF. 1/2\n", - "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_ls_map.tif\n", + "INFO:root: Converting to a Cloud Optimized GeoTIFF. 1/4\n", + "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_corr.tif\n", + "\n", + "Adding overviews...\n", + "Updating dataset tags...\n", + "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_corr.tif\n", + "INFO:root: Converting to a Cloud Optimized GeoTIFF. 2/4\n", + "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.tif\n", + "\n", + "Adding overviews...\n", + "Updating dataset tags...\n", + "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.tif\n", + "INFO:root: Converting to a Cloud Optimized GeoTIFF. 3/4\n", + "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_amp.tif\n", "\n", "Adding overviews...\n", "Updating dataset tags...\n", - "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_ls_map.tif\n", - "INFO:root: Converting to a Cloud Optimized GeoTIFF. 2/2\n", - "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_VV.tif\n", + "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_amp.tif\n", + "INFO:root: Converting to a Cloud Optimized GeoTIFF. 4/4\n", + "Reading input: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_water_mask.tif\n", "\n", "Adding overviews...\n", "Updating dataset tags...\n", - "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_VV.tif\n" + "Writing output to: /Users/tylere/Documents/GitHub/gee-community/sar-asf-to-gee/temp_downloads/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_water_mask.tif\n" ] } ], @@ -505,51 +529,55 @@ "data": { "application/json": { "browse_images": [ - "https://d3gm2hf49xd6jj.cloudfront.net/71e0ec77-3fee-4dac-87f8-41b6778c24ed/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.png" + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_color_phase.png", + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.png" ], "credit_cost": 1, "expiration_time": "2024-03-28T00:00:00+00:00", "files": [ { "extracted": { - "VV": "S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_VV.tif", - "ls_map": "S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_ls_map.tif" + "amp": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_amp.tif", + "corr": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_corr.tif", + "unw_phase": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.tif", + "water_mask": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_water_mask.tif" }, - "filename": "S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.zip", + "filename": "S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip", "s3": { "bucket": "hyp3-edc-prod-contentbucket-1fv14ed36ifj6", - "key": "71e0ec77-3fee-4dac-87f8-41b6778c24ed/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.zip" + "key": "4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip" }, - "size": 86630727, - "url": "https://d3gm2hf49xd6jj.cloudfront.net/71e0ec77-3fee-4dac-87f8-41b6778c24ed/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9.zip" + "size": 93984120, + "url": "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2.zip" } ], - "job_id": "71e0ec77-3fee-4dac-87f8-41b6778c24ed", + "job_id": "4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b", "job_parameters": { - "dem_matching": false, - "dem_name": "copernicus", + "apply_water_mask": false, "granules": [ - "S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8" + "S1A_IW_SLC__1SDV_20240311T185926_20240311T185953_052938_066872_3CAD", + "S1A_IW_SLC__1SDV_20231206T185929_20231206T185956_051538_0638B3_78A8" ], "include_dem": false, + "include_displacement_maps": false, "include_inc_map": false, - "include_rgb": false, - "include_scattering_area": false, - "radiometry": "gamma0", - "resolution": 30, - "scale": "power", - "speckle_filter": false + "include_look_vectors": false, + "include_los_displacement": false, + "include_wrapped_phase": false, + "looks": "20x4", + "phase_filter_parameter": 0.6 }, - "job_type": "RTC_GAMMA", + "job_type": "INSAR_GAMMA", "logs": [], - "name": "RTC_processing_example", + "name": "INSAR_GAMMA_processing_example", "processing_times": [ - 279.726 + 2425.944 ], - "request_time": "2024-03-13T00:29:49+00:00", + "request_time": "2024-03-13T20:56:36+00:00", "status_code": "SUCCEEDED", "thumbnail_images": [ - "https://d3gm2hf49xd6jj.cloudfront.net/71e0ec77-3fee-4dac-87f8-41b6778c24ed/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_thumb.png" + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_color_phase_thumb.png", + "https://d3gm2hf49xd6jj.cloudfront.net/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase_thumb.png" ], "user_id": "tylere" }, @@ -638,9 +666,13 @@ "text": [ "INFO:root:Starting to_gcs()\n", "INFO:root:GCS file already exists:\n", - " hyp3-data-staging/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_ls_map.tif\n", + " hyp3-data-staging/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_corr.tif\n", "INFO:root:GCS file already exists:\n", - " hyp3-data-staging/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9/S1A_IW_20150621T120220_SVP_RTC30_G_gpuned_B1C9_VV.tif\n" + " hyp3-data-staging/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_unw_phase.tif\n", + "INFO:root:GCS file already exists:\n", + " hyp3-data-staging/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_amp.tif\n", + "INFO:root:GCS file already exists:\n", + " hyp3-data-staging/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2/S1AA_20231206T185929_20240311T185926_VVR096_INT80_G_ueF_39A2_water_mask.tif\n" ] } ], @@ -753,10 +785,36 @@ "output_type": "stream", "text": [ "INFO:root:Starting create_gee_asset()\n", - "INFO:root:Finished creating a GEE asset:\n", - " {assetname}.\n", - "INFO:root:Finished creating a GEE asset:\n", - " {assetname}.\n" + "INFO:root:Unable to create GEE asset. It may already exist.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "e = Cannot overwrite asset 'projects/sar-asf-to-gee/assets/example-INSAR_GAMMA_processing_example/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b_corr'.\n" + ] + }, + { + "ename": "EEException", + "evalue": "Cannot overwrite asset 'projects/sar-asf-to-gee/assets/example-INSAR_GAMMA_processing_example/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b_corr'.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHttpError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/ee/data.py:394\u001b[0m, in \u001b[0;36m_execute_cloud_call\u001b[0;34m(call, num_retries)\u001b[0m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 394\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_retries\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m googleapiclient\u001b[38;5;241m.\u001b[39merrors\u001b[38;5;241m.\u001b[39mHttpError \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/googleapiclient/_helpers.py:130\u001b[0m, in \u001b[0;36mpositional..positional_decorator..positional_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 129\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarning(message)\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/googleapiclient/http.py:938\u001b[0m, in \u001b[0;36mHttpRequest.execute\u001b[0;34m(self, http, num_retries)\u001b[0m\n\u001b[1;32m 937\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m:\n\u001b[0;32m--> 938\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError(resp, content, uri\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muri)\n\u001b[1;32m 939\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostproc(resp, content)\n", + "\u001b[0;31mHttpError\u001b[0m: ", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mEEException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[21], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_gee_asset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[20], line 76\u001b[0m, in \u001b[0;36mcreate_gee_asset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 74\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m(e)\n\u001b[1;32m 75\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 76\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m(e) \u001b[38;5;66;03m# TODO: Add logic to parse the EEException message.\u001b[39;00m\n\u001b[1;32m 77\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGEE asset already exists. Skipping.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "Cell \u001b[0;32mIn[20], line 66\u001b[0m, in \u001b[0;36mcreate_gee_asset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 64\u001b[0m logging\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124massetname = \u001b[39m\u001b[38;5;132;01m{\u001b[39;00massetname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 65\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 66\u001b[0m \u001b[43mee\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreateAsset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 67\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 68\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43massetname\u001b[49m\n\u001b[1;32m 69\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 70\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mFinished creating a GEE asset:\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m \u001b[39m\u001b[38;5;132;01m{\u001b[39;00massetname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ee\u001b[38;5;241m.\u001b[39mEEException \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/ee/_utils.py:38\u001b[0m, in \u001b[0;36maccept_opt_prefix..opt_fixed..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_key \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m kwargs:\n\u001b[1;32m 37\u001b[0m kwargs[new_key] \u001b[38;5;241m=\u001b[39m old_key_val\n\u001b[0;32m---> 38\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/ee/data.py:1512\u001b[0m, in \u001b[0;36mcreateAsset\u001b[0;34m(value, path, properties)\u001b[0m\n\u001b[1;32m 1509\u001b[0m asset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m _cloud_api_utils\u001b[38;5;241m.\u001b[39mconvert_asset_type_for_create_asset(\n\u001b[1;32m 1510\u001b[0m asset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m 1511\u001b[0m parent, asset_id \u001b[38;5;241m=\u001b[39m _cloud_api_utils\u001b[38;5;241m.\u001b[39msplit_asset_name(asset\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[0;32m-> 1512\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_execute_cloud_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1513\u001b[0m \u001b[43m \u001b[49m\u001b[43m_get_cloud_projects\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1514\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43massets\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1515\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1516\u001b[0m \u001b[43m \u001b[49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1517\u001b[0m \u001b[43m \u001b[49m\u001b[43massetId\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43masset_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1518\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43masset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1519\u001b[0m \u001b[43m \u001b[49m\u001b[43mprettyPrint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1520\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1521\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/envs/default/lib/python3.11/site-packages/ee/data.py:396\u001b[0m, in \u001b[0;36m_execute_cloud_call\u001b[0;34m(call, num_retries)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m call\u001b[38;5;241m.\u001b[39mexecute(num_retries\u001b[38;5;241m=\u001b[39mnum_retries)\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m googleapiclient\u001b[38;5;241m.\u001b[39merrors\u001b[38;5;241m.\u001b[39mHttpError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 396\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m _translate_cloud_exception(e)\n", + "\u001b[0;31mEEException\u001b[0m: Cannot overwrite asset 'projects/sar-asf-to-gee/assets/example-INSAR_GAMMA_processing_example/4e1fd265-7a5c-42e4-b68f-4e079f4c0c0b_corr'." ] } ], @@ -770,7 +828,7 @@ "metadata": {}, "source": [ "Code Editor script for visualizing the assets:\n", - "https://code.earthengine.google.com/c738e4671471ee8b1bd82797fac56b9c" + "https://code.earthengine.google.com/0b4c787db9ee4a95bdad22b530d502fd" ] }, { @@ -787,7 +845,7 @@ { "cell_type": "code", "execution_count": null, - "id": "45ce7cf3-a01d-4f73-b64a-ed8ed6fc9455", + "id": "16261c4b-6499-4674-8e51-df0b8bc862a6", "metadata": {}, "outputs": [], "source": [] diff --git a/sar_asf_to_gee/asf_hyp3.py b/sar_asf_to_gee/asf_hyp3.py new file mode 100644 index 0000000..af83a60 --- /dev/null +++ b/sar_asf_to_gee/asf_hyp3.py @@ -0,0 +1,191 @@ +# AUTOGENERATED! DO NOT EDIT! File to edit: ../01_hyp3.ipynb. + +# %% auto 0 +__all__ = ['Transfer'] + +# %% ../01_hyp3.ipynb 2 +import asf_search +import datetime +from IPython.display import JSON +import ee +from fastcore.basics import patch +import gcsfs +from hyp3_sdk import HyP3 +import logging +import os +from pprint import pprint +import re +from rio_cogeo import cogeo +import subprocess +import zipfile + +from sar_asf_to_gee.core import ( + FORMAT_GEE_DATETIME_STRING, + create_gee_image_collection +) + +# %% ../01_hyp3.ipynb 20 +class Transfer(): + def __init__( + self, + job_dict, # HyP3 job dictionary + gcs_bucket, # GCS bucket + gee_gcp_project, # GCP project used by Earth Engine + gee_image_collection=None, # Name of the Earth Engine ImageCollection (optional) + local_storage=None, + ): + self.job_dict = job_dict + self.gcs_bucket = gcs_bucket + self.gee_gcp_project = gee_gcp_project + self.gee_image_collection = gee_image_collection + if local_storage: + self.tempdir = None + self.local_storage = local_storage + else: + self.tempdir = tempfile.TemporaryDirectory() + self.local_storage = self.tempdir.name + logging.debug(f'created temporary directory: {self.tempdir.name}') + +# %% ../01_hyp3.ipynb 24 +@patch +def hpy3_results_to_local( + self:Transfer, +): + "Transfer HyP3 results to local system, unzip, and update the job dictionary." + logging.info(f'Starting hpy3_results_to_local()') + for file in self.job_dict['files']: + logging.info(f'Processing {file["filename"]}') + asf_search.download_url( + url=file['url'], + path=self.local_storage, + filename=file['filename'], + ) + # Unzip the file + logging.info(f' Unzipping the file') + with zipfile.ZipFile(os.path.join(self.local_storage, file['filename']), 'r') as zip_ref: + zip_ref.extractall(self.local_storage) + + # List the TIF files. + scene_name = file['filename'].removesuffix('.zip') + tifs = [x for x in os.listdir( + os.path.join('temp_downloads', scene_name)) + if x.endswith('.tif')] + + for count, tif in enumerate(tifs): + logging.info(f' Converting to a Cloud Optimized GeoTIFF. {count + 1}/{len(tifs)}') + subprocess.run([ + "rio", + "cogeo", + "create", + os.path.join(self.local_storage, scene_name, tif), + os.path.join(self.local_storage, scene_name, tif) + ]) + + tif_dict = {} + pattern = rf'^({scene_name}_(.+).tif)$' + for i in tifs: + groups = re.search(pattern, i).groups() + tif_dict[groups[1]] = os.path.join(scene_name, groups[0]) + + file['extracted'] = tif_dict + +# %% ../01_hyp3.ipynb 30 +@patch +def to_gcs( + self:Transfer, +): + logging.info('Starting to_gcs()') + + fs = gcsfs.GCSFileSystem(token='google_default') + + for file in self.job_dict['files']: + for band, filename in file['extracted'].items(): + gcs_path = f'{self.gcs_bucket}/{filename}' + if fs.exists(gcs_path): + logging.info(f'GCS file already exists:\n {gcs_path}') + else: + logging.info(f'Starting to transfer file to GCS:\n {gcs_path}') + # Transfer the local file to GCS. + fs.put_file( + lpath=f"{self.local_storage}/{filename}", + rpath=gcs_path + ) + logging.info(f'Transferred file to GCS: {gcs_path}') + +# %% ../01_hyp3.ipynb 34 +@patch +def create_gee_asset( + self:Transfer, +): + "Create an Earth Engine asset." + logging.info(f'Starting create_gee_asset()') + + ee.Initialize(project=self.gee_gcp_project) + + create_gee_image_collection(self.gee_gcp_project, self.gee_image_collection) + + granule_names = self.job_dict['job_parameters']['granules'] + granules = asf_search.granule_search(granule_names) + + granule_times = [datetime.datetime.fromisoformat(x.properties['stopTime']) for x in granules] + start_time = min(granule_times) + end_time = max(granule_times) + + id = f"{self.job_dict['job_id']}" + + props = granules[0].properties + description = (f"{props['platform']}" + f" - {props['processingLevel']}" + f" - {props['beamModeType']}") + + for file_dict in self.job_dict['files']: + for band, filename in file_dict['extracted'].items(): + + # Skip non-geocoded (native range-doppler coordinates) TIFFs. + if filename.endswith('_rdr.tif'): + continue + + gcs_path = f'{self.gcs_bucket}/{filename}' + + request = { + 'type': 'IMAGE', + 'bands': { # TODO: Update this once multi-band COG assets are supported + 'id': band + }, + 'gcs_location': { + 'uris': [f'gs://{gcs_path}'] + }, + 'properties': { + 'source': file_dict['url'], + 'band': band # TODO: Remove this once multi-band COG assets are supported + }, + 'startTime': start_time.strftime(FORMAT_GEE_DATETIME_STRING), + 'endTime': end_time.strftime(FORMAT_GEE_DATETIME_STRING), + 'description': description + } + + path_parts = [ + 'projects', + self.gee_gcp_project, + 'assets', + self.gee_image_collection, + # TODO: Remove the band suffix once multi-band COG assets are supported + f'{id}_{band}'.replace(".", "_") + ] + assetname = os.path.join(*[x for x in path_parts if x is not None]) + + logging.debug(f'request = {request}') + logging.debug(f'assetname = {assetname}') + try: + ee.data.createAsset( + value=request, + path=assetname + ) + logging.info(f'Finished creating a GEE asset:\n {assetname}.') + except ee.EEException as e: + print(f'e = {e}') + if "does not exist or doesn't allow this operation" in str(e): + raise(e) + else: + raise(e) # TODO: Add logic to parse the EEException message. + logging.info('GEE asset already exists. Skipping.') diff --git a/sar_asf_to_gee/core.py b/sar_asf_to_gee/core.py index 5475cf5..73dba9f 100644 --- a/sar_asf_to_gee/core.py +++ b/sar_asf_to_gee/core.py @@ -1,12 +1,15 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../00_core.ipynb. # %% auto 0 -__all__ = ['FORMAT_GEE_DATETIME_STRING', 'create_gee_image_collection'] +__all__ = ['FORMAT_GEE_DATETIME_STRING', 'create_gee_image_collection', 'filter_jobs'] # %% ../00_core.ipynb 3 -import ee +import datetime +from dateutil import parser import logging +import ee + # %% ../00_core.ipynb 4 FORMAT_GEE_DATETIME_STRING = '%Y-%m-%dT%H:%M:%SZ' @@ -23,3 +26,35 @@ def create_gee_image_collection(gee_gcp_project, gee_image_collection): logging.info('Unable to create GEE asset. It may already exist.') else: raise(e) + +# %% ../00_core.ipynb 5 +def filter_jobs( + # hyp3_batch, + jobs, + expired=None, + status_code=None, +): + "Filter ASF batch jobs by specified criteria." + # jobs = hyp3_batch.jobs + # Filter by expiration status. + if expired is False: + jobs = [ + job for job in jobs + if parser.parse(job.to_dict()['expiration_time']) > datetime.datetime.now(datetime.timezone.utc) + ] + elif expired is True: + jobs = [ + job for job in jobs + if parser.parse(job.to_dict()['expiration_time']) <= datetime.datetime.now(datetime.timezone.utc) + ] + # Filter by status code. + if isinstance(status_code, str): + print('Status code is a string') + status_code = [status_code] + if isinstance(status_code, list): + print('Status code is a list') + jobs = [ + job for job in jobs + if job.to_dict()['status_code'] in status_code + ] + return jobs