diff --git a/01_hyp3.ipynb b/01_hyp3.ipynb index a4a5a1d..21324cb 100644 --- a/01_hyp3.ipynb +++ b/01_hyp3.ipynb @@ -56,8 +56,9 @@ "source": [ "# Prerequisites\n", "\n", - "Authenication for NASA Earthdata\n", + "Authentication for NASA Earthdata\n", "- See: https://nasa-openscapes.github.io/2021-Cloud-Hackathon/tutorials/04_NASA_Earthdata_Authentication.html\n", + "- See: https://urs.earthdata.nasa.gov/documentation/for_users/data_access/create_net_rc_file\n", "\n", "Authenticate to Google Cloud\n", "- See: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/loginca\n", @@ -514,30 +515,18 @@ "name": "stderr", "output_type": "stream", "text": [ - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_corr.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_corr.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_los_disp.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_los_disp.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_vert_disp.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_vert_disp.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_water_mask.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_water_mask.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_phi.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_phi.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map_ell.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map_ell.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_theta.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_theta.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_unw_phase.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_unw_phase.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_amp.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_amp.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_wrapped_phase.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_wrapped_phase.tif\n", - "INFO:root:Starting to transfer file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_dem.tif\n", - "INFO:root:Transferred file to GCS: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_dem.tif\n" + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_corr.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_los_disp.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_vert_disp.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_water_mask.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_phi.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map_ell.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_theta.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_unw_phase.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_amp.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_wrapped_phase.tif\n", + "INFO:root:GCS file already exists: hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_dem.tif\n" ] } ], @@ -642,172 +631,40 @@ "id": "c2b3c84f-0c22-43b4-81d6-352cbcf2df10", "metadata": {}, "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_corr.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_los_disp.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_vert_disp.tif\n" - ] - }, { "name": "stderr", "output_type": "stream", "text": [ - "INFO:root:Finished creating a GEE asset.\n" + "INFO:root:Unable to create GEE asset. It may already exist.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_water_mask.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_phi.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_inc_map_ell.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_lv_theta.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_unw_phase.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_amp.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_wrapped_phase.tif\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_dem.tif\n" + "hyp3-data-staging/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75/S1AA_20230331T140803_20230412T140803_VVP012_INT80_G_ueF_3F75_corr.tif\n" ] }, { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:root:Finished creating a GEE asset.\n" + "ename": "EEException", + "evalue": "Cannot overwrite asset 'projects/sar-asf-to-gee/assets/test_ic/008a6a57-3711-4fc9-b873-21b6cc8780a7_corr'.", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mHttpError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/ee/data.py:394\u001b[0m, in \u001b[0;36m_execute_cloud_call\u001b[0;34m(call, num_retries)\u001b[0m\n\u001b[1;32m 393\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 394\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mcall\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnum_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnum_retries\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m googleapiclient\u001b[38;5;241m.\u001b[39merrors\u001b[38;5;241m.\u001b[39mHttpError \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/googleapiclient/_helpers.py:130\u001b[0m, in \u001b[0;36mpositional..positional_decorator..positional_wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 129\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarning(message)\n\u001b[0;32m--> 130\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mwrapped\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/googleapiclient/http.py:938\u001b[0m, in \u001b[0;36mHttpRequest.execute\u001b[0;34m(self, http, num_retries)\u001b[0m\n\u001b[1;32m 937\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m resp\u001b[38;5;241m.\u001b[39mstatus \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m300\u001b[39m:\n\u001b[0;32m--> 938\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HttpError(resp, content, uri\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39muri)\n\u001b[1;32m 939\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpostproc(resp, content)\n", + "\u001b[0;31mHttpError\u001b[0m: ", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mEEException\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[13], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mt\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_gee_asset\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", + "Cell \u001b[0;32mIn[12], line 72\u001b[0m, in \u001b[0;36mcreate_gee_asset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m(e)\n\u001b[1;32m 71\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m---> 72\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m(e) \u001b[38;5;66;03m# TODO: Add logic to parse the EEException message.\u001b[39;00m\n\u001b[1;32m 73\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGEE asset already exists. Skipping.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n", + "Cell \u001b[0;32mIn[12], line 63\u001b[0m, in \u001b[0;36mcreate_gee_asset\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 60\u001b[0m assetname \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;241m*\u001b[39m[x \u001b[38;5;28;01mfor\u001b[39;00m x \u001b[38;5;129;01min\u001b[39;00m path_parts \u001b[38;5;28;01mif\u001b[39;00m x \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m])\n\u001b[1;32m 62\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m---> 63\u001b[0m \u001b[43mee\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreateAsset\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 64\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalue\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mrequest\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 65\u001b[0m \u001b[43m \u001b[49m\u001b[43mpath\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43massetname\u001b[49m\n\u001b[1;32m 66\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m \n\u001b[1;32m 67\u001b[0m logging\u001b[38;5;241m.\u001b[39minfo(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mFinished creating a GEE asset.\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m 68\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m ee\u001b[38;5;241m.\u001b[39mEEException \u001b[38;5;28;01mas\u001b[39;00m e:\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/ee/_utils.py:38\u001b[0m, in \u001b[0;36maccept_opt_prefix..opt_fixed..wrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m 36\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m new_key \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;129;01min\u001b[39;00m kwargs:\n\u001b[1;32m 37\u001b[0m kwargs[new_key] \u001b[38;5;241m=\u001b[39m old_key_val\n\u001b[0;32m---> 38\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfunc\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/ee/data.py:1512\u001b[0m, in \u001b[0;36mcreateAsset\u001b[0;34m(value, path, properties)\u001b[0m\n\u001b[1;32m 1509\u001b[0m asset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m] \u001b[38;5;241m=\u001b[39m _cloud_api_utils\u001b[38;5;241m.\u001b[39mconvert_asset_type_for_create_asset(\n\u001b[1;32m 1510\u001b[0m asset[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mtype\u001b[39m\u001b[38;5;124m'\u001b[39m])\n\u001b[1;32m 1511\u001b[0m parent, asset_id \u001b[38;5;241m=\u001b[39m _cloud_api_utils\u001b[38;5;241m.\u001b[39msplit_asset_name(asset\u001b[38;5;241m.\u001b[39mpop(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m'\u001b[39m))\n\u001b[0;32m-> 1512\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43m_execute_cloud_call\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1513\u001b[0m \u001b[43m \u001b[49m\u001b[43m_get_cloud_projects\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1514\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43massets\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1515\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1516\u001b[0m \u001b[43m \u001b[49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1517\u001b[0m \u001b[43m \u001b[49m\u001b[43massetId\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43masset_id\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1518\u001b[0m \u001b[43m \u001b[49m\u001b[43mbody\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43masset\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1519\u001b[0m \u001b[43m \u001b[49m\u001b[43mprettyPrint\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1520\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1521\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/Documents/GitHub/gee-community/sar-asf-to-gee/.pixi/env/lib/python3.11/site-packages/ee/data.py:396\u001b[0m, in \u001b[0;36m_execute_cloud_call\u001b[0;34m(call, num_retries)\u001b[0m\n\u001b[1;32m 394\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m call\u001b[38;5;241m.\u001b[39mexecute(num_retries\u001b[38;5;241m=\u001b[39mnum_retries)\n\u001b[1;32m 395\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m googleapiclient\u001b[38;5;241m.\u001b[39merrors\u001b[38;5;241m.\u001b[39mHttpError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[0;32m--> 396\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m _translate_cloud_exception(e)\n", + "\u001b[0;31mEEException\u001b[0m: Cannot overwrite asset 'projects/sar-asf-to-gee/assets/test_ic/008a6a57-3711-4fc9-b873-21b6cc8780a7_corr'." ] } ], @@ -830,6 +687,17 @@ "id": "2229cc5c-4390-4ff6-96f5-9c6f2b3eadc1", "metadata": {}, "outputs": [], + "source": [ + "#| hide\n", + "import nbdev; nbdev.nbdev_export()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "45ce7cf3-a01d-4f73-b64a-ed8ed6fc9455", + "metadata": {}, + "outputs": [], "source": [] } ], diff --git a/README.md b/README.md index 510a85f..f4ea0fd 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,31 @@ -## Install +# Overview + +This package facilitates the transfer of ASF HyP3 processed SAR datasets +to Google Cloud Storage and the creation of Google Earth Engine assets. + +# Install + +## Prerequisites + +Authentication for NASA Earthdata - See: +https://nasa-openscapes.github.io/2021-Cloud-Hackathon/tutorials/04_NASA_Earthdata_Authentication.html - +See: +https://urs.earthdata.nasa.gov/documentation/for_users/data_access/create_net_rc_file + +Authenticate to Google Cloud - See: +https://cloud.google.com/sdk/gcloud/reference/auth/application-default/loginca - +`gcloud auth application-default login` + +Authenticate to Google Earth Engine - See: +https://developers.google.com/earth-engine/guides/auth - +`earthengine authenticate` + +## Package Installation + +Install directly from the GitHub repo. ``` sh pip install git+https://github.com/gee-community/sar-asf-to-gee.git @@ -10,24 +34,21 @@ pip install git+https://github.com/gee-community/sar-asf-to-gee.git ## How to use -### Setup Authentication Credentials - -- NASA Earthdata - (https://urs.earthdata.nasa.gov/documentation/for_users/data_access/create_net_rc_file) -- Earth Engine (https://developers.google.com/earth-engine/guides/auth) - ``` python from hyp3_sdk import HyP3 from sar_asf_to_gee.hyp3 import Transfer ``` -Define a job name for HyP3 generated files that previously submitted… +Define a job name for HyP3 generated files that previously submitted via +the [ASF Data Search +Vertex](https://hyp3-docs.asf.alaska.edu/using/vertex/) web application +or the [HyP3 Python SDK](https://hyp3-docs.asf.alaska.edu/using/sdk/). ``` python job_name = 'test submit_insar_job' ``` -Find jobs that were completed by HyP3. +Find HyP3 jobs that are completed. ``` python hyp3 = HyP3() @@ -37,6 +58,9 @@ batch_completed = hyp3.find_jobs( ) ``` +Loop through the jobs, transferring the results locally, then to Google +Cloud Storage, then create an Earth Engine asset. + ``` python for job in batch_completed: t = Transfer( diff --git a/index.ipynb b/index.ipynb index a4267ba..24d2410 100644 --- a/index.ipynb +++ b/index.ipynb @@ -13,33 +13,60 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Install" + "# Overview\n", + "\n", + "This package facilitates the transfer of ASF HyP3 processed SAR datasets to Google Cloud Storage and the creation of Google Earth Engine assets." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "```sh\n", - "pip install git+https://github.com/gee-community/sar-asf-to-gee.git\n", - "```" + "# Install" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## How to use" + "## Prerequisites\n", + "\n", + "Authentication for NASA Earthdata\n", + "- See: https://nasa-openscapes.github.io/2021-Cloud-Hackathon/tutorials/04_NASA_Earthdata_Authentication.html\n", + "- See: https://urs.earthdata.nasa.gov/documentation/for_users/data_access/create_net_rc_file\n", + "\n", + "Authenticate to Google Cloud\n", + "- See: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/loginca\n", + "- `gcloud auth application-default login`\n", + "\n", + "Authenticate to Google Earth Engine\n", + "- See: https://developers.google.com/earth-engine/guides/auth\n", + "- `earthengine authenticate`" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Setup Authentication Credentials\n", + "## Package Installation\n", "\n", - "- NASA Earthdata (https://urs.earthdata.nasa.gov/documentation/for_users/data_access/create_net_rc_file)\n", - "- Earth Engine (https://developers.google.com/earth-engine/guides/auth)" + "Install directly from the GitHub repo." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "```sh\n", + "pip install git+https://github.com/gee-community/sar-asf-to-gee.git\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How to use" ] }, { @@ -56,7 +83,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Define a job name for HyP3 generated files that previously submitted... " + "Define a job name for HyP3 generated files that previously submitted via the [ASF Data Search Vertex](https://hyp3-docs.asf.alaska.edu/using/vertex/) web application or the [HyP3 Python SDK](https://hyp3-docs.asf.alaska.edu/using/sdk/). " ] }, { @@ -72,7 +99,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Find jobs that were completed by HyP3." + "Find HyP3 jobs that are completed." ] }, { @@ -88,6 +115,13 @@ ")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Loop through the jobs, transferring the results locally, then to Google Cloud Storage, then create an Earth Engine asset." + ] + }, { "cell_type": "code", "execution_count": null,