v0.5.1

CHANGELOG.md

@@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.1]
+### Changed
+* Use hyp3-lib v3* to download orbits to be able to distribute load across ESA and ASF. Can be easily swapped out for `sentineleof` in future release.
+
 ## [0.5.0]
 ### Added
 * A `--input-bucket-prefix` argument to `calcDelaysGUNW` which will allow RAiDER to process ARIA GUNW products under one prefix and upload the final products to another prefix provided by the `--bucket-prefix` argument.

environment.yml

@@ -21,6 +21,7 @@ dependencies:
  - ecmwf-api-client
  - h5netcdf
  - h5py
+ - hyp3lib>=3,<4
  - herbie-data
  - isce3>=0.15.0
  - jsonschema==3.2.0  # this is for ASF DAAC ingest schema validation

test/test_GUNW.py

@@ -5,7 +5,6 @@
 import unittest
 from pathlib import Path
 
-import eof.download
 import jsonschema
 import numpy as np
 import pandas as pd
@@ -22,7 +21,7 @@
     check_weather_model_availability,
 )
 from RAiDER.cli.raider import calcDelaysGUNW
-from RAiDER.models.customExceptions import *
+from RAiDER.models.customExceptions import NoWeatherModelData
 
 
 def compute_transform(lats, lons):
@@ -282,7 +281,7 @@ def test_azimuth_timing_interp_against_center_time_interp(weather_model_name: st
                              ])
 
     mocker.patch(
-        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
         side_effect=[
             # For azimuth time
             [Path(orbit_dict_for_azimuth_time_test['reference'])],
@@ -313,11 +312,12 @@ def test_azimuth_timing_interp_against_center_time_interp(weather_model_name: st
     # Calls 4 times for azimuth time and 4 times for center time
     assert RAiDER.processWM.prepareWeatherModel.call_count == 8
     # Only calls once each ref and sec list of slcs
-    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids.call_count == 2
+    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib.call_count == 2
     # Only calls for azimuth timing: once for ref and sec
     assert RAiDER.s1_azimuth_timing.get_slc_id_from_point_and_time.call_count == 2
+    ## When we return to sentineleof
     # Once for center-time and azimuth-time each
-    assert eof.download.download_eofs.call_count == 2
+    # assert eof.download.download_eofs.call_count == 2
 
     for ifg_type in ['reference', 'secondary']:
         for var in ['troposphereHydrostatic', 'troposphereWet']:
@@ -430,7 +430,7 @@ def test_provenance_metadata_for_tropo_group(weather_model_name: str,
              # azimuth-time
             [Path(orbit_dict_for_azimuth_time_test['reference'])],
         ]
-        mocker.patch('eof.download.download_eofs',
+        mocker.patch('RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
                      side_effect=side_effect)
 
         # These outputs are not needed since the orbits are specified above
@@ -444,7 +444,7 @@ def test_provenance_metadata_for_tropo_group(weather_model_name: str,
                                  ])
 
         mocker.patch(
-            'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+            'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
             side_effect=[
                 # For azimuth time
                 [Path(orbit_dict_for_azimuth_time_test['reference'])],
@@ -546,7 +546,7 @@ def test_GUNW_workflow_fails_if_a_download_fails(gunw_azimuth_test, orbit_dict_f
                                 ])
 
     mocker.patch(
-        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
         side_effect=[
             # For azimuth time
             [Path(orbit_dict_for_azimuth_time_test['reference'])],

test/test_s1_time_grid.py

@@ -88,7 +88,7 @@ def test_s1_timing_array_wrt_slc_center_time(gunw_azimuth_test: Path,
 
     # Azimuth time grid
     mocker.patch(
-        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
         side_effect=[
             [Path(orbit_dict_for_azimuth_time_test[ifg_type])],
         ]
@@ -105,7 +105,7 @@ def test_s1_timing_array_wrt_slc_center_time(gunw_azimuth_test: Path,
     assert np.all(abs_diff < 40)
 
     assert RAiDER.s1_azimuth_timing._asf_query.call_count == 1
-    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids.call_count == 1
+    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib.call_count == 1
 
 
 @pytest.mark.parametrize('ifg_type', ['reference', 'secondary'])
@@ -135,7 +135,7 @@ def test_s1_timing_array_wrt_variance(gunw_azimuth_test: Path,
 
     # Azimuth time grid
     mocker.patch(
-        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
         side_effect=[
             [Path(orbit_dict_for_azimuth_time_test[ifg_type])],
         ]
@@ -151,7 +151,7 @@ def test_s1_timing_array_wrt_variance(gunw_azimuth_test: Path,
     assert np.all(std_hgt < 2e-3)
 
     assert RAiDER.s1_azimuth_timing._asf_query.call_count == 1
-    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids.call_count == 1
+    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib.call_count == 1
 
 
 def test_n_closest_dts():
@@ -341,7 +341,7 @@ def test_duplicate_orbits(mocker, orbit_paths_for_duplicate_orbit_xml_test):
                  side_effect=[['slc_id_0', 'slc_id_1', 'slc_id_2', 'slc_id_3']])
 
     mocker.patch(
-        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids',
+        'RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib',
         side_effect=[
             [Path(o_path) for o_path in orbit_paths_for_duplicate_orbit_xml_test],
         ]
@@ -352,7 +352,7 @@ def test_duplicate_orbits(mocker, orbit_paths_for_duplicate_orbit_xml_test):
     assert time_grid.shape == (len(hgt), len(lat), len(lon))
 
     assert RAiDER.s1_azimuth_timing.get_slc_id_from_point_and_time.call_count == 1
-    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids.call_count == 1
+    assert RAiDER.s1_azimuth_timing.get_orbits_from_slc_ids_hyp3lib.call_count == 1
 
 
 def test_get_times_for_az():

tools/RAiDER/aria/prepFromGUNW.py

@@ -22,7 +22,7 @@
 from RAiDER.models import credentials
 from RAiDER.models.hrrr import HRRR_CONUS_COVERAGE_POLYGON, AK_GEO, check_hrrr_dataset_availability
 from RAiDER.s1_azimuth_timing import get_times_for_azimuth_interpolation
-from RAiDER.s1_orbits import download_eofs
+from RAiDER.s1_orbits import get_orbits_from_slc_ids_hyp3lib
 
 ## cube spacing in degrees for each model
 DCT_POSTING = {'HRRR': 0.05, 'HRES': 0.10, 'GMAO': 0.10, 'ERA5': 0.10, 'ERA5T': 0.10, 'MERRA2': 0.1}
@@ -262,20 +262,12 @@ def get_orbit_file(self):
 
         ds   = xr.open_dataset(self.path_gunw, group=f'{group}')
         slcs = ds['L1InputGranules']
-        nslcs = slcs.count().item()
+        # Convert to list of strings
+        slcs_lst = [slc for slc in slcs.data.tolist() if slc]
+        # Remove .zip from the granule ids included in this field
+        slcs_lst = list(map(lambda slc: slc.replace('.zip', ''), slcs_lst))
 
-        if nslcs == 1:
-            slc    = slcs.item()
-        else:
-            for j in range(nslcs):
-                slc = slcs.data[j]
-                if slc:
-                    break
-
-        sat = slc.split('_')[0]
-        dt  = datetime.strptime(f'{self.dates[0]}T{self.mid_time}', '%Y%m%dT%H:%M:%S')
-
-        path_orb = download_eofs([dt], [sat], str(orbit_dir))
+        path_orb = get_orbits_from_slc_ids_hyp3lib(slcs_lst)
 
         return [str(o) for o in path_orb]
 
@@ -352,9 +344,6 @@ def make_cube(self):
         lats = np.arange(lat_st, lat_en, DCT_POSTING[self.wmodel])
         lons = np.arange(lon_st, lon_en, DCT_POSTING[self.wmodel])
 
-        S, N = lats.min(), lats.max()
-        W, E = lons.min(), lons.max()
-
         ds = xr.Dataset(coords={'latitude': lats, 'longitude': lons, 'heights': self.heights})
         dst_cube = os.path.join(self.out_dir, f'GeoCube_{self.name}.nc')
         ds.to_netcdf(dst_cube)

tools/RAiDER/s1_azimuth_timing.py

@@ -12,7 +12,7 @@
     isce = None
 
 from RAiDER.losreader import get_orbit as get_isce_orbit
-from RAiDER.s1_orbits import get_orbits_from_slc_ids
+from RAiDER.s1_orbits import get_orbits_from_slc_ids_hyp3lib
 
 
 def _asf_query(point: Point,
@@ -89,7 +89,7 @@ def get_azimuth_time_grid(lon_mesh: np.ndarray,
     Technically, this is "sensor neutral" since it uses an orb object.
     '''
     if isce is None:
-        raise ImportError(f'isce3 is required for this function. Use conda to install isce3`')
+        raise ImportError('isce3 is required for this function. Use conda to install isce3`')
 
     num_iteration = 100
     residual_threshold = 1.0e-7
@@ -183,7 +183,7 @@ def get_s1_azimuth_time_grid(lon: np.ndarray,
                          dtype='datetime64[ms]')
         return az_arr
 
-    orb_files = get_orbits_from_slc_ids(slc_ids)
+    orb_files = get_orbits_from_slc_ids_hyp3lib(slc_ids)
     orb_files = [str(of) for of in orb_files]
 
     orb = get_isce_orbit(orb_files, dt, pad=600)

tools/RAiDER/s1_orbits.py

@@ -6,6 +6,7 @@
 from typing import List, Optional
 
 import eof.download
+from hyp3lib import get_orb
 from RAiDER.logger import logger
 
 
@@ -53,7 +54,7 @@ def ensure_orbit_credentials() -> Optional[int]:
         username = os.environ.get('EARTHDATA_USERNAME')
         password = os.environ.get('EARTHDATA_PASSWORD')
         if username is None or password is None:
-            raise ValueError(f'Credentials are required for fetching orbit data from s1qc.asf.alaska.edu!\n'
+            raise ValueError('Credentials are required for fetching orbit data from s1qc.asf.alaska.edu!\n'
                              'Either add your credentials to ~/.netrc or set the EARTHDATA_USERNAME and'
                              ' EARTHDATA_PASSWORD environment variables.')
 
@@ -78,6 +79,32 @@ def get_orbits_from_slc_ids(slc_ids: List[str], directory=Path.cwd()) -> List[Pa
     return orb_files
 
 
+def get_orbits_from_slc_ids_hyp3lib(
+    slc_ids: list, orbit_directory: str = None
+) -> dict:
+    """Reference: https://github.com/ACCESS-Cloud-Based-InSAR/DockerizedTopsApp/blob/dev/isce2_topsapp/localize_orbits.py#L23"""
+
+    # Populates env variables to netrc as required for sentineleof
+    _ = ensure_orbit_credentials()
+    esa_username, _, esa_password = netrc.netrc().authenticators(ESA_CDSE_HOST)
+    esa_credentials = esa_username, esa_password
+
+    orbit_directory = orbit_directory or 'orbits'
+    orbit_dir = Path(orbit_directory)
+    orbit_dir.mkdir(exist_ok=True)
+
+    orbit_fetcher = get_orb.downloadSentinelOrbitFile
+
+    orbits = []
+    for scene in slc_ids:
+        orbit_file, _ = orbit_fetcher(scene, str(orbit_dir), esa_credentials=esa_credentials, providers=('ASF', 'ESA'))
+        orbits.append(orbit_file)
+
+    orbits = sorted(list(set(orbits)))
+
+    return orbits
+
+
 def download_eofs(dts: list, missions: list, save_dir: str):
     """Wrapper around sentineleof to first try downloading from ASF and fall back to CDSE"""
     _ = ensure_orbit_credentials()
@@ -90,7 +117,7 @@ def download_eofs(dts: list, missions: list, save_dir: str):
         try:
             orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=True)
         except:
-            logger.error(f'Could not download orbit from ASF, trying ESA...')
+            logger.error('Could not download orbit from ASF, trying ESA...')
             orb_file = eof.download.download_eofs(dt, mission, save_dir=save_dir, force_asf=False)
 
         orb_file = orb_file[0] if isinstance(orb_file, list) else orb_file