Merge pull request #641 from dbekaert/dev

Release v0.5.0 -- orbits and stand-alone HyP3 job support
dbekaert · Apr 29, 2024 · c1ef3f1 · c1ef3f1
2 parents 85a26cd + 42d77d3
commit c1ef3f1
Show file tree

Hide file tree

Showing 15 changed files with 366 additions and 329 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.0]
+### Added
+* A `--input-bucket-prefix` argument to `calcDelaysGUNW` which will allow RAiDER to process ARIA GUNW products under one prefix and upload the final products to another prefix provided by the `--bucket-prefix` argument.
+### Fixed
+* [613](https://github.com/dbekaert/RAiDER/issues/613) - ensure NASA Earthdata credentials for downloading orbits from ASF
+* [634](https://github.com/dbekaert/RAiDER/issues/634) - download orbits from ASF before trying ESA
+* [630](https://github.com/dbekaert/RAiDER/pull/630) - use correct model name so (hrrr-ak) in azimuth_timing_grid
+* [620](https://github.com/dbekaert/RAiDER/issues/620) - Fix MERRA-2 access because API was updated
+
 
 ## [0.4.7]
 ### Fixed

diff --git a/test/__init__.py b/test/__init__.py
@@ -16,7 +16,7 @@
 WM_DIR   = os.path.join(TEST_DIR, 'weather_files')
 ORB_DIR  = os.path.join(TEST_DIR, 'orbit_files')
 
-WM = 'GMAO'
+WM = 'MERRA2'
 
 @contextmanager
 def pushd(dir):

diff --git a/test/test_GUNW.py b/test/test_GUNW.py
@@ -100,7 +100,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
 
     # We only need to make sure the json file is passed, the netcdf file name will not have
     # any impact on subsequent testing
-    mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path])
+    mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path, 'foo.png'])
     mocker.patch("RAiDER.aws.upload_file_to_s3")
     mocker.patch("RAiDER.aria.prepFromGUNW.main", return_value=['my_path_cfg', 'my_wavelength'])
     mocker.patch('RAiDER.aria.prepFromGUNW.check_hrrr_dataset_availablity_for_s1_azimuth_time_interpolation',
@@ -111,7 +111,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
 
     iargs = ['--weather-model', 'HRES',
              '--bucket', 'myBucket',
-             '--bucket-prefix', 'myPrefix']
+             '--bucket-prefix', 'myPrefix',]
     calcDelaysGUNW(iargs)
 
     metadata = json.loads(temp_json_path.read_text())
@@ -123,6 +123,7 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
     assert aws.get_s3_file.mock_calls == [
         unittest.mock.call('myBucket', 'myPrefix', '.nc'),
         unittest.mock.call('myBucket', 'myPrefix', '.json'),
+        unittest.mock.call('myBucket', 'myPrefix', '.png'),
     ]
 
     RAiDER.aria.prepFromGUNW.main.assert_called_once()
@@ -138,6 +139,60 @@ def test_GUNW_hyp3_metadata_update(test_gunw_json_path, test_gunw_json_schema_pa
     assert aws.upload_file_to_s3.mock_calls == [
         unittest.mock.call('foo.nc', 'myBucket', 'myPrefix'),
         unittest.mock.call(temp_json_path, 'myBucket', 'myPrefix'),
+        unittest.mock.call('foo.png', 'myBucket', 'myPrefix'),
+    ]
+
+
+def test_GUNW_hyp3_metadata_update_different_prefix(test_gunw_json_path, test_gunw_json_schema_path, tmp_path, mocker):
+    """This test performs the GUNW entrypoint using a different input bucket/prefix than the output bucket/prefix.
+    Only updates the json. Monkey patches the upload/download to/from s3 and the actual computation.
+    """
+    temp_json_path = tmp_path / 'temp.json'
+    shutil.copy(test_gunw_json_path, temp_json_path)
+
+    # We only need to make sure the json file is passed, the netcdf file name will not have
+    # any impact on subsequent testing
+    mocker.patch("RAiDER.aws.get_s3_file", side_effect=['foo.nc', temp_json_path, 'foo.png'])
+    mocker.patch("RAiDER.aws.upload_file_to_s3")
+    mocker.patch("RAiDER.aria.prepFromGUNW.main", return_value=['my_path_cfg', 'my_wavelength'])
+    mocker.patch('RAiDER.aria.prepFromGUNW.check_hrrr_dataset_availablity_for_s1_azimuth_time_interpolation',
+                 side_effect=[True])
+    mocker.patch("RAiDER.aria.prepFromGUNW.check_weather_model_availability", return_value=True)
+    mocker.patch("RAiDER.cli.raider.calcDelays", return_value=['file1', 'file2'])
+    mocker.patch("RAiDER.aria.calcGUNW.tropo_gunw_slc")
+
+    iargs = ['--weather-model', 'HRES',
+             '--bucket', 'myBucket',
+             '--bucket-prefix', 'myOutputPrefix',
+             '--input-bucket-prefix', 'myInputPrefix',]
+    calcDelaysGUNW(iargs)
+
+    metadata = json.loads(temp_json_path.read_text())
+    schema = json.loads(test_gunw_json_schema_path.read_text())
+
+    assert metadata['metadata']['weather_model'] == ['HRES']
+    assert (jsonschema.validate(instance=metadata, schema=schema) is None)
+
+    assert aws.get_s3_file.mock_calls == [
+        unittest.mock.call('myBucket', 'myInputPrefix', '.nc'),
+        unittest.mock.call('myBucket', 'myInputPrefix', '.json'),
+        unittest.mock.call('myBucket', 'myInputPrefix', '.png'),
+    ]
+
+    RAiDER.aria.prepFromGUNW.main.assert_called_once()
+
+    raider.calcDelays.assert_called_once_with(['my_path_cfg'])
+
+    RAiDER.aria.calcGUNW.tropo_gunw_slc.assert_called_once_with(
+        ['file1', 'file2'],
+        'foo.nc',
+        'my_wavelength',
+    )
+
+    assert aws.upload_file_to_s3.mock_calls == [
+        unittest.mock.call('foo.nc', 'myBucket', 'myOutputPrefix'),
+        unittest.mock.call(temp_json_path, 'myBucket', 'myOutputPrefix'),
+        unittest.mock.call('foo.png', 'myBucket', 'myOutputPrefix'),
     ]
 
 
@@ -277,7 +332,7 @@ def test_azimuth_timing_interp_against_center_time_interp(weather_model_name: st
             assert np.nanmax(abs_diff_mm) < 1
 
 
-@pytest.mark.parametrize('weather_model_name', ['GMAO', 'HRRR', 'HRES', 'ERA5', 'ERA5'])
+@pytest.mark.parametrize('weather_model_name', ['HRRR', 'HRES', 'ERA5', 'ERA5T'])
 def test_check_weather_model_availability(test_gunw_path_factory, weather_model_name, mocker):
     # Should be True for all weather models
     # S1-GUNW-D-R-071-tops-20200130_20200124-135156-34956N_32979N-PP-913f-v2_0_4.nc
@@ -288,12 +343,12 @@ def test_check_weather_model_availability(test_gunw_path_factory, weather_model_
     mocker.patch("RAiDER.aria.prepFromGUNW.get_acq_time_from_slc_id", side_effect=[pd.Timestamp('2015-01-01'),
                                                                                    pd.Timestamp('2014-01-01')])
     cond = check_weather_model_availability(test_gunw_path, weather_model_name)
-    if weather_model_name in ['HRRR', 'GMAO']:
+    if weather_model_name in ['HRRR', 'MERRA2']:
         cond = not cond
     assert cond
 
 
-@pytest.mark.parametrize('weather_model_name', ['GMAO', 'HRRR'])
+@pytest.mark.parametrize('weather_model_name', ['HRRR'])
 def test_check_weather_model_availability_over_alaska(test_gunw_path_factory, weather_model_name, mocker):
     # Should be True for all weather models
     # S1-GUNW-D-R-059-tops-20230320_20220418-180300-00179W_00051N-PP-c92e-v2_0_6.nc
@@ -309,7 +364,7 @@ def test_check_weather_model_availability_over_alaska(test_gunw_path_factory, we
     assert cond
 
 
-@pytest.mark.parametrize('weather_model_name', ['HRRR', 'GMAO'])
+@pytest.mark.parametrize('weather_model_name', ['HRRR'])
 @pytest.mark.parametrize('location', ['california-t71', 'alaska'])
 def test_weather_model_availability_integration_using_valid_range(location,
                                                                   test_gunw_path_factory,

diff --git a/test/test_s1_orbits.py b/test/test_s1_orbits.py
@@ -12,101 +12,186 @@ class EmptyNetrc():
         def __init__(self, netrc_file):
             self.netrc_file = netrc_file
             self.hosts = {}
+
         def __str__(self):
             return str(self.hosts)
 
-    # No .netrc, no ESA CDSE env variables
+    # No .netrc, no ESA CDSE or Earthdata env variables
+    with monkeypatch.context() as mp:
+        mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
+        mp.delenv('ESA_USERNAME', raising=False)
+        mp.delenv('ESA_PASSWORD', raising=False)
+        mp.delenv('EARTHDATA_USERNAME', raising=False)
+        mp.delenv('EARTHDATA_PASSWORD', raising=False)
+        with pytest.raises(ValueError):
+            s1_orbits.ensure_orbit_credentials()
+
+    # No .netrc or Earthdata env vars, set ESA CDSE env variables
+    with monkeypatch.context() as mp:
+        mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
+        mp.setenv('ESA_USERNAME', 'foo')
+        mp.setenv('ESA_PASSWORD', 'bar')
+        mp.delenv('EARTHDATA_USERNAME', raising=False)
+        mp.delenv('EARTHDATA_PASSWORD', raising=False)
+        with pytest.raises(ValueError):
+            s1_orbits.ensure_orbit_credentials()
+
+    # No .netrc or ESA CDSE env vars, set Earthdata env variables
     with monkeypatch.context() as mp:
         mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
         mp.delenv('ESA_USERNAME', raising=False)
         mp.delenv('ESA_PASSWORD', raising=False)
+        mp.setenv('EARTHDATA_USERNAME', 'fizz')
+        mp.setenv('EARTHDATA_PASSWORD', 'buzz')
         with pytest.raises(ValueError):
             s1_orbits.ensure_orbit_credentials()
 
-    # No .netrc, set ESA CDSE env variables
+    # No .netrc, set Earthdata and ESA CDSE env variables
     with monkeypatch.context() as mp:
         mp.setattr(netrc, 'netrc', EmptyNetrc, raising=False)
         mp.setenv('ESA_USERNAME', 'foo')
         mp.setenv('ESA_PASSWORD', 'bar')
+        mp.setenv('EARTHDATA_USERNAME', 'fizz')
+        mp.setenv('EARTHDATA_PASSWORD', 'buzz')
         mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
         written_credentials = s1_orbits.ensure_orbit_credentials()
-        assert written_credentials == str({s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')})
+        assert written_credentials == str({
+            s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
+            s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')
+        })
 
     class NoCDSENetrc():
         def __init__(self, netrc_file):
             self.netrc_file = netrc_file
-            self.hosts = {'fizz.buzz.org': ('foo', None, 'bar')}
+            self.hosts = {s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')}
+
         def __str__(self):
             return str(self.hosts)
 
-    # No CDSE in .netrc, no ESA CDSE env variables
+    # No CDSE in .netrc or ESA CDSE env variables, Earthdata in .netrc
     with monkeypatch.context() as mp:
         mp.setattr(netrc, 'netrc', NoCDSENetrc, raising=False)
         mp.delenv('ESA_USERNAME', raising=False)
         mp.delenv('ESA_PASSWORD', raising=False)
         with pytest.raises(ValueError):
             s1_orbits.ensure_orbit_credentials()
 
-    # No CDSE in .netrc, set ESA CDSE env variables
+    # No CDSE in .netrc, set ESA CDSE env variables, Earthdata in .netrc
     with monkeypatch.context() as mp:
         mp.setattr(netrc, 'netrc', NoCDSENetrc, raising=False)
         mp.setenv('ESA_USERNAME', 'foo')
         mp.setenv('ESA_PASSWORD', 'bar')
         mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
         written_credentials = s1_orbits.ensure_orbit_credentials()
-        assert written_credentials == str({'fizz.buzz.org': ('foo', None, 'bar'), s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')})
+        assert written_credentials == str({
+            s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz'),
+            s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
+        })
 
-    class CDSENetrc():
+    class NoEarthdataNetrc():
         def __init__(self, netrc_file):
             self.netrc_file = netrc_file
             self.hosts = {s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar')}
+
         def __str__(self):
             return str(self.hosts)
 
-    # cdse in .netrc, no ESA CDSE env variables
+    # cdse in .netrc, no ESA CDSE env variables, Earthdata env variables
     with monkeypatch.context() as mp:
-        mp.setattr(netrc, 'netrc', CDSENetrc, raising=False)
+        mp.setattr(netrc, 'netrc', NoEarthdataNetrc, raising=False)
         mp.delenv('ESA_USERNAME', raising=False)
         mp.delenv('ESA_PASSWORD', raising=False)
+        mp.setenv('EARTHDATA_USERNAME', 'fizz')
+        mp.setenv('EARTHDATA_PASSWORD', 'buzz')
+        mp.setattr(Path, 'write_text', lambda self, write_text: write_text)
+        written_credentials = s1_orbits.ensure_orbit_credentials()
+        assert written_credentials == str({
+            s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
+            s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz'),
+        })
+
+    class CDSEAndEarthdataNetrc():
+        def __init__(self, netrc_file):
+            self.netrc_file = netrc_file
+            self.hosts = {
+                s1_orbits.ESA_CDSE_HOST: ('foo', None, 'bar'),
+                s1_orbits.NASA_EDL_HOST: ('fizz', None, 'buzz')
+            }
+
+        def __str__(self):
+            return str(self.hosts)
+
+    # cdse and Earthdata in netrc, no env variables
+    with monkeypatch.context() as mp:
+        mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
+        written_credentials = s1_orbits.ensure_orbit_credentials()
+        assert written_credentials is None
+
+    with monkeypatch.context() as mp:
+        mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
+        mp.delenv('ESA_USERNAME', raising=False)
+        mp.delenv('ESA_PASSWORD', raising=False)
+        mp.setenv('EARTHDATA_USERNAME', 'fizz')
+        mp.setenv('EARTHDATA_PASSWORD', 'buzz')
         written_credentials = s1_orbits.ensure_orbit_credentials()
         assert written_credentials is None
 
-    # cdse in .netrc, set ESA CDSE env variables
     with monkeypatch.context() as mp:
-        mp.setattr(netrc, 'netrc', CDSENetrc, raising=False)
+        mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
         mp.setenv('ESA_USERNAME', 'foo')
         mp.setenv('ESA_PASSWORD', 'bar')
+        mp.setenv('EARTHDATA_USERNAME', 'fizz')
+        mp.setenv('EARTHDATA_PASSWORD', 'buzz')
+        written_credentials = s1_orbits.ensure_orbit_credentials()
+        assert written_credentials is None
+
+    with monkeypatch.context() as mp:
+        mp.setattr(netrc, 'netrc', CDSEAndEarthdataNetrc, raising=False)
+        mp.setenv('ESA_USERNAME', 'foo')
+        mp.setenv('ESA_PASSWORD', 'bar')
+        mp.delenv('ESA_USERNAME', raising=False)
+        mp.delenv('ESA_PASSWORD', raising=False)
         written_credentials = s1_orbits.ensure_orbit_credentials()
         assert written_credentials is None
 
 
 def test_get_orbits_from_slc_ids(mocker):
     side_effect = [
-        [Path('foo.txt')],
-        [Path('bar.txt'), Path('fiz.txt')],
+        [Path('foo_start.txt'), Path('foo_stop.txt')],
+        [Path('bar_start.txt'), Path('bar_end.txt'),
+         Path('fiz_start.txt'), Path('fiz_end')],
     ]
     mocker.patch('eof.download.download_eofs',
-                 side_effect=side_effect)
+                 side_effect=side_effect[0])
 
     orbit_files = s1_orbits.get_orbits_from_slc_ids(
         ['S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8']
     )
-    assert orbit_files == [Path('foo.txt')]
-    assert eof.download.download_eofs.call_count == 1
-    eof.download.download_eofs.assert_called_with(
-        [ '20150621T120220', '20150621T120232'],
-        ['S1A'] * 2,
-        save_dir=str(Path.cwd())
-    )
+    assert orbit_files == side_effect[0]
+    assert eof.download.download_eofs.call_count == 2
+    for dt in '20150621T120220 20150621T120232'.split():
+        eof.download.download_eofs.assert_any_call(
+            [dt],
+            ['S1A'],
+            save_dir=str(Path.cwd()),
+            force_asf=True
+        )
+
+    mocker.patch('eof.download.download_eofs',
+                 side_effect=side_effect[1])
 
     orbit_files = s1_orbits.get_orbits_from_slc_ids(
         ['S1B_IW_SLC__1SDV_20201115T162313_20201115T162340_024278_02E29D_5C54',
          'S1A_IW_SLC__1SDV_20201203T162353_20201203T162420_035524_042744_6D5C']
     )
-    assert orbit_files == [Path('bar.txt'), Path('fiz.txt')]
-    assert eof.download.download_eofs.call_count == 2
-    eof.download.download_eofs.assert_called_with(
-        ['20201115T162313', '20201203T162353', '20201115T162340', '20201203T162420'],
-        ['S1B', 'S1A'] * 2,
-        save_dir=str(Path.cwd())
-    )
+    assert orbit_files == side_effect[1]
+    assert eof.download.download_eofs.call_count == 4
+    missions = 'S1B S1B S1A S1A'.split()
+    dts = '20201115T162313 20201115T162340 20201203T162353 20201203T162420'.split()
+    for dt, mission in zip(dts, missions):
+        eof.download.download_eofs.assert_any_call(
+            [dt],
+            [mission],
+            save_dir=str(Path.cwd()),
+            force_asf=True
+        )