Merge pull request #569 from bbuzz31/tests_for_new_fillna

Tests for new fillna

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
 and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## Latest updates:
++ Update tests to account for different interpolation scheme
 + Dont error out when the weather model contains nan values (HRRR)
 + Fix bug in fillna3D for NaNs at elevations higher than present in the weather model
 + write delays even if they contain nans

test/test_datelist.py

@@ -10,9 +10,8 @@
 
 def test_datelist():
     SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist')
-
     if os.path.exists(SCENARIO_DIR):
-        shutil.rmtree(SCENARIO_DIR) 
+        shutil.rmtree(SCENARIO_DIR)
     os.makedirs(SCENARIO_DIR, exist_ok=False)
 
     dates = ['20200124', '20200130']

test/test_intersect.py

@@ -14,10 +14,10 @@ def test_cube_intersect(wm):
     os.makedirs(SCENARIO_DIR, exist_ok=True)
 
     ## make the lat lon grid
-    S, N, W, E = 33, 34.5, -118.0, -117.0
+    S, N, W, E = 33.5, 34, -118.0, -117.5
     date       = 20200130
     time       ='13:52:45'
-    f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.5)
+    f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.25)
 
     ## make the template file
     grp = {
@@ -39,12 +39,12 @@ def test_cube_intersect(wm):
     assert proc.returncode == 0, 'RAiDER Failed.'
 
     ## hard code what it should be and check it matches
-    gold = {'ERA5': 2.2906463, 'GMAO': np.nan, 'HRRR': np.nan}
+    gold = {'ERA5': 2.29017997, 'GMAO': np.nan, 'HRRR': np.nan}
 
     path_delays = os.path.join(SCENARIO_DIR, f'{wm}_hydro_{date}T{time.replace(":", "")}_ztd.tiff')
     da  = xrr.open_rasterio(path_delays, band_as_variable=True)['band_1']
     hyd = da.sel(x=-117.8, y=33.4, method='nearest').item()
-    np.testing.assert_almost_equal(gold[wm], hyd)
+    np.testing.assert_almost_equal(hyd, gold[wm])
 
     # Clean up files
     shutil.rmtree(SCENARIO_DIR)
@@ -54,6 +54,7 @@ def test_cube_intersect(wm):
     return
 
 
+
 @pytest.mark.parametrize('wm', 'ERA5'.split())
 def test_gnss_intersect(wm):
     SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT")
@@ -81,7 +82,7 @@ def test_gnss_intersect(wm):
     proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
     assert proc.returncode == 0, 'RAiDER Failed.'
 
-    gold = {'ERA5': 2.3451457, 'GMAO': np.nan, 'HRRR': np.nan}
+    gold = {'ERA5': 2.34513194, 'GMAO': np.nan, 'HRRR': np.nan}
 
     df = pd.read_csv(os.path.join(SCENARIO_DIR, f'{wm}_Delay_{date}T{time.replace(":", "")}_ztd.csv'))
 

test/test_slant.py

@@ -40,15 +40,15 @@ def test_slant_proj(weather_model_name):
     proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
     assert proc.returncode == 0, 'RAiDER Failed.'
 
-    gold = {'ERA5': [33.4, -117.8, 0, 2.3338790]}
+    gold = {'ERA5': [33.4, -117.8, 0, 2.333865144]}
     lat, lon, hgt, val = gold[weather_model_name]
     path_delays = os.path.join(SCENARIO_DIR,
                     make_delay_name(weather_model_name, date, time, 'std'))
     with xr.open_dataset(path_delays) as ds:
         delay = (ds['hydro'] + ds['wet']).sel(
             y=lat, x=lon, z=hgt, method='nearest').item()
 
-    np.testing.assert_almost_equal(delay, val)
+    np.testing.assert_almost_equal(val, delay)
 
     # Clean up files
     shutil.rmtree(SCENARIO_DIR)
@@ -92,15 +92,15 @@ def test_ray_tracing(weather_model_name):
     assert proc.returncode == 0, 'RAiDER Failed.'
 
     # model to lat/lon/correct value
-    gold = {'ERA5': [33.4, -117.8, 0, 2.9771327]}
+    gold = {'ERA5': [33.4, -117.8, 0, 2.97711681]}
     lat, lon, hgt, val = gold[weather_model_name]
 
     path_delays = os.path.join(SCENARIO_DIR,
                     make_delay_name(weather_model_name, date, time, 'ray'))
     with xr.open_dataset(path_delays) as ds:
         delay = (ds['hydro'] + ds['wet']).sel(
             y=lat, x=lon, z=hgt, method='nearest').item()
-    np.testing.assert_almost_equal(delay, val)
+    np.testing.assert_almost_equal(val, delay)
 
     # Clean up files
     shutil.rmtree(SCENARIO_DIR)

test/test_synthetic.py

@@ -171,6 +171,7 @@ def make_config_dict(self):
         }
         return dct
 
+
 @pytest.mark.skip()
 @pytest.mark.parametrize('region', 'AK LA Fort'.split())
 def test_dl_real(region, mod='ERA5'):
@@ -186,7 +187,6 @@ def test_dl_real(region, mod='ERA5'):
     dct_cfg['download_only'] = True
 
     cfg = update_yaml(dct_cfg)
-
     ## run raider to download the real weather model
     cmd  = f'raider.py {cfg}'
 
@@ -257,6 +257,7 @@ def test_hydrostatic_eq(region, mod='ERA-5'):
     da.close()
     del da
 
+
 @pytest.mark.parametrize('region', 'AK LA Fort'.split())
 def test_wet_eq_linear(region, mod='ERA-5'):
     """ Test linear part of wet equation.

tools/RAiDER/cli/raider.py

@@ -272,8 +272,12 @@ def calcDelays(iargs=None):
             continue
 
         if len(wfiles)==0:
-             logger.error('No weather model data was successfully obtained.')
-             raise RuntimeError
+            logger.error('No weather model data was successfully obtained.')
+            if len(params['date_list']) == 1:
+                raise RuntimeError
+            # skip date if mnultiple are requested
+            else:
+                continue
 
         # nearest weather model time
         elif len(wfiles)==1 and len(times)==1:

tools/RAiDER/models/hrrr.py

@@ -47,7 +47,12 @@ def download_hrrr_file(ll_bounds, DATE, out, model='hrrr', product='prs', fxx=0,
     )
 
     # Iterate through the list of datasets
-    ds_list = H.xarray(":(SPFH|PRES|TMP|HGT):", verbose=verbose)
+    try:
+        ds_list = H.xarray(":(SPFH|PRES|TMP|HGT):", verbose=verbose)
+    except ValueError as E:
+        logger.error (E)
+        raise ValueError
+
     ds_out = None
     for ds in ds_list:
         if ('isobaricInhPa' in ds._coord_names) or ('levels' in ds._coord_names):

tools/RAiDER/models/weatherModel.py

@@ -160,9 +160,12 @@ def fetch(self, out, time):
         # write the error raised by the weather model API to the log
         try:
             self._fetch(out)
+            err = False
 
-        except Exception as E:
-            logger.error(E)
+        except Exception:
+            err = True
+
+        return err
 
 
     @abstractmethod
@@ -567,6 +570,7 @@ def checkContainment(self: weatherModel,
 
         return weather_model_box.contains(input_box)
 
+
     def _isOutside(self, extent1, extent2):
         '''
         Determine whether any of extent1  lies outside extent2

tools/RAiDER/processWM.py

@@ -67,7 +67,10 @@ def prepareWeatherModel(
 
     # if no weather model files supplied, check the standard location
     else:
-        weather_model.fetch(path_wm_raw, time)
+        E = weather_model.fetch(path_wm_raw, time)
+        if E:
+            print ('raise runtimeerror')
+            raise RuntimeError
 
     # If only downloading, exit now
     if download_only:
@@ -86,7 +89,7 @@ def prepareWeatherModel(
         )
 
         containment = weather_model.checkContainment(ll_bounds)
-        if not containment:
+        if not containment and weather_model.Model() in 'GMAO ERA5 ERA5T HRES'.split():
             msg = 'The weather model passed does not cover all of the input ' \
                 'points; you may need to download a larger area.'
             logger.error(msg)
@@ -129,10 +132,12 @@ def prepareWeatherModel(
         logger.exception("Unable to save weathermodel to file")
         logger.exception(e)
         raise RuntimeError("Unable to save weathermodel to file")
+
     finally:
+        wm = weather_model.Model()
         del weather_model
 
-    if not containment:
+    if not containment and wm in 'GMAO ERA5 ERA5T HRES'.split():
         msg = 'The weather model passed does not cover all of the input ' \
             'points; you may need to download a larger area.'
         logger.error(msg)