diff --git a/CHANGELOG.md b/CHANGELOG.md index 658a22535..07e400fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Latest updates: ++ Update tests to account for different interpolation scheme + Dont error out when the weather model contains nan values (HRRR) + Fix bug in fillna3D for NaNs at elevations higher than present in the weather model + write delays even if they contain nans diff --git a/test/test_datelist.py b/test/test_datelist.py index 1e4834f08..fd0a276bb 100644 --- a/test/test_datelist.py +++ b/test/test_datelist.py @@ -10,9 +10,8 @@ def test_datelist(): SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist') - if os.path.exists(SCENARIO_DIR): - shutil.rmtree(SCENARIO_DIR) + shutil.rmtree(SCENARIO_DIR) os.makedirs(SCENARIO_DIR, exist_ok=False) dates = ['20200124', '20200130'] diff --git a/test/test_intersect.py b/test/test_intersect.py index 950119f49..f792de1b1 100644 --- a/test/test_intersect.py +++ b/test/test_intersect.py @@ -14,10 +14,10 @@ def test_cube_intersect(wm): os.makedirs(SCENARIO_DIR, exist_ok=True) ## make the lat lon grid - S, N, W, E = 33, 34.5, -118.0, -117.0 + S, N, W, E = 33.5, 34, -118.0, -117.5 date = 20200130 time ='13:52:45' - f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.5) + f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.25) ## make the template file grp = { @@ -39,12 +39,12 @@ def test_cube_intersect(wm): assert proc.returncode == 0, 'RAiDER Failed.' ## hard code what it should be and check it matches - gold = {'ERA5': 2.2906463, 'GMAO': np.nan, 'HRRR': np.nan} + gold = {'ERA5': 2.29017997, 'GMAO': np.nan, 'HRRR': np.nan} path_delays = os.path.join(SCENARIO_DIR, f'{wm}_hydro_{date}T{time.replace(":", "")}_ztd.tiff') da = xrr.open_rasterio(path_delays, band_as_variable=True)['band_1'] hyd = da.sel(x=-117.8, y=33.4, method='nearest').item() - np.testing.assert_almost_equal(gold[wm], hyd) + np.testing.assert_almost_equal(hyd, gold[wm]) # Clean up files shutil.rmtree(SCENARIO_DIR) @@ -54,6 +54,7 @@ def test_cube_intersect(wm): return + @pytest.mark.parametrize('wm', 'ERA5'.split()) def test_gnss_intersect(wm): SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT") @@ -81,7 +82,7 @@ def test_gnss_intersect(wm): proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) assert proc.returncode == 0, 'RAiDER Failed.' - gold = {'ERA5': 2.3451457, 'GMAO': np.nan, 'HRRR': np.nan} + gold = {'ERA5': 2.34513194, 'GMAO': np.nan, 'HRRR': np.nan} df = pd.read_csv(os.path.join(SCENARIO_DIR, f'{wm}_Delay_{date}T{time.replace(":", "")}_ztd.csv')) diff --git a/test/test_slant.py b/test/test_slant.py index 397dc04e0..1453e2548 100644 --- a/test/test_slant.py +++ b/test/test_slant.py @@ -40,7 +40,7 @@ def test_slant_proj(weather_model_name): proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) assert proc.returncode == 0, 'RAiDER Failed.' - gold = {'ERA5': [33.4, -117.8, 0, 2.3338790]} + gold = {'ERA5': [33.4, -117.8, 0, 2.333865144]} lat, lon, hgt, val = gold[weather_model_name] path_delays = os.path.join(SCENARIO_DIR, make_delay_name(weather_model_name, date, time, 'std')) @@ -48,7 +48,7 @@ def test_slant_proj(weather_model_name): delay = (ds['hydro'] + ds['wet']).sel( y=lat, x=lon, z=hgt, method='nearest').item() - np.testing.assert_almost_equal(delay, val) + np.testing.assert_almost_equal(val, delay) # Clean up files shutil.rmtree(SCENARIO_DIR) @@ -92,7 +92,7 @@ def test_ray_tracing(weather_model_name): assert proc.returncode == 0, 'RAiDER Failed.' # model to lat/lon/correct value - gold = {'ERA5': [33.4, -117.8, 0, 2.9771327]} + gold = {'ERA5': [33.4, -117.8, 0, 2.97711681]} lat, lon, hgt, val = gold[weather_model_name] path_delays = os.path.join(SCENARIO_DIR, @@ -100,7 +100,7 @@ def test_ray_tracing(weather_model_name): with xr.open_dataset(path_delays) as ds: delay = (ds['hydro'] + ds['wet']).sel( y=lat, x=lon, z=hgt, method='nearest').item() - np.testing.assert_almost_equal(delay, val) + np.testing.assert_almost_equal(val, delay) # Clean up files shutil.rmtree(SCENARIO_DIR) diff --git a/test/test_synthetic.py b/test/test_synthetic.py index 0ae27d260..307148fa0 100644 --- a/test/test_synthetic.py +++ b/test/test_synthetic.py @@ -171,6 +171,7 @@ def make_config_dict(self): } return dct + @pytest.mark.skip() @pytest.mark.parametrize('region', 'AK LA Fort'.split()) def test_dl_real(region, mod='ERA5'): @@ -186,7 +187,6 @@ def test_dl_real(region, mod='ERA5'): dct_cfg['download_only'] = True cfg = update_yaml(dct_cfg) - ## run raider to download the real weather model cmd = f'raider.py {cfg}' @@ -257,6 +257,7 @@ def test_hydrostatic_eq(region, mod='ERA-5'): da.close() del da + @pytest.mark.parametrize('region', 'AK LA Fort'.split()) def test_wet_eq_linear(region, mod='ERA-5'): """ Test linear part of wet equation. diff --git a/test/weather_files/ERA-5_2019_11_17_T20_51_58.nc b/test/weather_files/ERA-5_2019_11_17_T20_51_58.nc index b2b343df9..b9e5d532d 100644 Binary files a/test/weather_files/ERA-5_2019_11_17_T20_51_58.nc and b/test/weather_files/ERA-5_2019_11_17_T20_51_58.nc differ diff --git a/test/weather_files/ERA-5_2019_11_17_T20_51_58_5S_2S_41W_37W.nc b/test/weather_files/ERA-5_2019_11_17_T20_51_58_5S_2S_41W_37W.nc index ee5625f0d..f7b849a73 100644 Binary files a/test/weather_files/ERA-5_2019_11_17_T20_51_58_5S_2S_41W_37W.nc and b/test/weather_files/ERA-5_2019_11_17_T20_51_58_5S_2S_41W_37W.nc differ diff --git a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc index e95140b17..3c666583e 100644 Binary files a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc and b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc differ diff --git a/test/weather_files/ERA-5_2020_01_30_T13_52_45_32N_35N_120W_115W.nc b/test/weather_files/ERA-5_2020_01_30_T13_52_45_32N_35N_120W_115W.nc index 9241e3b4c..3d46b0a71 100644 Binary files a/test/weather_files/ERA-5_2020_01_30_T13_52_45_32N_35N_120W_115W.nc and b/test/weather_files/ERA-5_2020_01_30_T13_52_45_32N_35N_120W_115W.nc differ diff --git a/test/weather_files/ERA-5_2022_08_29_T17_00_01.nc b/test/weather_files/ERA-5_2022_08_29_T17_00_01.nc index 07b781e97..7442a6303 100644 Binary files a/test/weather_files/ERA-5_2022_08_29_T17_00_01.nc and b/test/weather_files/ERA-5_2022_08_29_T17_00_01.nc differ diff --git a/test/weather_files/ERA-5_2022_08_29_T17_00_01_69N_73N_159W_152W.nc b/test/weather_files/ERA-5_2022_08_29_T17_00_01_69N_73N_159W_152W.nc index 7b8532579..e5bb3d812 100644 Binary files a/test/weather_files/ERA-5_2022_08_29_T17_00_01_69N_73N_159W_152W.nc and b/test/weather_files/ERA-5_2022_08_29_T17_00_01_69N_73N_159W_152W.nc differ diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index faaa3cd85..d5a433590 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -272,8 +272,12 @@ def calcDelays(iargs=None): continue if len(wfiles)==0: - logger.error('No weather model data was successfully obtained.') - raise RuntimeError + logger.error('No weather model data was successfully obtained.') + if len(params['date_list']) == 1: + raise RuntimeError + # skip date if mnultiple are requested + else: + continue # nearest weather model time elif len(wfiles)==1 and len(times)==1: diff --git a/tools/RAiDER/models/hrrr.py b/tools/RAiDER/models/hrrr.py index 3435c5d45..0652944df 100644 --- a/tools/RAiDER/models/hrrr.py +++ b/tools/RAiDER/models/hrrr.py @@ -47,7 +47,12 @@ def download_hrrr_file(ll_bounds, DATE, out, model='hrrr', product='prs', fxx=0, ) # Iterate through the list of datasets - ds_list = H.xarray(":(SPFH|PRES|TMP|HGT):", verbose=verbose) + try: + ds_list = H.xarray(":(SPFH|PRES|TMP|HGT):", verbose=verbose) + except ValueError as E: + logger.error (E) + raise ValueError + ds_out = None for ds in ds_list: if ('isobaricInhPa' in ds._coord_names) or ('levels' in ds._coord_names): diff --git a/tools/RAiDER/models/weatherModel.py b/tools/RAiDER/models/weatherModel.py index 8d2c7056b..16b5f28af 100755 --- a/tools/RAiDER/models/weatherModel.py +++ b/tools/RAiDER/models/weatherModel.py @@ -160,9 +160,12 @@ def fetch(self, out, time): # write the error raised by the weather model API to the log try: self._fetch(out) + err = False - except Exception as E: - logger.error(E) + except Exception: + err = True + + return err @abstractmethod @@ -567,6 +570,7 @@ def checkContainment(self: weatherModel, return weather_model_box.contains(input_box) + def _isOutside(self, extent1, extent2): ''' Determine whether any of extent1 lies outside extent2 diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 8a1c26386..5bedcc1a8 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -67,7 +67,10 @@ def prepareWeatherModel( # if no weather model files supplied, check the standard location else: - weather_model.fetch(path_wm_raw, time) + E = weather_model.fetch(path_wm_raw, time) + if E: + print ('raise runtimeerror') + raise RuntimeError # If only downloading, exit now if download_only: @@ -86,7 +89,7 @@ def prepareWeatherModel( ) containment = weather_model.checkContainment(ll_bounds) - if not containment: + if not containment and weather_model.Model() in 'GMAO ERA5 ERA5T HRES'.split(): msg = 'The weather model passed does not cover all of the input ' \ 'points; you may need to download a larger area.' logger.error(msg) @@ -129,10 +132,12 @@ def prepareWeatherModel( logger.exception("Unable to save weathermodel to file") logger.exception(e) raise RuntimeError("Unable to save weathermodel to file") + finally: + wm = weather_model.Model() del weather_model - if not containment: + if not containment and wm in 'GMAO ERA5 ERA5T HRES'.split(): msg = 'The weather model passed does not cover all of the input ' \ 'points; you may need to download a larger area.' logger.error(msg)