diff --git a/CHANGELOG.md b/CHANGELOG.md index c50648016..f4a0c807a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + Fix bug in fillna3D for NaNs at elevations higher than present in the weather model + write delays even if they contain nans + check that the aoi is contained within HRRR extent ++ streamline some unit tests to remove downloading + move the ray building out of the _build_cube_ray and into its own function for cleaner testing + update the tests to use the new build_ray function + If the processed weather file exists use it; otherwise check if raw exists and covers study area; otherwise download new diff --git a/test/scenario_5/hgt.hdr b/test/scenario_5/hgt.hdr deleted file mode 100644 index 58ad111db..000000000 --- a/test/scenario_5/hgt.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 1 -lines = 1 -bands = 1 -header offset = 0 -file type = ENVI Standard -data type = 4 -interleave = bsq -byte order = 0 diff --git a/test/scenario_5/hgt.rdr b/test/scenario_5/hgt.rdr deleted file mode 100644 index 593f4708d..000000000 Binary files a/test/scenario_5/hgt.rdr and /dev/null differ diff --git a/test/scenario_5/hgt.rdr.aux.xml b/test/scenario_5/hgt.rdr.aux.xml deleted file mode 100644 index 9c772deb7..000000000 --- a/test/scenario_5/hgt.rdr.aux.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - BAND - - - 1 - 0 - 4 - ENVI Standard - 0 - bsq - 1 - 1 - - - - 0 - 0 - 0 - 0 - - - diff --git a/test/scenario_5/lat.hdr b/test/scenario_5/lat.hdr deleted file mode 100644 index 12b90e981..000000000 --- a/test/scenario_5/lat.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 226 -lines = 45 -bands = 1 -header offset = 0 -file type = ENVI Standard -data type = 5 -interleave = bsq -byte order = 0 diff --git a/test/scenario_5/lat.rdr b/test/scenario_5/lat.rdr deleted file mode 100644 index c3b3d4bdc..000000000 Binary files a/test/scenario_5/lat.rdr and /dev/null differ diff --git a/test/scenario_5/lat.rdr.aux.xml b/test/scenario_5/lat.rdr.aux.xml deleted file mode 100644 index ba2940605..000000000 --- a/test/scenario_5/lat.rdr.aux.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - BAND - - - 1 - 0 - 4 - ENVI Standard - 0 - bsq - 1 - 1 - - - - 20 - 20 - 20 - 0 - - - diff --git a/test/scenario_5/lat.rdr.vrt b/test/scenario_5/lat.rdr.vrt deleted file mode 100644 index 702cf03c3..000000000 --- a/test/scenario_5/lat.rdr.vrt +++ /dev/null @@ -1,18 +0,0 @@ - - - - 21.493668454337 - 18.706121111652 - 15.763767828026 - 1.5837206786523 - - 0 - - lat.rdr - 1 - - - - - - diff --git a/test/scenario_5/lon.hdr b/test/scenario_5/lon.hdr deleted file mode 100644 index 12b90e981..000000000 --- a/test/scenario_5/lon.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 226 -lines = 45 -bands = 1 -header offset = 0 -file type = ENVI Standard -data type = 5 -interleave = bsq -byte order = 0 diff --git a/test/scenario_5/lon.rdr b/test/scenario_5/lon.rdr deleted file mode 100644 index d4617cc24..000000000 Binary files a/test/scenario_5/lon.rdr and /dev/null differ diff --git a/test/scenario_5/lon.rdr.aux.xml b/test/scenario_5/lon.rdr.aux.xml deleted file mode 100644 index 90f3d6273..000000000 --- a/test/scenario_5/lon.rdr.aux.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - BAND - - - 1 - 0 - 4 - ENVI Standard - 0 - bsq - 1 - 1 - - - - -100 - -100 - -100 - 0 - - - diff --git a/test/scenario_5/lon.rdr.vrt b/test/scenario_5/lon.rdr.vrt deleted file mode 100644 index 7ee762472..000000000 --- a/test/scenario_5/lon.rdr.vrt +++ /dev/null @@ -1,18 +0,0 @@ - - - - -98.241840984674 - -99.860320396801 - -101.6383676311 - 0.74882734423551 - - 0 - - lon.rdr - 1 - - - - - - diff --git a/test/scenario_5/los.hdr b/test/scenario_5/los.hdr deleted file mode 100644 index 9dd8425a4..000000000 --- a/test/scenario_5/los.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 226 -lines = 45 -bands = 2 -header offset = 0 -file type = ENVI Standard -data type = 4 -interleave = bsq -byte order = 0 diff --git a/test/scenario_5/los.rdr b/test/scenario_5/los.rdr deleted file mode 100644 index 86e750dd1..000000000 Binary files a/test/scenario_5/los.rdr and /dev/null differ diff --git a/test/scenario_5/los.rdr.vrt b/test/scenario_5/los.rdr.vrt deleted file mode 100644 index 88eb53580..000000000 --- a/test/scenario_5/los.rdr.vrt +++ /dev/null @@ -1,16 +0,0 @@ - - - los.rdr - LSB - 0 - 8 - 1808 - - - los.rdr - LSB - 4 - 8 - 1808 - - diff --git a/test/scenario_5/warpedDEM.dem b/test/scenario_5/warpedDEM.dem deleted file mode 100644 index b84c03025..000000000 --- a/test/scenario_5/warpedDEM.dem +++ /dev/null @@ -1 +0,0 @@ -™w1E \ No newline at end of file diff --git a/test/scenario_5/warpedDEM.dem.vrt b/test/scenario_5/warpedDEM.dem.vrt deleted file mode 100644 index b1bba6750..000000000 --- a/test/scenario_5/warpedDEM.dem.vrt +++ /dev/null @@ -1,9 +0,0 @@ - - - warpedDEM.dem - LSB - 0 - 4 - 904 - - diff --git a/test/scenario_5/warpedDEM.hdr b/test/scenario_5/warpedDEM.hdr deleted file mode 100644 index c60fbad61..000000000 --- a/test/scenario_5/warpedDEM.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 226 -lines = 45 -bands = 1 -header offset = 0 -file type = ENVI Standard -data type = 4 -interleave = bsq -byte order = 0 diff --git a/test/scenario_5/wmtype b/test/scenario_5/wmtype deleted file mode 100644 index 7e60c298f..000000000 --- a/test/scenario_5/wmtype +++ /dev/null @@ -1,2 +0,0 @@ -wrf -20100625000000 diff --git a/test/scenario_5/wrfout_d01_2010-06-25 b/test/scenario_5/wrfout_d01_2010-06-25 deleted file mode 100644 index 9365fc662..000000000 Binary files a/test/scenario_5/wrfout_d01_2010-06-25 and /dev/null differ diff --git a/test/scenario_5/wrfplev_d01_2010-06-25 b/test/scenario_5/wrfplev_d01_2010-06-25 deleted file mode 100644 index 94b2c754f..000000000 Binary files a/test/scenario_5/wrfplev_d01_2010-06-25 and /dev/null differ diff --git a/test/scenario_6/lat.hdr b/test/scenario_6/lat.hdr index 12b90e981..4e4b4b565 100644 --- a/test/scenario_6/lat.hdr +++ b/test/scenario_6/lat.hdr @@ -1,9 +1,14 @@ ENVI -samples = 226 -lines = 45 +description = { +lat.rdr} +samples = 7 +lines = 7 bands = 1 header offset = 0 file type = ENVI Standard -data type = 5 +data type = 4 interleave = bsq byte order = 0 +band names = { +Band 1} +data ignore value = -9999 diff --git a/test/scenario_6/lat.rdr b/test/scenario_6/lat.rdr index c3b3d4bdc..dbc9fa15e 100644 Binary files a/test/scenario_6/lat.rdr and b/test/scenario_6/lat.rdr differ diff --git a/test/scenario_6/lat.rdr.aux.xml b/test/scenario_6/lat.rdr.aux.xml new file mode 100644 index 000000000..1b55543e6 --- /dev/null +++ b/test/scenario_6/lat.rdr.aux.xml @@ -0,0 +1,31 @@ + + + BAND + + + Band 1 + + + 1 + {Band 1} + 0 + -9999 + 4 + {lat.rdr} + ENVI Standard + 0 + bsq + 7 + 7 + + + -9.99900000000000E+03 + + 16.536458969116 + 16.150135176522 + 15.763768196106 + 0.25285638219246 + 100 + + + diff --git a/test/scenario_6/lat.rdr.vrt b/test/scenario_6/lat.rdr.vrt deleted file mode 100644 index 702cf03c3..000000000 --- a/test/scenario_6/lat.rdr.vrt +++ /dev/null @@ -1,18 +0,0 @@ - - - - 21.493668454337 - 18.706121111652 - 15.763767828026 - 1.5837206786523 - - 0 - - lat.rdr - 1 - - - - - - diff --git a/test/scenario_6/lon.hdr b/test/scenario_6/lon.hdr index 12b90e981..28e98f018 100644 --- a/test/scenario_6/lon.hdr +++ b/test/scenario_6/lon.hdr @@ -1,9 +1,14 @@ ENVI -samples = 226 -lines = 45 +description = { +lon.rdr} +samples = 7 +lines = 7 bands = 1 header offset = 0 file type = ENVI Standard -data type = 5 +data type = 4 interleave = bsq byte order = 0 +band names = { +Band 1} +data ignore value = -9999 diff --git a/test/scenario_6/lon.rdr b/test/scenario_6/lon.rdr index d4617cc24..ab97f9cad 100644 Binary files a/test/scenario_6/lon.rdr and b/test/scenario_6/lon.rdr differ diff --git a/test/scenario_6/lon.rdr.aux.xml b/test/scenario_6/lon.rdr.aux.xml index 99011b51f..422ef5178 100644 --- a/test/scenario_6/lon.rdr.aux.xml +++ b/test/scenario_6/lon.rdr.aux.xml @@ -2,22 +2,30 @@ BAND + + Band 1 + 1 + {Band 1} 0 - 5 + -9999 + 4 + {lon.rdr} ENVI Standard 0 bsq - 45 - 226 + 7 + 7 + -9.99900000000000E+03 - 0 - -96.050506796609 - -101.6383676311 - 19.143507420765 + -100.44786071777 + -100.56314943275 + -100.67877960205 + 0.057789502319966 + 100 diff --git a/test/scenario_6/lon.rdr.vrt b/test/scenario_6/lon.rdr.vrt deleted file mode 100644 index 7ee762472..000000000 --- a/test/scenario_6/lon.rdr.vrt +++ /dev/null @@ -1,18 +0,0 @@ - - - - -98.241840984674 - -99.860320396801 - -101.6383676311 - 0.74882734423551 - - 0 - - lon.rdr - 1 - - - - - - diff --git a/test/scenario_6/stations.csv b/test/scenario_6/stations.csv new file mode 100644 index 000000000..bb64add01 --- /dev/null +++ b/test/scenario_6/stations.csv @@ -0,0 +1,4 @@ +ID,Lat,Lon,Hgt_m +HOOT,33.10514,-116.78536,617.21204 +TORP,33.79780,-118.33060,-5.21643 + diff --git a/test/scenario_6/warpedDEM.dem b/test/scenario_6/warpedDEM.dem deleted file mode 100644 index 1ef74c301..000000000 Binary files a/test/scenario_6/warpedDEM.dem and /dev/null differ diff --git a/test/scenario_6/warpedDEM.dem.aux.xml b/test/scenario_6/warpedDEM.dem.aux.xml deleted file mode 100644 index cc6ff8dc6..000000000 --- a/test/scenario_6/warpedDEM.dem.aux.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - BAND - - - 1 - 0 - 4 - ENVI Standard - 0 - bsq - 45 - 226 - - - 0.00000000000000E+00 - - diff --git a/test/scenario_6/warpedDEM.hdr b/test/scenario_6/warpedDEM.hdr deleted file mode 100644 index c60fbad61..000000000 --- a/test/scenario_6/warpedDEM.hdr +++ /dev/null @@ -1,9 +0,0 @@ -ENVI -samples = 226 -lines = 45 -bands = 1 -header offset = 0 -file type = ENVI Standard -data type = 4 -interleave = bsq -byte order = 0 diff --git a/test/scenario_6/wmtype b/test/scenario_6/wmtype deleted file mode 100644 index ff7600d03..000000000 --- a/test/scenario_6/wmtype +++ /dev/null @@ -1,2 +0,0 @@ -era5 -20100625000000 diff --git a/test/test_datelist.py b/test/test_datelist.py index fd0a276bb..a840c9333 100644 --- a/test/test_datelist.py +++ b/test/test_datelist.py @@ -1,12 +1,8 @@ +import datetime import os -import glob -import pytest -import subprocess import shutil -import yaml -import numpy as np -from test import TEST_DIR, WM - +from test import TEST_DIR, WM, update_yaml +from RAiDER.cli.raider import read_template_file def test_datelist(): SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist') @@ -15,9 +11,10 @@ def test_datelist(): os.makedirs(SCENARIO_DIR, exist_ok=False) dates = ['20200124', '20200130'] + true_dates = [datetime.datetime(2020,1,24), datetime.datetime(2020,1,30)] dct_group = { - 'aoi_group': {'bounding_box': [28, 39, -123, -112]}, + 'aoi_group': {'bounding_box': [28, 28.3, -116.3, -116]}, 'date_group': {'date_list': dates}, 'time_group': {'time': '00:00:00'}, 'weather_model': WM, @@ -26,35 +23,16 @@ def test_datelist(): 'weather_model_directory': os.path.join(SCENARIO_DIR, 'weather_files') } } - - params = dct_group - dst = os.path.join(SCENARIO_DIR, 'temp.yaml') - - with open(dst, 'w') as fh: - yaml.dump(params, fh, default_flow_style=False) - - - ## run raider on new file (two dates) - cmd = f'raider.py {dst}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) - assert np.isclose(proc.returncode, 0) - - ## check that four files (2x date) were created - n_files = len(glob.glob(os.path.join(SCENARIO_DIR, 'weather_files/*.nc'))) - n_dates = len(dates) - assert np.isclose(n_files, n_dates*2), 'Incorrect number of files produced' - - ## clean up - shutil.rmtree(SCENARIO_DIR) - - return dst + + cfg = update_yaml(dct_group, 'temp.yaml') + param_dict = read_template_file(cfg) + assert param_dict['date_list'] == true_dates def test_datestep(): - SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist') - os.makedirs(SCENARIO_DIR, exist_ok=False) + SCENARIO_DIR = os.path.join(TEST_DIR, 'scenario_5') st, en, step = '20200124', '20200130', 3 - n_dates = 3 + true_dates = [datetime.datetime(2020,1,24), datetime.datetime(2020,1,27), datetime.datetime(2020,1,30)] dct_group = { 'aoi_group': {'bounding_box': [28, 39, -123, -112]}, @@ -66,22 +44,7 @@ def test_datestep(): 'weather_model_directory': os.path.join(SCENARIO_DIR, 'weather_files') } } - - params = dct_group - dst = os.path.join(SCENARIO_DIR, 'temp.yaml') - - with open(dst, 'w') as fh: - yaml.dump(params, fh, default_flow_style=False) - - - ## run raider on new file (two dates) - cmd = f'raider.py {dst}' - proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) - assert np.isclose(proc.returncode, 0) - - ## check that four files (2x date) were created - n_files = len(glob.glob(os.path.join(SCENARIO_DIR, 'weather_files/*.nc'))) - assert np.isclose(n_files, n_dates*2), 'Incorrect number of files produced' - - ## clean up - shutil.rmtree(SCENARIO_DIR) + + cfg = update_yaml(dct_group, 'temp.yaml') + param_dict = read_template_file(cfg) + assert param_dict['date_list'] == true_dates \ No newline at end of file diff --git a/test/test_intersect.py b/test/test_intersect.py index f792de1b1..534c51292 100644 --- a/test/test_intersect.py +++ b/test/test_intersect.py @@ -1,8 +1,7 @@ -import glob -import shutil - import pandas as pd -import rioxarray as xrr +import rasterio + +from scipy.interpolate import griddata from test import * @@ -10,7 +9,7 @@ @pytest.mark.parametrize('wm', 'ERA5'.split()) def test_cube_intersect(wm): """ Test the intersection of lat/lon files with the DEM (model height levels?) """ - SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT") + SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_6") os.makedirs(SCENARIO_DIR, exist_ok=True) ## make the lat lon grid @@ -24,10 +23,15 @@ def test_cube_intersect(wm): 'date_group': {'date_start': date}, 'time_group': {'time': time, 'interpolate_time': False}, 'weather_model': wm, - 'aoi_group': {'lat_file': f_lat, 'lon_file': f_lon}, - 'runtime_group': {'output_directory': SCENARIO_DIR, - 'weather_model_directory': WM_DIR, - }, + 'aoi_group': { + 'lat_file': os.path.join(SCENARIO_DIR, 'lat.rdr'), + 'lon_file': os.path.join(SCENARIO_DIR, 'lon.rdr') + }, + 'runtime_group': { + 'output_directory': SCENARIO_DIR, + 'weather_model_directory': WM_DIR, + }, + 'verbose': False, } ## generate the default template file and overwrite it with new parms @@ -39,17 +43,19 @@ def test_cube_intersect(wm): assert proc.returncode == 0, 'RAiDER Failed.' ## hard code what it should be and check it matches - gold = {'ERA5': 2.29017997, 'GMAO': np.nan, 'HRRR': np.nan} + gold = {'ERA5': 2.2787, 'GMAO': np.nan, 'HRRR': np.nan} path_delays = os.path.join(SCENARIO_DIR, f'{wm}_hydro_{date}T{time.replace(":", "")}_ztd.tiff') - da = xrr.open_rasterio(path_delays, band_as_variable=True)['band_1'] - hyd = da.sel(x=-117.8, y=33.4, method='nearest').item() - np.testing.assert_almost_equal(hyd, gold[wm]) + latf = os.path.join(SCENARIO_DIR, 'lat.rdr') + lonf = os.path.join(SCENARIO_DIR, 'lon.rdr') + + hyd = rasterio.open(path_delays).read(1) + lats = rasterio.open(latf).read(1) + lons = rasterio.open(lonf).read(1) + hyd = griddata(np.stack([lons.flatten(), lats.flatten()], axis=-1), hyd.flatten(), (-100.6, 16.15), method='nearest') - # Clean up files - shutil.rmtree(SCENARIO_DIR) - [os.remove(f) for f in glob.glob(f'{wm}*')] - os.remove('temp.yaml') + # test for equality with golden data + np.testing.assert_almost_equal(hyd, gold[wm], decimal=4) return @@ -57,9 +63,10 @@ def test_cube_intersect(wm): @pytest.mark.parametrize('wm', 'ERA5'.split()) def test_gnss_intersect(wm): - SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT") - os.makedirs(SCENARIO_DIR, exist_ok=True) - gnss_file = os.path.join(TEST_DIR, 'scenario_2', 'stations2.csv') + SCENARIO_DIR = os.path.join(TEST_DIR, 'scenario_6') + gnss_file = os.path.join(SCENARIO_DIR, 'stations.csv') + id = 'TORP' + date = 20200130 time ='13:52:45' @@ -69,9 +76,11 @@ def test_gnss_intersect(wm): 'time_group': {'time': time, 'interpolate_time': False}, 'weather_model': wm, 'aoi_group': {'station_file': gnss_file}, - 'runtime_group': {'output_directory': SCENARIO_DIR, - 'weather_model_directory': WM_DIR, - } + 'runtime_group': { + 'output_directory': SCENARIO_DIR, + 'weather_model_directory': WM_DIR, + }, + 'verbose': False, } ## generate the default template file and overwrite it with new parms @@ -82,16 +91,11 @@ def test_gnss_intersect(wm): proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True) assert proc.returncode == 0, 'RAiDER Failed.' - gold = {'ERA5': 2.34513194, 'GMAO': np.nan, 'HRRR': np.nan} - + gold = {'ERA5': 2.34514, 'GMAO': np.nan, 'HRRR': np.nan} df = pd.read_csv(os.path.join(SCENARIO_DIR, f'{wm}_Delay_{date}T{time.replace(":", "")}_ztd.csv')) + td = df['totalDelay'][df['ID']==id].values - id = 'TORP' - td = df.set_index('ID').loc[id, 'totalDelay'] - np.testing.assert_almost_equal(gold[wm], td.item()) - - shutil.rmtree(SCENARIO_DIR) - [os.remove(f) for f in glob.glob(f'{wm}*')] - os.remove('temp.yaml') + # test for equality with golden data + np.testing.assert_almost_equal(td.item(), gold[wm], decimal=4) return diff --git a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc index 3c666583e..a09297aac 100644 Binary files a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc and b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc differ diff --git a/tools/RAiDER/models/weatherModel.py b/tools/RAiDER/models/weatherModel.py index 04552d2a9..fcbfda1c3 100755 --- a/tools/RAiDER/models/weatherModel.py +++ b/tools/RAiDER/models/weatherModel.py @@ -162,8 +162,8 @@ def fetch(self, out, time): self._fetch(out) err = False - except Exception: - err = True + except Exception as E: + err = E return err diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 5bedcc1a8..b1b034061 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -69,7 +69,7 @@ def prepareWeatherModel( else: E = weather_model.fetch(path_wm_raw, time) if E: - print ('raise runtimeerror') + logger.warning (E) raise RuntimeError # If only downloading, exit now