diff --git a/CHANGELOG.md b/CHANGELOG.md
index c50648016..f4a0c807a 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -13,6 +13,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+ Fix bug in fillna3D for NaNs at elevations higher than present in the weather model
+ write delays even if they contain nans
+ check that the aoi is contained within HRRR extent
++ streamline some unit tests to remove downloading
+ move the ray building out of the _build_cube_ray and into its own function for cleaner testing
+ update the tests to use the new build_ray function
+ If the processed weather file exists use it; otherwise check if raw exists and covers study area; otherwise download new
diff --git a/test/scenario_5/hgt.hdr b/test/scenario_5/hgt.hdr
deleted file mode 100644
index 58ad111db..000000000
--- a/test/scenario_5/hgt.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 1
-lines = 1
-bands = 1
-header offset = 0
-file type = ENVI Standard
-data type = 4
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_5/hgt.rdr b/test/scenario_5/hgt.rdr
deleted file mode 100644
index 593f4708d..000000000
Binary files a/test/scenario_5/hgt.rdr and /dev/null differ
diff --git a/test/scenario_5/hgt.rdr.aux.xml b/test/scenario_5/hgt.rdr.aux.xml
deleted file mode 100644
index 9c772deb7..000000000
--- a/test/scenario_5/hgt.rdr.aux.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
- BAND
-
-
- 1
- 0
- 4
- ENVI Standard
- 0
- bsq
- 1
- 1
-
-
-
- 0
- 0
- 0
- 0
-
-
-
diff --git a/test/scenario_5/lat.hdr b/test/scenario_5/lat.hdr
deleted file mode 100644
index 12b90e981..000000000
--- a/test/scenario_5/lat.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 226
-lines = 45
-bands = 1
-header offset = 0
-file type = ENVI Standard
-data type = 5
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_5/lat.rdr b/test/scenario_5/lat.rdr
deleted file mode 100644
index c3b3d4bdc..000000000
Binary files a/test/scenario_5/lat.rdr and /dev/null differ
diff --git a/test/scenario_5/lat.rdr.aux.xml b/test/scenario_5/lat.rdr.aux.xml
deleted file mode 100644
index ba2940605..000000000
--- a/test/scenario_5/lat.rdr.aux.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
- BAND
-
-
- 1
- 0
- 4
- ENVI Standard
- 0
- bsq
- 1
- 1
-
-
-
- 20
- 20
- 20
- 0
-
-
-
diff --git a/test/scenario_5/lat.rdr.vrt b/test/scenario_5/lat.rdr.vrt
deleted file mode 100644
index 702cf03c3..000000000
--- a/test/scenario_5/lat.rdr.vrt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
- 21.493668454337
- 18.706121111652
- 15.763767828026
- 1.5837206786523
-
- 0
-
- lat.rdr
- 1
-
-
-
-
-
-
diff --git a/test/scenario_5/lon.hdr b/test/scenario_5/lon.hdr
deleted file mode 100644
index 12b90e981..000000000
--- a/test/scenario_5/lon.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 226
-lines = 45
-bands = 1
-header offset = 0
-file type = ENVI Standard
-data type = 5
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_5/lon.rdr b/test/scenario_5/lon.rdr
deleted file mode 100644
index d4617cc24..000000000
Binary files a/test/scenario_5/lon.rdr and /dev/null differ
diff --git a/test/scenario_5/lon.rdr.aux.xml b/test/scenario_5/lon.rdr.aux.xml
deleted file mode 100644
index 90f3d6273..000000000
--- a/test/scenario_5/lon.rdr.aux.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-
-
- BAND
-
-
- 1
- 0
- 4
- ENVI Standard
- 0
- bsq
- 1
- 1
-
-
-
- -100
- -100
- -100
- 0
-
-
-
diff --git a/test/scenario_5/lon.rdr.vrt b/test/scenario_5/lon.rdr.vrt
deleted file mode 100644
index 7ee762472..000000000
--- a/test/scenario_5/lon.rdr.vrt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
- -98.241840984674
- -99.860320396801
- -101.6383676311
- 0.74882734423551
-
- 0
-
- lon.rdr
- 1
-
-
-
-
-
-
diff --git a/test/scenario_5/los.hdr b/test/scenario_5/los.hdr
deleted file mode 100644
index 9dd8425a4..000000000
--- a/test/scenario_5/los.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 226
-lines = 45
-bands = 2
-header offset = 0
-file type = ENVI Standard
-data type = 4
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_5/los.rdr b/test/scenario_5/los.rdr
deleted file mode 100644
index 86e750dd1..000000000
Binary files a/test/scenario_5/los.rdr and /dev/null differ
diff --git a/test/scenario_5/los.rdr.vrt b/test/scenario_5/los.rdr.vrt
deleted file mode 100644
index 88eb53580..000000000
--- a/test/scenario_5/los.rdr.vrt
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
- los.rdr
- LSB
- 0
- 8
- 1808
-
-
- los.rdr
- LSB
- 4
- 8
- 1808
-
-
diff --git a/test/scenario_5/warpedDEM.dem b/test/scenario_5/warpedDEM.dem
deleted file mode 100644
index b84c03025..000000000
--- a/test/scenario_5/warpedDEM.dem
+++ /dev/null
@@ -1 +0,0 @@
-™w1E
\ No newline at end of file
diff --git a/test/scenario_5/warpedDEM.dem.vrt b/test/scenario_5/warpedDEM.dem.vrt
deleted file mode 100644
index b1bba6750..000000000
--- a/test/scenario_5/warpedDEM.dem.vrt
+++ /dev/null
@@ -1,9 +0,0 @@
-
-
- warpedDEM.dem
- LSB
- 0
- 4
- 904
-
-
diff --git a/test/scenario_5/warpedDEM.hdr b/test/scenario_5/warpedDEM.hdr
deleted file mode 100644
index c60fbad61..000000000
--- a/test/scenario_5/warpedDEM.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 226
-lines = 45
-bands = 1
-header offset = 0
-file type = ENVI Standard
-data type = 4
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_5/wmtype b/test/scenario_5/wmtype
deleted file mode 100644
index 7e60c298f..000000000
--- a/test/scenario_5/wmtype
+++ /dev/null
@@ -1,2 +0,0 @@
-wrf
-20100625000000
diff --git a/test/scenario_5/wrfout_d01_2010-06-25 b/test/scenario_5/wrfout_d01_2010-06-25
deleted file mode 100644
index 9365fc662..000000000
Binary files a/test/scenario_5/wrfout_d01_2010-06-25 and /dev/null differ
diff --git a/test/scenario_5/wrfplev_d01_2010-06-25 b/test/scenario_5/wrfplev_d01_2010-06-25
deleted file mode 100644
index 94b2c754f..000000000
Binary files a/test/scenario_5/wrfplev_d01_2010-06-25 and /dev/null differ
diff --git a/test/scenario_6/lat.hdr b/test/scenario_6/lat.hdr
index 12b90e981..4e4b4b565 100644
--- a/test/scenario_6/lat.hdr
+++ b/test/scenario_6/lat.hdr
@@ -1,9 +1,14 @@
ENVI
-samples = 226
-lines = 45
+description = {
+lat.rdr}
+samples = 7
+lines = 7
bands = 1
header offset = 0
file type = ENVI Standard
-data type = 5
+data type = 4
interleave = bsq
byte order = 0
+band names = {
+Band 1}
+data ignore value = -9999
diff --git a/test/scenario_6/lat.rdr b/test/scenario_6/lat.rdr
index c3b3d4bdc..dbc9fa15e 100644
Binary files a/test/scenario_6/lat.rdr and b/test/scenario_6/lat.rdr differ
diff --git a/test/scenario_6/lat.rdr.aux.xml b/test/scenario_6/lat.rdr.aux.xml
new file mode 100644
index 000000000..1b55543e6
--- /dev/null
+++ b/test/scenario_6/lat.rdr.aux.xml
@@ -0,0 +1,31 @@
+
+
+ BAND
+
+
+ Band 1
+
+
+ 1
+ {Band 1}
+ 0
+ -9999
+ 4
+ {lat.rdr}
+ ENVI Standard
+ 0
+ bsq
+ 7
+ 7
+
+
+ -9.99900000000000E+03
+
+ 16.536458969116
+ 16.150135176522
+ 15.763768196106
+ 0.25285638219246
+ 100
+
+
+
diff --git a/test/scenario_6/lat.rdr.vrt b/test/scenario_6/lat.rdr.vrt
deleted file mode 100644
index 702cf03c3..000000000
--- a/test/scenario_6/lat.rdr.vrt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
- 21.493668454337
- 18.706121111652
- 15.763767828026
- 1.5837206786523
-
- 0
-
- lat.rdr
- 1
-
-
-
-
-
-
diff --git a/test/scenario_6/lon.hdr b/test/scenario_6/lon.hdr
index 12b90e981..28e98f018 100644
--- a/test/scenario_6/lon.hdr
+++ b/test/scenario_6/lon.hdr
@@ -1,9 +1,14 @@
ENVI
-samples = 226
-lines = 45
+description = {
+lon.rdr}
+samples = 7
+lines = 7
bands = 1
header offset = 0
file type = ENVI Standard
-data type = 5
+data type = 4
interleave = bsq
byte order = 0
+band names = {
+Band 1}
+data ignore value = -9999
diff --git a/test/scenario_6/lon.rdr b/test/scenario_6/lon.rdr
index d4617cc24..ab97f9cad 100644
Binary files a/test/scenario_6/lon.rdr and b/test/scenario_6/lon.rdr differ
diff --git a/test/scenario_6/lon.rdr.aux.xml b/test/scenario_6/lon.rdr.aux.xml
index 99011b51f..422ef5178 100644
--- a/test/scenario_6/lon.rdr.aux.xml
+++ b/test/scenario_6/lon.rdr.aux.xml
@@ -2,22 +2,30 @@
BAND
+
+ Band 1
+
1
+ {Band 1}
0
- 5
+ -9999
+ 4
+ {lon.rdr}
ENVI Standard
0
bsq
- 45
- 226
+ 7
+ 7
+ -9.99900000000000E+03
- 0
- -96.050506796609
- -101.6383676311
- 19.143507420765
+ -100.44786071777
+ -100.56314943275
+ -100.67877960205
+ 0.057789502319966
+ 100
diff --git a/test/scenario_6/lon.rdr.vrt b/test/scenario_6/lon.rdr.vrt
deleted file mode 100644
index 7ee762472..000000000
--- a/test/scenario_6/lon.rdr.vrt
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
- -98.241840984674
- -99.860320396801
- -101.6383676311
- 0.74882734423551
-
- 0
-
- lon.rdr
- 1
-
-
-
-
-
-
diff --git a/test/scenario_6/stations.csv b/test/scenario_6/stations.csv
new file mode 100644
index 000000000..bb64add01
--- /dev/null
+++ b/test/scenario_6/stations.csv
@@ -0,0 +1,4 @@
+ID,Lat,Lon,Hgt_m
+HOOT,33.10514,-116.78536,617.21204
+TORP,33.79780,-118.33060,-5.21643
+
diff --git a/test/scenario_6/warpedDEM.dem b/test/scenario_6/warpedDEM.dem
deleted file mode 100644
index 1ef74c301..000000000
Binary files a/test/scenario_6/warpedDEM.dem and /dev/null differ
diff --git a/test/scenario_6/warpedDEM.dem.aux.xml b/test/scenario_6/warpedDEM.dem.aux.xml
deleted file mode 100644
index cc6ff8dc6..000000000
--- a/test/scenario_6/warpedDEM.dem.aux.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
- BAND
-
-
- 1
- 0
- 4
- ENVI Standard
- 0
- bsq
- 45
- 226
-
-
- 0.00000000000000E+00
-
-
diff --git a/test/scenario_6/warpedDEM.hdr b/test/scenario_6/warpedDEM.hdr
deleted file mode 100644
index c60fbad61..000000000
--- a/test/scenario_6/warpedDEM.hdr
+++ /dev/null
@@ -1,9 +0,0 @@
-ENVI
-samples = 226
-lines = 45
-bands = 1
-header offset = 0
-file type = ENVI Standard
-data type = 4
-interleave = bsq
-byte order = 0
diff --git a/test/scenario_6/wmtype b/test/scenario_6/wmtype
deleted file mode 100644
index ff7600d03..000000000
--- a/test/scenario_6/wmtype
+++ /dev/null
@@ -1,2 +0,0 @@
-era5
-20100625000000
diff --git a/test/test_datelist.py b/test/test_datelist.py
index fd0a276bb..a840c9333 100644
--- a/test/test_datelist.py
+++ b/test/test_datelist.py
@@ -1,12 +1,8 @@
+import datetime
import os
-import glob
-import pytest
-import subprocess
import shutil
-import yaml
-import numpy as np
-from test import TEST_DIR, WM
-
+from test import TEST_DIR, WM, update_yaml
+from RAiDER.cli.raider import read_template_file
def test_datelist():
SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist')
@@ -15,9 +11,10 @@ def test_datelist():
os.makedirs(SCENARIO_DIR, exist_ok=False)
dates = ['20200124', '20200130']
+ true_dates = [datetime.datetime(2020,1,24), datetime.datetime(2020,1,30)]
dct_group = {
- 'aoi_group': {'bounding_box': [28, 39, -123, -112]},
+ 'aoi_group': {'bounding_box': [28, 28.3, -116.3, -116]},
'date_group': {'date_list': dates},
'time_group': {'time': '00:00:00'},
'weather_model': WM,
@@ -26,35 +23,16 @@ def test_datelist():
'weather_model_directory': os.path.join(SCENARIO_DIR, 'weather_files')
}
}
-
- params = dct_group
- dst = os.path.join(SCENARIO_DIR, 'temp.yaml')
-
- with open(dst, 'w') as fh:
- yaml.dump(params, fh, default_flow_style=False)
-
-
- ## run raider on new file (two dates)
- cmd = f'raider.py {dst}'
- proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
- assert np.isclose(proc.returncode, 0)
-
- ## check that four files (2x date) were created
- n_files = len(glob.glob(os.path.join(SCENARIO_DIR, 'weather_files/*.nc')))
- n_dates = len(dates)
- assert np.isclose(n_files, n_dates*2), 'Incorrect number of files produced'
-
- ## clean up
- shutil.rmtree(SCENARIO_DIR)
-
- return dst
+
+ cfg = update_yaml(dct_group, 'temp.yaml')
+ param_dict = read_template_file(cfg)
+ assert param_dict['date_list'] == true_dates
def test_datestep():
- SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist')
- os.makedirs(SCENARIO_DIR, exist_ok=False)
+ SCENARIO_DIR = os.path.join(TEST_DIR, 'scenario_5')
st, en, step = '20200124', '20200130', 3
- n_dates = 3
+ true_dates = [datetime.datetime(2020,1,24), datetime.datetime(2020,1,27), datetime.datetime(2020,1,30)]
dct_group = {
'aoi_group': {'bounding_box': [28, 39, -123, -112]},
@@ -66,22 +44,7 @@ def test_datestep():
'weather_model_directory': os.path.join(SCENARIO_DIR, 'weather_files')
}
}
-
- params = dct_group
- dst = os.path.join(SCENARIO_DIR, 'temp.yaml')
-
- with open(dst, 'w') as fh:
- yaml.dump(params, fh, default_flow_style=False)
-
-
- ## run raider on new file (two dates)
- cmd = f'raider.py {dst}'
- proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
- assert np.isclose(proc.returncode, 0)
-
- ## check that four files (2x date) were created
- n_files = len(glob.glob(os.path.join(SCENARIO_DIR, 'weather_files/*.nc')))
- assert np.isclose(n_files, n_dates*2), 'Incorrect number of files produced'
-
- ## clean up
- shutil.rmtree(SCENARIO_DIR)
+
+ cfg = update_yaml(dct_group, 'temp.yaml')
+ param_dict = read_template_file(cfg)
+ assert param_dict['date_list'] == true_dates
\ No newline at end of file
diff --git a/test/test_intersect.py b/test/test_intersect.py
index f792de1b1..534c51292 100644
--- a/test/test_intersect.py
+++ b/test/test_intersect.py
@@ -1,8 +1,7 @@
-import glob
-import shutil
-
import pandas as pd
-import rioxarray as xrr
+import rasterio
+
+from scipy.interpolate import griddata
from test import *
@@ -10,7 +9,7 @@
@pytest.mark.parametrize('wm', 'ERA5'.split())
def test_cube_intersect(wm):
""" Test the intersection of lat/lon files with the DEM (model height levels?) """
- SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT")
+ SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_6")
os.makedirs(SCENARIO_DIR, exist_ok=True)
## make the lat lon grid
@@ -24,10 +23,15 @@ def test_cube_intersect(wm):
'date_group': {'date_start': date},
'time_group': {'time': time, 'interpolate_time': False},
'weather_model': wm,
- 'aoi_group': {'lat_file': f_lat, 'lon_file': f_lon},
- 'runtime_group': {'output_directory': SCENARIO_DIR,
- 'weather_model_directory': WM_DIR,
- },
+ 'aoi_group': {
+ 'lat_file': os.path.join(SCENARIO_DIR, 'lat.rdr'),
+ 'lon_file': os.path.join(SCENARIO_DIR, 'lon.rdr')
+ },
+ 'runtime_group': {
+ 'output_directory': SCENARIO_DIR,
+ 'weather_model_directory': WM_DIR,
+ },
+ 'verbose': False,
}
## generate the default template file and overwrite it with new parms
@@ -39,17 +43,19 @@ def test_cube_intersect(wm):
assert proc.returncode == 0, 'RAiDER Failed.'
## hard code what it should be and check it matches
- gold = {'ERA5': 2.29017997, 'GMAO': np.nan, 'HRRR': np.nan}
+ gold = {'ERA5': 2.2787, 'GMAO': np.nan, 'HRRR': np.nan}
path_delays = os.path.join(SCENARIO_DIR, f'{wm}_hydro_{date}T{time.replace(":", "")}_ztd.tiff')
- da = xrr.open_rasterio(path_delays, band_as_variable=True)['band_1']
- hyd = da.sel(x=-117.8, y=33.4, method='nearest').item()
- np.testing.assert_almost_equal(hyd, gold[wm])
+ latf = os.path.join(SCENARIO_DIR, 'lat.rdr')
+ lonf = os.path.join(SCENARIO_DIR, 'lon.rdr')
+
+ hyd = rasterio.open(path_delays).read(1)
+ lats = rasterio.open(latf).read(1)
+ lons = rasterio.open(lonf).read(1)
+ hyd = griddata(np.stack([lons.flatten(), lats.flatten()], axis=-1), hyd.flatten(), (-100.6, 16.15), method='nearest')
- # Clean up files
- shutil.rmtree(SCENARIO_DIR)
- [os.remove(f) for f in glob.glob(f'{wm}*')]
- os.remove('temp.yaml')
+ # test for equality with golden data
+ np.testing.assert_almost_equal(hyd, gold[wm], decimal=4)
return
@@ -57,9 +63,10 @@ def test_cube_intersect(wm):
@pytest.mark.parametrize('wm', 'ERA5'.split())
def test_gnss_intersect(wm):
- SCENARIO_DIR = os.path.join(TEST_DIR, "INTERSECT")
- os.makedirs(SCENARIO_DIR, exist_ok=True)
- gnss_file = os.path.join(TEST_DIR, 'scenario_2', 'stations2.csv')
+ SCENARIO_DIR = os.path.join(TEST_DIR, 'scenario_6')
+ gnss_file = os.path.join(SCENARIO_DIR, 'stations.csv')
+ id = 'TORP'
+
date = 20200130
time ='13:52:45'
@@ -69,9 +76,11 @@ def test_gnss_intersect(wm):
'time_group': {'time': time, 'interpolate_time': False},
'weather_model': wm,
'aoi_group': {'station_file': gnss_file},
- 'runtime_group': {'output_directory': SCENARIO_DIR,
- 'weather_model_directory': WM_DIR,
- }
+ 'runtime_group': {
+ 'output_directory': SCENARIO_DIR,
+ 'weather_model_directory': WM_DIR,
+ },
+ 'verbose': False,
}
## generate the default template file and overwrite it with new parms
@@ -82,16 +91,11 @@ def test_gnss_intersect(wm):
proc = subprocess.run(cmd.split(), stdout=subprocess.PIPE, universal_newlines=True)
assert proc.returncode == 0, 'RAiDER Failed.'
- gold = {'ERA5': 2.34513194, 'GMAO': np.nan, 'HRRR': np.nan}
-
+ gold = {'ERA5': 2.34514, 'GMAO': np.nan, 'HRRR': np.nan}
df = pd.read_csv(os.path.join(SCENARIO_DIR, f'{wm}_Delay_{date}T{time.replace(":", "")}_ztd.csv'))
+ td = df['totalDelay'][df['ID']==id].values
- id = 'TORP'
- td = df.set_index('ID').loc[id, 'totalDelay']
- np.testing.assert_almost_equal(gold[wm], td.item())
-
- shutil.rmtree(SCENARIO_DIR)
- [os.remove(f) for f in glob.glob(f'{wm}*')]
- os.remove('temp.yaml')
+ # test for equality with golden data
+ np.testing.assert_almost_equal(td.item(), gold[wm], decimal=4)
return
diff --git a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc
index 3c666583e..a09297aac 100644
Binary files a/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc and b/test/weather_files/ERA-5_2020_01_30_T13_52_45.nc differ
diff --git a/tools/RAiDER/models/weatherModel.py b/tools/RAiDER/models/weatherModel.py
index 04552d2a9..fcbfda1c3 100755
--- a/tools/RAiDER/models/weatherModel.py
+++ b/tools/RAiDER/models/weatherModel.py
@@ -162,8 +162,8 @@ def fetch(self, out, time):
self._fetch(out)
err = False
- except Exception:
- err = True
+ except Exception as E:
+ err = E
return err
diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py
index 5bedcc1a8..b1b034061 100755
--- a/tools/RAiDER/processWM.py
+++ b/tools/RAiDER/processWM.py
@@ -69,7 +69,7 @@ def prepareWeatherModel(
else:
E = weather_model.fetch(path_wm_raw, time)
if E:
- print ('raise runtimeerror')
+ logger.warning (E)
raise RuntimeError
# If only downloading, exit now