Skip to content

Commit

Permalink
Merge pull request #656 from garlic-os/port-example-args-to-yamls
Browse files Browse the repository at this point in the history
Re-add example run configurations
  • Loading branch information
jlmaurer authored Jul 9, 2024
2 parents e37c73c + 5190b53 commit e4131dc
Show file tree
Hide file tree
Showing 18 changed files with 123,716 additions and 148 deletions.
7 changes: 5 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,16 @@ and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/)
and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]
### Added
* [656](https://github.com/dbekaert/RAiDER/pull/656) - Example run configuration files available through `raider.py --generate_config <example_name>`.
### Changed
* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
* [662](https://github.com/dbekaert/RAiDER/pull/662) - Bumped dem-stitcher to >= v2.5.6, which updates the URL for reading the Geoid EGM 2008.
### Fixed
* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and added unit tests and bug fixes.
* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
* [658](https://github.com/dbekaert/RAiDER/pull/658) - Fixed opaque error message if a GUNW file is not produced while HyP3 independently against a previous INSAR_ISCE.
* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
* [662](https://github.com/dbekaert/RAiDER/pull/662) - Ensures dem-stitcher to be >= v2.5.6, which updates the url for reading the Geoid EGM 2008.

## [0.5.1]
### Changed
Expand Down
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@ RAiDER has the ability to download weather models from third-parties; some of wh
## 3. Running RAiDER and Documentation

For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs).
We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).
``` raider.py -h ``` provides a help menu and list of example commands to get started.
We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).
``` raider.py -h ``` provides a help menu and set of example run configurations to get started.
The RAiDER scripts are highly modularized in Python and allows for building your own processing workflow.

------
Expand Down
17 changes: 11 additions & 6 deletions test/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,25 +32,30 @@ def pushd(dir):
def update_yaml(dct_cfg:dict, dst:str='temp.yaml'):
""" Write a new yaml file from a dictionary.
Updates parameters in the default 'raider.yaml' file.
Updates parameters in the default 'template.yaml' file.
Each key:value pair will in 'dct_cfg' will overwrite that in the default
"""
import RAiDER, yaml

template_file = os.path.join(
os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml')
run_config_path = os.path.join(
os.path.dirname(RAiDER.__file__),
'cli',
'examples',
'template',
'template.yaml'
)

with open(template_file, 'r') as f:
with open(run_config_path, 'r') as f:
try:
params = yaml.safe_load(f)
except yaml.YAMLError as exc:
print(exc)
raise ValueError(f'Something is wrong with the yaml file {template_file}')
raise ValueError(f'Something is wrong with the yaml file {run_config_path}')

params = {**params, **dct_cfg}

with open(dst, 'w') as fh:
yaml.safe_dump(params, fh, default_flow_style=False)
yaml.safe_dump(params, fh, default_flow_style=False)

return dst

Expand Down
6 changes: 3 additions & 3 deletions test/test_datelist.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os
import shutil
from test import TEST_DIR, WM, update_yaml
from RAiDER.cli.raider import read_template_file
from RAiDER.cli.raider import read_run_config_file

def test_datelist():
SCENARIO_DIR = os.path.join(TEST_DIR, 'datelist')
Expand All @@ -27,7 +27,7 @@ def test_datelist():
}

cfg = update_yaml(dct_group, 'temp.yaml')
param_dict = read_template_file(cfg)
param_dict = read_run_config_file(cfg)
assert param_dict['date_list'] == true_dates


Expand All @@ -52,5 +52,5 @@ def test_datestep():
}

cfg = update_yaml(dct_group, 'temp.yaml')
param_dict = read_template_file(cfg)
param_dict = read_run_config_file(cfg)
assert param_dict['date_list'] == true_dates
73 changes: 73 additions & 0 deletions test/test_generate_config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
'''
Tests the functionality of `raider.py --generate_config <example_file>`.
For any given example file name, the script should produce that run
configuration file in the current working directory, along wihh any
corresponding data files.
If such a file already exists, the script should prompt the user to
confirm overwriting the file.
'''
import pytest

import os
import subprocess
import tempfile
from contextlib import contextmanager


@contextmanager
def cd_to_temp_dir():
old_pwd = os.getcwd()
try:
with tempfile.TemporaryDirectory() as tmpdir:
os.chdir(tmpdir)
yield
finally:
os.chdir(old_pwd)

@pytest.mark.parametrize(
'name,data_files',
[
('template', []),
('example_LA_bbox', []),
('example_LA_GNSS', ['example_LA_GNSS.csv']),
('example_UK_isce', ['example_UK_isce-S1B_OPER_AUX_POEORB_OPOD_20211122T112354_V20211101T225942_20211103T005942.EOF']),
]
)
def test_generate_config(name, data_files):
with cd_to_temp_dir():
subprocess.run(['raider.py', '--generate_config', name])
assert os.path.exists(f'{name}.yaml')
for data_file_name in data_files:
assert os.path.exists(data_file_name)


def test_confirm_overwrite_yes():
with cd_to_temp_dir():
with open('template.yaml', 'w') as f:
f.write('overwrite me')
process = subprocess.Popen(
['raider.py', '--generate_config', 'template'],
stdin=subprocess.PIPE,
)
process.stdin.write(b'y\n')
process.stdin.close()
process.wait()
assert os.path.exists('template.yaml')
with open('template.yaml') as f:
assert f.read() != 'overwrite me'


def test_confirm_overwrite_no():
with cd_to_temp_dir():
with open('template.yaml', 'w') as f:
f.write("don't overwrite me")
process = subprocess.Popen(
['raider.py', '--generate_config', 'template'],
stdin=subprocess.PIPE,
)
process.stdin.write(b'n\n')
process.stdin.close()
process.wait()
assert os.path.exists('template.yaml')
with open('template.yaml') as f:
assert f.read() == "don't overwrite me"
8 changes: 4 additions & 4 deletions test/test_intersect.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ def test_cube_intersect(wm):
time ='13:52:45'
# f_lat, f_lon = makeLatLonGrid([S, N, W, E], 'LA', SCENARIO_DIR, 0.25)

## make the template file
## make the run config file
grp = {
'date_group': {'date_start': date},
'time_group': {'time': time, 'interpolate_time': 'none'},
Expand All @@ -35,7 +35,7 @@ def test_cube_intersect(wm):
'verbose': False,
}

## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp, 'temp.yaml')

## run raider and intersect
Expand Down Expand Up @@ -68,7 +68,7 @@ def test_gnss_intersect(wm):
date = 20200130
time ='13:52:45'

## make the template file
## make the run config file
grp = {
'date_group': {'date_start': date},
'time_group': {'time': time, 'interpolate_time': 'none'},
Expand All @@ -81,7 +81,7 @@ def test_gnss_intersect(wm):
'verbose': False,
}

## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp)

## run raider and intersect
Expand Down
8 changes: 4 additions & 4 deletions test/test_slant.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ def test_slant_proj(weather_model_name):
date = 20200130
time ='13:52:45'

## make the template file
## make the run config file
grp = {
'date_group': {'date_start': date},
'height_group': {'height_levels': [0, 100, 500, 1000]},
Expand All @@ -32,7 +32,7 @@ def test_slant_proj(weather_model_name):
}
}

## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp, 'temp.yaml')

## run raider and intersect
Expand Down Expand Up @@ -66,7 +66,7 @@ def test_ray_tracing(weather_model_name):
date = 20200130
time ='13:52:45'

## make the template file
## make the run config file
grp = {
'date_group': {'date_start': date},
'height_group': {'height_levels': [0, 100, 500, 1000]},
Expand All @@ -83,7 +83,7 @@ def test_ray_tracing(weather_model_name):
}
}

## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp, 'temp.yaml')

## run raider and intersect
Expand Down
4 changes: 2 additions & 2 deletions test/test_temporal_interpolate.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ def test_cube_timemean():
## run raider without interpolation for two exact weather model times
for hr in [hr1, hr2]:
grp['time_group'].update({'time': f'{hr}:00:00'})
## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp)

## run raider for the default date
Expand Down Expand Up @@ -95,7 +95,7 @@ def test_cube_weighting():
## run raider without interpolation for two exact weather model times
for hr in [hr1, hr2]:
grp['time_group'].update({'time': f'{hr}:00:00'})
## generate the default template file and overwrite it with new parms
## generate the default run config file and overwrite it with new parms
cfg = update_yaml(grp)

## run raider for the default date
Expand Down
15 changes: 10 additions & 5 deletions tools/RAiDER/aria/prepFromGUNW.py
Original file line number Diff line number Diff line change
Expand Up @@ -360,19 +360,24 @@ def make_cube(self):
def update_yaml(dct_cfg:dict, dst:str='GUNW.yaml'):
""" Write a new yaml file from a dictionary.
Updates parameters in the default 'raider.yaml' file.
Updates parameters in the default 'template.yaml' file.
Each key:value pair will in 'dct_cfg' will overwrite that in the default
"""

template_file = os.path.join(
os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml')
run_config_path = os.path.join(
os.path.dirname(RAiDER.__file__),
'cli',
'examples',
'template',
'template.yaml'
)

with open(template_file, 'r') as f:
with open(run_config_path, 'r') as f:
try:
params = yaml.safe_load(f)
except yaml.YAMLError as exc:
print(exc)
raise ValueError(f'Something is wrong with the yaml file {template_file}')
raise ValueError(f'Something is wrong with the yaml file {run_config_path}')

params = {**params, **dct_cfg}

Expand Down
66 changes: 33 additions & 33 deletions tools/RAiDER/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,37 +8,37 @@ class AttributeDict(dict):

DEFAULT_DICT = AttributeDict(
dict(
look_dir='right',
date_start=None,
date_end=None,
date_step=None,
date_list=None,
time=None,
end_time=None,
weather_model=None,
lat_file=None,
lon_file=None,
station_file=None,
bounding_box=None,
geocoded_file=None,
dem=None,
use_dem_latlon=False,
height_levels=None,
height_file_rdr=None,
ray_trace=False,
zref=_ZREF,
cube_spacing_in_m=_CUBE_SPACING_IN_M,
los_file=None,
los_convention='isce',
los_cube=None,
orbit_file=None,
verbose=True,
raster_format='GTiff',
file_format='GTiff',
download_only=False,
output_directory='.',
weather_model_directory=None,
output_projection='EPSG:4326',
interpolate_time='center_time',
)
look_dir='right',
date_start=None,
date_end=None,
date_step=None,
date_list=None,
time=None,
end_time=None,
weather_model=None,
lat_file=None,
lon_file=None,
station_file=None,
bounding_box=None,
geocoded_file=None,
dem=None,
use_dem_latlon=False,
height_levels=None,
height_file_rdr=None,
ray_trace=False,
zref=_ZREF,
cube_spacing_in_m=_CUBE_SPACING_IN_M,
los_file=None,
los_convention='isce',
los_cube=None,
orbit_file=None,
verbose=True,
raster_format='GTiff',
file_format='GTiff',
download_only=False,
output_directory='.',
weather_model_directory=None,
output_projection='EPSG:4326',
interpolate_time='center_time',
)
)
Loading

0 comments on commit e4131dc

Please sign in to comment.