From 34f9c6c5754e06824bec6bc26cfe51677f174997 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Sat, 3 Dec 2022 13:50:00 -0900 Subject: [PATCH 01/82] add missing deps --- environment.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/environment.yml b/environment.yml index 8f5e10837..e37e42732 100644 --- a/environment.yml +++ b/environment.yml @@ -31,11 +31,14 @@ dependencies: - progressbar - pydap>3.2.2 - pyproj>=2.2.0 + - pyyaml - rasterio>=1.3.0 + - requests - s3fs - scipy - shapely - sysroot_linux-64 + - tqdm - xarray # For packaging and testing - autopep8 From d71ad2d48473122951eda7a2be326cd7306d9f0c Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 7 Dec 2022 15:28:50 -0800 Subject: [PATCH 02/82] Update environment for dem-stitcher --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index e37e42732..7c02e4354 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,7 @@ dependencies: - cxx-compiler - cython - dask - - dem_stitcher>=2.3.0 + - dem_stitcher>=2.3.1 - ecmwf-api-client - h5py - herbie-data From 45fff89d8161b3f07629a687de8c8d4f3abfab85 Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 7 Dec 2022 15:31:08 -0800 Subject: [PATCH 03/82] Update CHANGELOG.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 185a25b9b..e34c9619d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.2.1] + +* Update dem-stitcher to use version with new urls for GLO-30. + ## [0.2.0] RAiDER package was refactored to use a configure file (yaml) to parse parameters. In addition, ocker container images From a0ad031d321baf28b409037e13848414b87d290e Mon Sep 17 00:00:00 2001 From: Charlie Marshak Date: Wed, 7 Dec 2022 17:03:38 -0800 Subject: [PATCH 04/82] Update CHANGELOG.md Co-authored-by: Joseph H Kennedy --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e34c9619d..a5490b032 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.2.1] -* Update dem-stitcher to use version with new urls for GLO-30. +* Upgrade dem-stitcher to [`>=2.3.1`](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/blob/dev/CHANGELOG.md#231) so that the updated urls for the GLO-30 DEM are used. ## [0.2.0] From 49ffd6ff8980f95db348d549f7c886eaa9ae3107 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 5 Dec 2022 22:00:09 -0600 Subject: [PATCH 05/82] rearrange to make library access clearer --- tools/RAiDER/cli/raider.py | 44 +++++- tools/RAiDER/delay.py | 276 ++++++++++++++++--------------------- 2 files changed, 153 insertions(+), 167 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index b417732b7..883c5cf55 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -6,13 +6,14 @@ import re, glob import RAiDER +from RAiDER.checkArgs import checkArgs +from RAiDER.cli.validators import ( + enforce_time, enforce_bbox, parse_dates, get_query_region, get_heights, get_los, enforce_wm +) from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M +from RAiDER.delay import tropo_delay from RAiDER.logger import logger, logging -from RAiDER.cli.validators import (enforce_time, enforce_bbox, parse_dates, - get_query_region, get_heights, get_los, enforce_wm) - -from RAiDER.checkArgs import checkArgs -from RAiDER.delay import main as main_delay +from RAiDER.processWM import prepareWeatherModel HELP_MESSAGE = """ @@ -231,6 +232,7 @@ def drop_nans(d): return d + ########################################################################## def main(iargs=None): # parse @@ -253,8 +255,38 @@ def main(iargs=None): params['wetFilenames'], params['hydroFilenames'] ): + + los = params['los'] + aoi = params['aoi'] + model = params['weather_model'] + + if los.ray_trace(): + ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing + else: + ll_bounds = aoi.bounds() + + ########################################################### + # weather model calculation + logger.debug('Starting to run the weather model calculation') + logger.debug('Time: {}'.format(dt.strftime('%Y%m%d'))) + logger.debug('Beginning weather model pre-processing') try: - (_, _) = main_delay(t, w, f, params) + weather_model_file = prepareWeatherModel( + model, t, + ll_bounds=ll_bounds, # SNWE + wmLoc=params['weather_model_directory'], + zref=params['zref'], + makePlots=params['verbose'], + ) except RuntimeError: logger.exception("Date %s failed", t) continue + + # Now process the delays + try: + tropo_delay(t, w, f, weather_model_file, aoi, los, params) + except RuntimeError: + logger.exception("Date %s failed", t) + continue + + diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 10a957163..19d12d653 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -16,23 +16,130 @@ from pyproj.exceptions import CRSError import isce3.ext.isce3 as isce + from RAiDER.constants import _STEP from RAiDER.delayFcns import ( getInterpolators, - calculate_start_points, - get_delays, ) -from RAiDER.dem import getHeights -from RAiDER.logger import logger -from RAiDER.llreader import BoundingBox +from RAiDER.logger import logger, logging from RAiDER.losreader import Zenith, Conventional, Raytracing, get_sv, getTopOfAtmosphere -from RAiDER.processWM import prepareWeatherModel from RAiDER.utilFcns import ( - writeDelays, projectDelays, writePnts2HDF5, - lla2ecef, transform_bbox, clip_bbox, rio_profile, + writeDelays, lla2ecef, transform_bbox, clip_bbox, rio_profile, ) +############################################################################### +def tropo_delay(dt, wetFilename, hydroFilename, weather_model_file, aoi, los, params): + """ + raider main function for calculating delays. + + Parameterss + ---------- + args: dict Parameters and inputs needed for processing, + containing the following key-value pairs: + + los - LOS-class object + aoi - AOI-class object + heights - see checkArgs for format + weather_model_file - file containing pre-processed weather model data + params - runtime parameters + """ + # unpacking the dictionairy + zref = params['zref'] + outformat = params['raster_format'] + + logger.debug('Max integration height is {:1.1f} m'.format(zref)) + + if aoi.type() == 'bounding_box' or \ + (params['height_levels'] and aoi.type() != 'station_file'): + # This branch is specifically for cube generation + try: + #TODO: expose this as library function + tropo_delay_cube( + dt, wetFilename, params, + model_file=weather_model_file, + ) + except Exception as e: + logger.error(e) + raise RuntimeError('Something went wrong in calculating delays on the cube') + return None, None + + ########################################################### + # Load the downloaded model file for CRS information + wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] + if wm_proj is None: + print("WARNING: I can't find a CRS in the weather model file, so I will assume you are using WGS84") + wm_proj = CRS.from_epsg(4326) + else: + wm_proj = CRS.from_wkt(wm_proj.to_wkt()) + #################################################################### + + #################################################################### + # Calculate delays + if isinstance(los, (Zenith, Conventional)): + # Start actual processing here + logger.debug("Beginning DEM calculation") + # Lats, Lons will be translated from file to array here if needed + + # read lats/lons + lats, lons = aoi.readLL() + hgts = aoi.readZ() + + los.setPoints(lats, lons, hgts) + + # Transform query points if needed + pnt_proj = CRS.from_epsg(4326) + if wm_proj != pnt_proj: + pnts = transformPoints( + lats, + lons, + hgts, + pnt_proj, + wm_proj, + ).T + else: + # interpolators require y, x, z + pnts = np.stack([lats, lons, hgts], axis=-1) + + # either way I'll need the ZTD + ifWet, ifHydro = getInterpolators(weather_model_file, 'total') + wetDelay = ifWet(pnts) + hydroDelay = ifHydro(pnts) + + # return the delays (ZTD or STD) + wetDelay = los(wetDelay) + hydroDelay = los(hydroDelay) + + elif isinstance(los, Raytracing): + raise NotImplementedError + else: + raise ValueError("Unknown operation type") + + ########################################################### + # Write the delays to file + # Different options depending on the inputs + + if not isinstance(wetFilename, str): + wetFilename = wetFilename[0] + hydroFilename = hydroFilename[0] + + if aoi.type() == 'station_file': + wetFilename = f'{os.path.splitext(wetFilename)[0]}.csv' + + writeDelays( + aoi, + wetDelay, + hydroDelay, + wetFilename, + hydroFilename, + outformat=outformat, + ) + logger.info('Finished writing data to file') + + return wetDelay, hydroDelay + + + def tropo_delay_cube(dt, wf, args, model_file=None): """ raider cube generation function. @@ -554,156 +661,3 @@ def build_cube_ray(xpts, ypts, zpts, ref_time, orbit_file, look_dir, model_crs, return outputArrs -############################################################################### -def main(dt, wetFilename, hydroFilename, args): - """ - raider main function for calculating delays. - - Parameterss - ---------- - args: dict Parameters and inputs needed for processing, - containing the following key-value pairs: - - los - tuple, Zenith class object, ('los', 2-band los file), or ('sv', orbit_file) - lats - ndarray or str - lons - ndarray or str - heights - see checkArgs for format - weather_model - type of weather model to use - wmLoc - Directory containing weather model files - zref - max integration height - outformat - File format to use for raster outputs - time - list of datetimes to calculate delays - download_only - Only download the raw weather model data and exit - wetFilename - - hydroFilename - - pnts_file - Input a points file from previous run - verbose - verbose printing - """ - # unpacking the dictionairy - los = args['los'] - heights = args['dem'] - weather_model = args['weather_model'] - wmLoc = args['weather_model_directory'] - zref = args['zref'] - outformat = args['raster_format'] - verbose = args['verbose'] - aoi = args['aoi'] - - download_only = args['download_only'] - - if los.ray_trace(): - ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing - else: - ll_bounds = aoi.bounds() - - # logging - logger.debug('Starting to run the weather model calculation') - logger.debug('Time type: {}'.format(type(dt))) - logger.debug('Time: {}'.format(dt.strftime('%Y%m%d'))) - logger.debug('Max integration height is {:1.1f} m'.format(zref)) - - ########################################################### - # weather model calculation - logger.debug('Beginning weather model pre-processing') - - weather_model_file = prepareWeatherModel( - weather_model, - dt, - wmLoc=wmLoc, - ll_bounds=ll_bounds, # SNWE - zref=zref, - download_only=download_only, - makePlots=verbose, - ) - - if download_only: - logger.debug('Weather model has downloaded. Finished.') - return None, None - - - if aoi.type() == 'bounding_box' or \ - (args['height_levels'] and aoi.type() != 'station_file'): - # This branch is specifically for cube generation - try: - tropo_delay_cube( - dt, wetFilename, args, - model_file=weather_model_file, - ) - except Exception as e: - logger.error(e) - raise RuntimeError('Something went wrong in calculating delays on the cube') - return None, None - - ########################################################### - # Load the downloaded model file for CRS information - wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] - if wm_proj is None: - print("WARNING: I can't find a CRS in the weather model file, so I will assume you are using WGS84") - wm_proj = CRS.from_epsg(4326) - else: - wm_proj = CRS.from_wkt(wm_proj.to_wkt()) - #################################################################### - - #################################################################### - # Calculate delays - if isinstance(los, (Zenith, Conventional)): - # Start actual processing here - logger.debug("Beginning DEM calculation") - # Lats, Lons will be translated from file to array here if needed - - # read lats/lons - lats, lons = aoi.readLL() - hgts = aoi.readZ() - - los.setPoints(lats, lons, hgts) - - # Transform query points if needed - pnt_proj = CRS.from_epsg(4326) - if wm_proj != pnt_proj: - pnts = transformPoints( - lats, - lons, - hgts, - pnt_proj, - wm_proj, - ).T - else: - # interpolators require y, x, z - pnts = np.stack([lats, lons, hgts], axis=-1) - - # either way I'll need the ZTD - ifWet, ifHydro = getInterpolators(weather_model_file, 'total') - wetDelay = ifWet(pnts) - hydroDelay = ifHydro(pnts) - - # return the delays (ZTD or STD) - wetDelay = los(wetDelay) - hydroDelay = los(hydroDelay) - - elif isinstance(los, Raytracing): - raise NotImplementedError - else: - raise ValueError("Unknown operation type") - - ########################################################### - # Write the delays to file - # Different options depending on the inputs - - if not isinstance(wetFilename, str): - wetFilename = wetFilename[0] - hydroFilename = hydroFilename[0] - - if aoi.type() == 'station_file': - wetFilename = f'{os.path.splitext(wetFilename)[0]}.csv' - - writeDelays( - aoi, - wetDelay, - hydroDelay, - wetFilename, - hydroFilename, - outformat=outformat, - ) - logger.info('Finished writing data to file') - - return wetDelay, hydroDelay From a4c41a738e7654d3b4b10452b5a5113f932740ee Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 5 Dec 2022 22:02:13 -0600 Subject: [PATCH 06/82] remove unneeded import --- tools/RAiDER/cli/raider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 883c5cf55..7ff0c8b02 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -8,7 +8,7 @@ import RAiDER from RAiDER.checkArgs import checkArgs from RAiDER.cli.validators import ( - enforce_time, enforce_bbox, parse_dates, get_query_region, get_heights, get_los, enforce_wm + enforce_time, parse_dates, get_query_region, get_heights, get_los, enforce_wm ) from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M from RAiDER.delay import tropo_delay From a9f1404d8b51479cbb87b4bb1719e86fe9873354 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 5 Dec 2022 22:12:13 -0600 Subject: [PATCH 07/82] fix small typo --- tools/RAiDER/cli/raider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 7ff0c8b02..5c5cd9c18 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -268,7 +268,7 @@ def main(iargs=None): ########################################################### # weather model calculation logger.debug('Starting to run the weather model calculation') - logger.debug('Time: {}'.format(dt.strftime('%Y%m%d'))) + logger.debug('Time: {}'.format(t.strftime('%Y%m%d'))) logger.debug('Beginning weather model pre-processing') try: weather_model_file = prepareWeatherModel( From 19c368f4d7f1136372dc4e2e97457ef3739754d3 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 11:18:30 -0600 Subject: [PATCH 08/82] refactor to aid in functionalizing --- tools/RAiDER/cli/raider.py | 39 +++++- tools/RAiDER/delay.py | 277 +++++++++++-------------------------- tools/RAiDER/delayFcns.py | 29 ++-- tools/RAiDER/llreader.py | 5 +- tools/RAiDER/losreader.py | 5 + 5 files changed, 142 insertions(+), 213 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 5c5cd9c18..7f9a28bad 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -14,6 +14,7 @@ from RAiDER.delay import tropo_delay from RAiDER.logger import logger, logging from RAiDER.processWM import prepareWeatherModel +from RAiDER.utilFcns import writeDelays HELP_MESSAGE = """ @@ -284,9 +285,45 @@ def main(iargs=None): # Now process the delays try: - tropo_delay(t, w, f, weather_model_file, aoi, los, params) + wet_delay, hydro_delay = tropo_delay( + t, weather_model_file, aoi, los, + params['height_levels'], + params['output_projection'], + params['look_dir'], + params['cube_spacing_in_m'] + ) except RuntimeError: logger.exception("Date %s failed", t) continue + + ########################################################### + # Write the delays to file + # Different options depending on the inputs + + if not los.ray_trace() and not los.is_Zenith(): + out_filename = w.replace("_ztd", "_std") + else: + out_filename = w.replace("_ztd", "_ray") + + if hydro_delay is None: + # means that a dataset was returned + ds = wet_delay + ext = os.path.splitext(out_filename) + if ext not in ['.nc', '.h5']: + out_filename = f'{os.path.splitext(out_filename)[0]}.nc' + + out_filename = out_filename.replace("wet", "tropo") + + if out_filename.endswith(".nc"): + ds.to_netcdf(out_filename, mode="w") + elif out_filename.endswith(".h5"): + ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) + + else: + if aoi.type() == 'station_file': + w = f'{os.path.splitext(w)[0]}.csv' + + if aoi.type() in ['station_file', 'radar_rasters']: + writeDelays(aoi, wet_delay, hydro_delay, w, f, outformat=params['raster_format']) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 19d12d653..9757e953b 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -11,9 +11,11 @@ import h5py import numpy as np import xarray + from netCDF4 import Dataset from pyproj import CRS, Transformer from pyproj.exceptions import CRSError +from scipy.interpolate import RegularGridInterpolator as Interpolator import isce3.ext.isce3 as isce @@ -24,170 +26,65 @@ from RAiDER.logger import logger, logging from RAiDER.losreader import Zenith, Conventional, Raytracing, get_sv, getTopOfAtmosphere from RAiDER.utilFcns import ( - writeDelays, lla2ecef, transform_bbox, clip_bbox, rio_profile, + lla2ecef, transform_bbox, clip_bbox, rio_profile, ) ############################################################################### -def tropo_delay(dt, wetFilename, hydroFilename, weather_model_file, aoi, los, params): +def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4326, look_dir='right', cube_spacing_m=None): """ raider main function for calculating delays. Parameterss ---------- - args: dict Parameters and inputs needed for processing, - containing the following key-value pairs: - - los - LOS-class object - aoi - AOI-class object - heights - see checkArgs for format - weather_model_file - file containing pre-processed weather model data - params - runtime parameters + dt: Datetime - Datetime object for determining when to calculate delays + weather_model_File: string - Name of the NETCDF file containing a pre-processed weather model + aoi: AOI object - AOI object + los: LOS object - LOS object + height_levels: list - (optional) list of height levels on which to calculate delays. Only needed for cube generation. + out_proj: int,str - (optional) EPSG code for output projection + look_dir: str - (optional) Satellite look direction. Only needed for slant delay calculation + cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes """ - # unpacking the dictionairy - zref = params['zref'] - outformat = params['raster_format'] - - logger.debug('Max integration height is {:1.1f} m'.format(zref)) - - if aoi.type() == 'bounding_box' or \ - (params['height_levels'] and aoi.type() != 'station_file'): - # This branch is specifically for cube generation - try: - #TODO: expose this as library function - tropo_delay_cube( - dt, wetFilename, params, - model_file=weather_model_file, - ) - except Exception as e: - logger.error(e) - raise RuntimeError('Something went wrong in calculating delays on the cube') - return None, None - - ########################################################### - # Load the downloaded model file for CRS information - wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] - if wm_proj is None: - print("WARNING: I can't find a CRS in the weather model file, so I will assume you are using WGS84") - wm_proj = CRS.from_epsg(4326) - else: - wm_proj = CRS.from_wkt(wm_proj.to_wkt()) - #################################################################### + # get heights + if height_levels is None: + with xarray.load_dataset(weather_model_file) as ds: + height_levels = ds.z.values - #################################################################### - # Calculate delays - if isinstance(los, (Zenith, Conventional)): - # Start actual processing here - logger.debug("Beginning DEM calculation") - # Lats, Lons will be translated from file to array here if needed + #TODO: expose this as library function + ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, los, out_proj = out_proj, cube_spacing_m = cube_spacing_m, look_dir = look_dir) - # read lats/lons - lats, lons = aoi.readLL() - hgts = aoi.readZ() - - los.setPoints(lats, lons, hgts) + if (aoi.type() == 'bounding_box') or (aoi.type() == 'Geocube'): + return ds, None - # Transform query points if needed + else: + with xarray.load_dataset(weather_model_file) as ds2: + wm_proj = CRS.from_wkt(ds2.CRS.attrs['crs_wkt']) pnt_proj = CRS.from_epsg(4326) - if wm_proj != pnt_proj: - pnts = transformPoints( - lats, - lons, - hgts, - pnt_proj, - wm_proj, - ).T - else: - # interpolators require y, x, z - pnts = np.stack([lats, lons, hgts], axis=-1) - - # either way I'll need the ZTD - ifWet, ifHydro = getInterpolators(weather_model_file, 'total') + lats, lons = aoi.readLL() + hgts = aoi.readZ() + pnts = transformPoints(lats, lons, hgts, pnt_proj, wm_proj).T + ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) # return the delays (ZTD or STD) - wetDelay = los(wetDelay) - hydroDelay = los(hydroDelay) - - elif isinstance(los, Raytracing): - raise NotImplementedError - else: - raise ValueError("Unknown operation type") - - ########################################################### - # Write the delays to file - # Different options depending on the inputs - - if not isinstance(wetFilename, str): - wetFilename = wetFilename[0] - hydroFilename = hydroFilename[0] - - if aoi.type() == 'station_file': - wetFilename = f'{os.path.splitext(wetFilename)[0]}.csv' - - writeDelays( - aoi, - wetDelay, - hydroDelay, - wetFilename, - hydroFilename, - outformat=outformat, - ) - logger.info('Finished writing data to file') + if los.is_Projected(): + los.setPoints(lats, lons, hgts) + wetDelay = los(wetDelay) + hydroDelay = los(hydroDelay) return wetDelay, hydroDelay -def tropo_delay_cube(dt, wf, args, model_file=None): +def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4326, cube_spacing_m=None, look_dir='right', nproc=1): """ raider cube generation function. - - Same as tropo_delay() above. """ - los = args['los'] - weather_model = args['weather_model'] - wmLoc = args['weather_model_directory'] - zref = args['zref'] - download_only = args['download_only'] - verbose = args['verbose'] - aoi = args['aoi'] - cube_spacing = args["cube_spacing_in_m"] - ll_bounds = aoi.bounds() - - try: - crs = CRS(args['output_projection']) - except CRSError: - raise ValueError('output_projection argument is not a valid CRS specifier') - # For testing multiprocessing # TODO - move this to configuration - nproc = 1 - - # logging - logger.debug('Starting to run the weather model cube calculation') - logger.debug(f'Time: {dt}') - logger.debug(f'Max integration height is {zref:1.1f} m') - logger.debug(f'Output cube projection is {crs.to_wkt()}') - logger.debug(f'Output cube spacing is {cube_spacing} m') - - # We are using zenith model only for now - logger.debug('Beginning weather model pre-processing') - - # If weather model file is not provided - if model_file is None: - weather_model_file = prepareWeatherModel( - weather_model, - dt, - wmLoc=wmLoc, - ll_bounds=ll_bounds, - zref=zref, - download_only=download_only, - makePlots=verbose - ) - else: - weather_model_file = model_file + crs = CRS(out_proj) # Determine the output grid extent here wesn = ll_bounds[2:] + ll_bounds[:2] @@ -197,25 +94,25 @@ def tropo_delay_cube(dt, wf, args, model_file=None): # Clip output grid to multiples of spacing # If output is desired in degrees - if crs.axis_info[0].unit_name == "degree": - out_spacing = cube_spacing / 1.0e5 # Scale by 100km - out_snwe = clip_bbox(out_snwe, out_spacing) - else: - out_spacing = cube_spacing - out_snwe = clip_bbox(out_snwe, out_spacing) + use_weather_model_cube = False - logger.debug(f"Output SNWE: {out_snwe}") - logger.debug(f"Output cube spacing: {out_spacing}") + if (cube_spacing_m is not None) or (crs != CRS(4326)): + + use_weather_model_cube = True - # Load downloaded weather model file to get projection info - with xarray.load_dataset(weather_model_file) as ds: - # Output grid points - North up grid - if args['height_levels'] is not None: - heights = args['height_levels'] + #TODO handle this better + if cube_spacing_m is None: + cube_spacing_m = 5000 + + if crs.axis_info[0].unit_name == "degree": + out_spacing = cube_spacing_m / 1.0e5 # Scale by 100km + out_snwe = clip_bbox(out_snwe, out_spacing) else: - heights = ds.z.values + out_spacing = cube_spacing_m + out_snwe = clip_bbox(out_snwe, out_spacing) - logger.debug(f'Output height range is {min(heights)} to {max(heights)}') + logger.debug(f"Output SNWE: {out_snwe}") + logger.debug(f"Output cube spacing: {out_spacing}") # Load CRS from weather model file wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] @@ -227,14 +124,18 @@ def tropo_delay_cube(dt, wf, args, model_file=None): # Build the output grid zpts = np.array(heights) - xpts = np.arange(out_snwe[2], out_snwe[3] + out_spacing, out_spacing) - ypts = np.arange(out_snwe[1], out_snwe[0] - out_spacing, -out_spacing) + if use_weather_model_cube: + xpts = np.arange(out_snwe[2], out_snwe[3] + out_spacing, out_spacing) + ypts = np.arange(out_snwe[1], out_snwe[0] - out_spacing, -out_spacing) + else: + with xarray.load_dataset(weather_model_file) as ds: + xpts = ds.x.values + ypts = ds.y.values # If no orbit is provided # Build zenith delay cube - if los.is_Zenith(): - out_type = "zenith" - out_filename = wf.replace("wet", "tropo") + if los.is_Zenith() or los.is_Projected: + out_type = ["zenith" if los.is_Zenith() else 'slant - projected'][0] # Get ZTD interpolators ifWet, ifHydro = getInterpolators(weather_model_file, "total") @@ -246,16 +147,7 @@ def tropo_delay_cube(dt, wf, args, model_file=None): [ifWet, ifHydro]) else: - out_type = "slant range" - if not los.ray_trace(): - out_filename = wf.replace("_ztd", "_std").replace("wet", "tropo") - else: - out_filename = wf.replace("_ztd", "_ray").replace("wet", "tropo") - - if args["look_dir"].lower() not in ["right", "left"]: - raise ValueError( - f'Unknown look direction: {args["look_dir"]}' - ) + out_type = "slant - raytracing" # Get pointwise interpolators ifWet, ifHydro = getInterpolators( @@ -264,25 +156,22 @@ def tropo_delay_cube(dt, wf, args, model_file=None): shared=(nproc > 1), ) - if los.ray_trace(): - # Build cube - if nproc == 1: - wetDelay, hydroDelay = build_cube_ray( - xpts, ypts, zpts, - dt, args["los"]._file, args["look_dir"], - wm_proj, crs, - [ifWet, ifHydro]) + # Build cube + if nproc == 1: + wetDelay, hydroDelay = build_cube_ray( + xpts, ypts, zpts, + dt, los._file, look_dir, + wm_proj, crs, + [ifWet, ifHydro]) + + ### Use multi-processing here + else: + # Pre-build output arrays - ### Use multi-processing here - else: - # Pre-build output arrays + # Create worker pool - # Create worker pool - - # Loop over heights - raise NotImplementedError - else: - raise NotImplementedError('Conventional STD is not yet implemented on the cube') + # Loop over heights + raise NotImplementedError # Write output file # Modify this as needed for NISAR / other projects @@ -312,7 +201,7 @@ def tropo_delay_cube(dt, wf, args, model_file=None): source=os.path.basename(weather_model_file), history=str(datetime.datetime.utcnow()) + " RAiDER", description=f"RAiDER geo cube - {out_type}", - reference_time=str(args["time"]), + reference_time=str(dt), ), ) @@ -347,19 +236,7 @@ def tropo_delay_cube(dt, wf, args, model_file=None): ds.x.attrs["long_name"] = "x-coordinate in projected coordinate system" ds.x.attrs["units"] = "m" - - ext = os.path.splitext(out_filename) - if ext not in '.nc .h5'.split(): - out_filename = f'{os.path.splitext(out_filename)[0]}.nc' - logger.debug('Invalid extension %s for cube. Defaulting to .nc', ext) - - if out_filename.endswith(".nc"): - ds.to_netcdf(out_filename, mode="w") - elif out_filename.endswith(".h5"): - ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) - - logger.info('Finished writing data to: %s', out_filename) - return + return ds def checkQueryPntsFile(pnts_file, query_shape): @@ -459,8 +336,10 @@ def build_cube(xpts, ypts, zpts, model_crs, pts_crs, interpolators): return outputArrs -def build_cube_ray(xpts, ypts, zpts, ref_time, orbit_file, look_dir, model_crs, - pts_crs, interpolators, outputArrs=None): +def build_cube_ray( + xpts, ypts, zpts, ref_time, orbit_file, look_dir, model_crs, + pts_crs, interpolators, outputArrs=None +): """ Iterate over interpolators and build a cube """ diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index ca1ebc347..bb2c87a91 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -92,18 +92,23 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): an interpolator ''' # Get the weather model data - with Dataset(wm_file, mode='r') as f: - xs_wm = np.array(f.variables['x'][:]) - ys_wm = np.array(f.variables['y'][:]) - zs_wm = np.array(f.variables['z'][:]) - - # Can get the point-wise or total delays, depending on what is requested - if kind == 'pointwise': - wet = np.array(f.variables['wet'][:]).transpose(1, 2, 0) - hydro = np.array(f.variables['hydro'][:]).transpose(1, 2, 0) - elif kind == 'total': - wet = np.array(f.variables['wet_total'][:]).transpose(1, 2, 0) - hydro = np.array(f.variables['hydro_total'][:]).transpose(1, 2, 0) + if isinstance(wm_file, str): + with Dataset(wm_file, mode='r') as ds: + xs_wm = np.array(ds.variables['x'][:]) + ys_wm = np.array(ds.variables['y'][:]) + zs_wm = np.array(ds.variables['z'][:]) + wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'wet'][:] + else: + ds = wm_file + xs_wm = np.array(ds.variables['x'][:]) + ys_wm = np.array(ds.variables['y'][:]) + zs_wm = np.array(ds.variables['z'][:]) + wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'wet'][:] + + wet = np.array(wet).transpose(1, 2, 0) + hydro = np.array(hydro).transpose(1, 2, 0) # If shared interpolators are requested # The arrays are not modified - so turning off lock for performance diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index 25b720b09..9a4865c94 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -40,7 +40,10 @@ def add_buffer(self, buffer): Check whether an extra lat/lon buffer is needed for raytracing ''' # if raytracing, add a 1-degree buffer all around - ll_bounds = self._bounding_box.copy() + try: + ll_bounds = self._bounding_box.copy() + except AttributeError: + ll_bounds = list(self._bounding_box) ll_bounds[0] = np.max([ll_bounds[0] - buffer, -90]) ll_bounds[1] = np.min([ll_bounds[1] + buffer, 90]) ll_bounds[2] = np.max([ll_bounds[2] - buffer,-180]) diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 98bd78b84..83ab06405 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -30,6 +30,7 @@ def __init__(self): self._look_vecs = None self._ray_trace = False self._is_zenith = False + self._is_projected = False def setPoints(self, lats, lons=None, heights=None): '''Set the pixel locations''' @@ -57,6 +58,9 @@ def setTime(self, dt): def is_Zenith(self): return self._is_zenith + def is_Projected(self): + return self._is_projected + def ray_trace(self): return self._ray_trace @@ -90,6 +94,7 @@ def __init__(self, filename=None, los_convention='isce', time=None, pad=None): self._file = filename self._time = time self._pad = pad + self._is_projected = True self._convention = los_convention if self._convention == 'hyp3': raise NotImplementedError() From c9ab92242cca40b34c1071e12950482b1cc030aa Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 11:21:35 -0600 Subject: [PATCH 09/82] mark downloaders as long --- test/test_geom/test_era5.nc | Bin 2216 -> 6456 bytes test/test_geom/test_era5t.nc | Bin 2216 -> 6456 bytes test/test_geom/test_erai.nc | Bin 3256 -> 16732 bytes test/weather_model/test_downloaders.py | 7 +++++-- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/test/test_geom/test_era5.nc b/test/test_geom/test_era5.nc index 89d77b7a5257e4c84aa39e14c96d97655ce37369..b5ca6699a0444ff24963a68c4e16d82e023b7fba 100644 GIT binary patch literal 6456 zcmb_g2~<d1U5F5q-LXCn7)CYo+7C{*dAwVF6Apt}g&P@Or4OScw z4bOsoj#vevR;v~hp(-c@p;jpb=Q`kkpwYtH=OX>B%l^x^R`196?ctt#&b|BW8v=`$ zcxY=49b>_a1didy2_ku-q;T#~9_RGgKlf(h(j&iD4tLhmzauu7(c?r&c@D;Zfi%kaWb`acCoW} zVK|t2F8j(fEY&nJkrx&!iVGERMV#;mx}BAsgO#m|sdZeu$ePQEO^L9MWhV;h96k>g zv<~A5tp9q26{Jk*EYo;)B0JWU9%pLJVRNFm*5Pas+sT?8&W?vWvBGm$ao38ti2^pC zZtLh|!(iCj*fFdeoot=#Y@O(~&hs1`=uY%7XZv~f97h|vb40iuongme)1BE|5JyK_ zr-%slJZC2xD*GPEc9;oQhXE>{@JWeY_xj=Vlpz$P~oV!W5{&xw1$00@WfY>;>R z1F!dPAH2Wb_~G>gafzZRJjI9Ce0NT~o)I}xOP2xm-TuRC;d}hM=Nl8t&U~V|R%@X{v?C|i=xQGZL7k9zi^KSay zXYSOk^~C2xJ)n^OYK~p@FxhH52MBlMQmJ>@b`LCKk~2R;Q2>2kh5y(c7Yv{00IZD*Z)+T z1>d0W=WZ8Yfa^Y3n}uQ`@@jo+BtHKS)@Dphq-jhfl*Se&hT}uEF=WMaIlKrS$22M_ zmKV+wk)NV}Un4`|+wq})9sf?P#_xr-MhtEJ|6Z$wLCmaqh5NpPIX+k`S3#4vf$V7q zKK~EaYJ?X|2*-zNH0~>IY&3D zEA?)4n^U^OjRu?vGyzU6-2wAM&yDDX7V`st_%jLj8~*IV?;rmCY6P4P#9i+J330;JH^SAqC69b02$yRU)*6b)~4d@Yy8O5CBY*38PD6=UvAhuXBW>IEo&CoK14!|%4 ze>xBYM;J4-W@*jDUXOQ=55Tv_3y6W^<$z`2V~fu%?^$u{$gOG;hi@GPd-PV-t?xmP zfISTQJy?vx*BY-iUAuJcBG}7-Ccu?z&A`iVE%qVP@~EY!v|2~)fuxh z=0FS!fH}b8%p8()&RBwHIfRqmCzk;+e89uL59wj`B}r;cSWP%EtcG2~0bm37au4-RD2MKEye+es=(qLwueChMfpWVMTLMou-MB7 z4rPjpHi9Pw1r&}g9ABsp)B|E~EE(yON`cITLJDAfeo6ke{H?%mfZOu7K#Zd;q~bH% z0bujY^DOh`0L_4wc@_}Q0SqaQ=aT0wa-wpgfxMguU=(NsD7KMrW1Nl5iO4p}{v>-6 z=)~;Fq#A-w$~Gi(*y3nnwxOg!^0VYTs1j^F=sD2yph{4sL_ty^QHXQISzeZPzY@_NYDsl@W#M=_Jj#BMO*;L5eJF)iIrlx zxI#jeFvL&9osyFhf5}8~cb1N1r(}Zo`Ye|>k9%bLn$Nt(vLb&tf#x;-3Vr>c?Ot)c#}eF>~QDANvd+vnC#c`-1F< zx86{nX#d$U!=0m#^wD_0u*E*c1n4&kSOf5|?}(g`3;F^%fOQ7Xfy_7|d&B_W0ojq( z5$!@Uz%J63qn%*!*MyyrBflA|;ZeDzMGyCsYmBbfHFc7S2CKzkzp}m_Up|1Lz4c3cCFb>LErGI_MR8 z2pzA5`p^SHAL1A20nD-&?LgaM)?H{5+6*h_q8vam+JLfA2FgSDNG5Z+h(LvqDTL@d z^ex&7SIE&JxclAc7t{oIEGO0wA;fZcPxA?ah#&%pC&X=HBatqaiYR0Hay5N%HEyrSg4&pE!TU)Z;}AXY>B-jOd^A~ z@wsHWg@juY^N8bfrj^ran>}-He}2l`%gn;U%3_=Gc@vG> z8|Q;g^K2VjeqOZJEs|N`yu*#lEOB&q@OQe&m`HzN8)IK%Hfbhxk*{Nejf6hyi#$*J zCI5EsV*a*N*2Q###oS5x|IHR7Lnj?c*g!9pv7R67beXDI=Ff z`ST`6{StM8XURLkJIc$89vQua*GBu@OvlW~%#v14`+;_erlxUe*|c4>A89@`A#EqE zo)$*iOlvUHGW@|X)zHmwqG73Fu%Vg3Q$wDiy}@gP6NYmP`V5X6L>s6Kb{Yg2XumoB zN5MeM;1_?aA6PV~J9v7)d*I$+u9hC?s=?D*lLnmzg@aXtjh#iEzjn><=C}PZP_)mt5~h4&g9pWmyvu3W0DJMW^XRE|?7DBGdtkCY3Q8*55x5^8L!TWV+4 z&Zsr2xm}xEd$hK(_FAn{PLZ#WquN_?4|%D)Q^u8Tlhw-VWy!LWvPb2kWofdea>H^r znNX%IpH%KAi;(S?^_Dvo_7_EOoGH~SbSTUyyjzrCz%Mvjs8h(zFUwaKI207->*g>)+M+8C_kv36HBoO0JJRu;g(QSkXJ~0=mk(h`gmJ$?nLRu!3Ne@d~r01k{(u>kt z(o@pw(ig>s(sGDi7n@7lq+L=?@svuN%E-#f%C<_AN>=5j%AYG9R+?4@SC&@nskmM_ zq0+M=y<$_vk&0{b+wx9%ll-v!9(Xt8$B${td*wayUym6c(>taoAC$j1HsP4bF>6(Z z3aN5bV%0v?R#mBLr|PasrMjq6v>LWjRIgNbTj{NHTc@^C)KYblI#HdhzM`&HA68d& zk5<1_zfeEuHtV+RrgzWkp5Of?;0ui_z)j0((KV3(KKo*G^aFIG`*T? z%|*@a*W)PnC=HbFDeEW~C>@l`6dCvp6gg!J4%n?=0EUS7HvmKlgagZN1sQCN2y1?N49&md!GAd_eS3|-(|i(`_}vN zz^?IW_l@>l>}%~)=WF0|(C4~O%d&2tX+GgTE5CAPsaXB2Jzq^{#jtW%^I3>x$?|70 z{Eqvb@q6gk>$leLf?q#tbilp9uLAc5ngrAb)CUFyP7g2+NDZ79m>OUeusz^@U}S(I zP$$qO@X+dnppIbgpsTBUgV@3IgFjh)BPb}?CwNa#T+qE>kKpcLeaO~_ppcF=M(jmw z&rq#3)7T6)J+ysIEPDa_@|rgGRQ92;jqE`7?yv~XsIau~DcmL8=&%sZ0N0E=gF}Qb z=FZ{fgkR@6aDR&k0+H}+(#f9!67pJ1baDG&%^1-}SR3K|4=<1WRO#XH0&#ka>NCoD`zN{CGumq1Cd zOZ+rZk$60jmKd9ONO)8z5#AN1iUdiYCE1Ijld6*)l8z?XidKj&C&#BONnV*8oxDG} zHl-t_KczRtB9)SQDYbfSX=>{_l=?i?Amvm_W=i|oAJ%f-KaF%v~lZKt!qtd zOzTJ+OdFYYH{CjIVBL%L_UoP3m!)mXFv)O9Uz{-w{!>w(@icRG#t97wP3D78UDFp3TUu2yuhC zrJlKo0mv?h6O1f$foAHZ7iA_*KEYVb$THc6DW8)C=pi6r*OgQfuyRG5! ts)s*+UE`OMUZ!!>&fEQ=|EJ(vkssoWlWfzxbIl6fO3zj4)mDJa0suP!+baM7 diff --git a/test/test_geom/test_era5t.nc b/test/test_geom/test_era5t.nc index 89d77b7a5257e4c84aa39e14c96d97655ce37369..b5ca6699a0444ff24963a68c4e16d82e023b7fba 100644 GIT binary patch literal 6456 zcmb_g2~<d1U5F5q-LXCn7)CYo+7C{*dAwVF6Apt}g&P@Or4OScw z4bOsoj#vevR;v~hp(-c@p;jpb=Q`kkpwYtH=OX>B%l^x^R`196?ctt#&b|BW8v=`$ zcxY=49b>_a1didy2_ku-q;T#~9_RGgKlf(h(j&iD4tLhmzauu7(c?r&c@D;Zfi%kaWb`acCoW} zVK|t2F8j(fEY&nJkrx&!iVGERMV#;mx}BAsgO#m|sdZeu$ePQEO^L9MWhV;h96k>g zv<~A5tp9q26{Jk*EYo;)B0JWU9%pLJVRNFm*5Pas+sT?8&W?vWvBGm$ao38ti2^pC zZtLh|!(iCj*fFdeoot=#Y@O(~&hs1`=uY%7XZv~f97h|vb40iuongme)1BE|5JyK_ zr-%slJZC2xD*GPEc9;oQhXE>{@JWeY_xj=Vlpz$P~oV!W5{&xw1$00@WfY>;>R z1F!dPAH2Wb_~G>gafzZRJjI9Ce0NT~o)I}xOP2xm-TuRC;d}hM=Nl8t&U~V|R%@X{v?C|i=xQGZL7k9zi^KSay zXYSOk^~C2xJ)n^OYK~p@FxhH52MBlMQmJ>@b`LCKk~2R;Q2>2kh5y(c7Yv{00IZD*Z)+T z1>d0W=WZ8Yfa^Y3n}uQ`@@jo+BtHKS)@Dphq-jhfl*Se&hT}uEF=WMaIlKrS$22M_ zmKV+wk)NV}Un4`|+wq})9sf?P#_xr-MhtEJ|6Z$wLCmaqh5NpPIX+k`S3#4vf$V7q zKK~EaYJ?X|2*-zNH0~>IY&3D zEA?)4n^U^OjRu?vGyzU6-2wAM&yDDX7V`st_%jLj8~*IV?;rmCY6P4P#9i+J330;JH^SAqC69b02$yRU)*6b)~4d@Yy8O5CBY*38PD6=UvAhuXBW>IEo&CoK14!|%4 ze>xBYM;J4-W@*jDUXOQ=55Tv_3y6W^<$z`2V~fu%?^$u{$gOG;hi@GPd-PV-t?xmP zfISTQJy?vx*BY-iUAuJcBG}7-Ccu?z&A`iVE%qVP@~EY!v|2~)fuxh z=0FS!fH}b8%p8()&RBwHIfRqmCzk;+e89uL59wj`B}r;cSWP%EtcG2~0bm37au4-RD2MKEye+es=(qLwueChMfpWVMTLMou-MB7 z4rPjpHi9Pw1r&}g9ABsp)B|E~EE(yON`cITLJDAfeo6ke{H?%mfZOu7K#Zd;q~bH% z0bujY^DOh`0L_4wc@_}Q0SqaQ=aT0wa-wpgfxMguU=(NsD7KMrW1Nl5iO4p}{v>-6 z=)~;Fq#A-w$~Gi(*y3nnwxOg!^0VYTs1j^F=sD2yph{4sL_ty^QHXQISzeZPzY@_NYDsl@W#M=_Jj#BMO*;L5eJF)iIrlx zxI#jeFvL&9osyFhf5}8~cb1N1r(}Zo`Ye|>k9%bLn$Nt(vLb&tf#x;-3Vr>c?Ot)c#}eF>~QDANvd+vnC#c`-1F< zx86{nX#d$U!=0m#^wD_0u*E*c1n4&kSOf5|?}(g`3;F^%fOQ7Xfy_7|d&B_W0ojq( z5$!@Uz%J63qn%*!*MyyrBflA|;ZeDzMGyCsYmBbfHFc7S2CKzkzp}m_Up|1Lz4c3cCFb>LErGI_MR8 z2pzA5`p^SHAL1A20nD-&?LgaM)?H{5+6*h_q8vam+JLfA2FgSDNG5Z+h(LvqDTL@d z^ex&7SIE&JxclAc7t{oIEGO0wA;fZcPxA?ah#&%pC&X=HBatqaiYR0Hay5N%HEyrSg4&pE!TU)Z;}AXY>B-jOd^A~ z@wsHWg@juY^N8bfrj^ran>}-He}2l`%gn;U%3_=Gc@vG> z8|Q;g^K2VjeqOZJEs|N`yu*#lEOB&q@OQe&m`HzN8)IK%Hfbhxk*{Nejf6hyi#$*J zCI5EsV*a*N*2Q###oS5x|IHR7Lnj?c*g!9pv7R67beXDI=Ff z`ST`6{StM8XURLkJIc$89vQua*GBu@OvlW~%#v14`+;_erlxUe*|c4>A89@`A#EqE zo)$*iOlvUHGW@|X)zHmwqG73Fu%Vg3Q$wDiy}@gP6NYmP`V5X6L>s6Kb{Yg2XumoB zN5MeM;1_?aA6PV~J9v7)d*I$+u9hC?s=?D*lLnmzg@aXtjh#iEzjn><=C}PZP_)mt5~h4&g9pWmyvu3W0DJMW^XRE|?7DBGdtkCY3Q8*55x5^8L!TWV+4 z&Zsr2xm}xEd$hK(_FAn{PLZ#WquN_?4|%D)Q^u8Tlhw-VWy!LWvPb2kWofdea>H^r znNX%IpH%KAi;(S?^_Dvo_7_EOoGH~SbSTUyyjzrCz%Mvjs8h(zFUwaKI207->*g>)+M+8C_kv36HBoO0JJRu;g(QSkXJ~0=mk(h`gmJ$?nLRu!3Ne@d~r01k{(u>kt z(o@pw(ig>s(sGDi7n@7lq+L=?@svuN%E-#f%C<_AN>=5j%AYG9R+?4@SC&@nskmM_ zq0+M=y<$_vk&0{b+wx9%ll-v!9(Xt8$B${td*wayUym6c(>taoAC$j1HsP4bF>6(Z z3aN5bV%0v?R#mBLr|PasrMjq6v>LWjRIgNbTj{NHTc@^C)KYblI#HdhzM`&HA68d& zk5<1_zfeEuHtV+RrgzWkp5Of?;0ui_z)j0((KV3(KKo*G^aFIG`*T? z%|*@a*W)PnC=HbFDeEW~C>@l`6dCvp6gg!J4%n?=0EUS7HvmKlgagZN1sQCN2y1?N49&md!GAd_eS3|-(|i(`_}vN zz^?IW_l@>l>}%~)=WF0|(C4~O%d&2tX+GgTE5CAPsaXB2Jzq^{#jtW%^I3>x$?|70 z{Eqvb@q6gk>$leLf?q#tbilp9uLAc5ngrAb)CUFyP7g2+NDZ79m>OUeusz^@U}S(I zP$$qO@X+dnppIbgpsTBUgV@3IgFjh)BPb}?CwNa#T+qE>kKpcLeaO~_ppcF=M(jmw z&rq#3)7T6)J+ysIEPDa_@|rgGRQ92;jqE`7?yv~XsIau~DcmL8=&%sZ0N0E=gF}Qb z=FZ{fgkR@6aDR&k0+H}+(#f9!67pJ1baDG&%^1-}SR3K|4=<1WRO#XH0&#ka>NCoD`zN{CGumq1Cd zOZ+rZk$60jmKd9ONO)8z5#AN1iUdiYCE1Ijld6*)l8z?XidKj&C&#BONnV*8oxDG} zHl-t_KczRtB9)SQDYbfSX=>{_l=?i?Amvm_W=i|oAJ%f-KaF%v~lZKt!qtd zOzTJ+OdFYYH{CjIVBL%L_UoP3m!)mXFv)O9Uz{-w{!>w(@icRG#t97wP3D78UDFp3TUu2yuhC zrJlKo0mv?h6O1f$foAHZ7iA_*KEYVb$THc6DW8)C=pi6r*OgQfuyRG5! ts)s*+UE`OMUZ!!>&fEQ=|EJ(vkssoWlWfzxbIl6fO3zj4)mDJa0suP!+baM7 diff --git a/test/test_geom/test_erai.nc b/test/test_geom/test_erai.nc index 2865fe82e3b1981a0b9b7f4daed2a688541cbaa0..376985019065f684bce04c388b875a5a5b53091f 100644 GIT binary patch literal 16732 zcmeHO2UJtb)~1OF8mYpzIWeU-uKr3)?0sBi`n^R_LMVwpKoT*37i#IHxD@( z85vmsWdH>Lr9>))NMbN!2%rMeG=Le2IL7Zyiqf`30*jC+?Uj*n0#IO($Ule6sDQe3 zPM9l|!Xi)@Br1gts*`Pst{y1c1&gFrndxyPI)h5%NOdq?7KOG!qfxeKl+9uU+7^Sh z!Pq+@JbnEU(HsPkMv9JPP$MY>20kVhg@Qr)Gq8432@brsHWi20qSOI-M=r zW-&^dTieH2N0MZAK+#!vRCFSffU>c1L@kOXEX3pRwm7>*b_Bfb zLiEDeh4yy#cF~LQc*ocn2h?(!mu~`Rfz%%aiuzA25*AuVO2?spI{i-!FZ$PpeHaOj zfyoOfcm#_1cd}uCyBPp~UNh->O4ILaF^x$fF(y~x$2zFQ5QsDafgVY~(HUUeq-}@AfEE8Y2!21nE~c|2U`A-t5VdGBeac z`NuscwI%gYYEP>FYOYyiY78MUa&o6eQkdju0u7A(vB$?IHJKSmY5gai(s|?oq;pEk zKXYajM`hCJ2s(*^Cm?JcZER3Bw%~)XvB6A!7TAD`K>C$0zuTCB$0ZUdF*vCwlN^Ip zAW~^09A)y-$@r7TAV^Bmdsj%L(37MkfYc}HHBvvNN$RIG7FFodNh4!%co0oL+hmtj zhW)0!a`_`rSH{J}L{ej8=>+LY|F}lh#auQ*^Y&h*v_H~=l$aQZ12Hh!FSR4f`(Z_u zOrnDuCPuRU);;OR@BM$~NH$|~ieGXt*M~~P(MXIqGJ+b5pfhQ}V*(2$tF+ge|i7YLHDoke><&EhAk_d_LukXETAVotC`p&?f(znzfuAbkwCPx zMA`oE`&WBmre6{PPl_es5phg1DTc(DJOO27ROt+ybZf-?DGq1Se~yp*$MJvQN@qSW zC!6}i=3niVzvRmK*fYnY#}uXgzvRlLbX`-$B|krD|9_AxWwNCu8bJqSp(h>x6;~>r z1ZonMA$^MA62Yjy;z)(~kK_Kp%{=MzN%qIq|L^Z3=g?}5!|@tvto@oBis+I1o8(Su z|9_AhS#K~QNWbC)=1(9e5x_Iv47yj&gWu*`1-|y*w~8&GBOwQ zcb8nux3j}q>@LBY#l~VyfR2!n!Rm{zW9RHH0lr921nKz$=9uJBdOl0fAL+iCdQm8&~44E3x8H%kwEg!LT1=Tl}a-U~+ zWW7P=#46PEWxjdj?zoK$4S3*$oRTR9rWlxFV2Xh$2BsL8Vql7aDF&t(m||dxf&Zrr z%-~JQ|8)kY^ZtEN;5$hKSoHI z_`CLlmw*4%n9fuGrvm?5FP!(gsQpcw$NOE(c;-I^$wTriCgn$qv@RWG2E?3a!ZQUT zt%v_jqIvj9iRQ)q6e2HrQet>9yig!vyhvUY&yN?tTh9yOh4N?f=KyH&XYvtzW4;N1 z0Uyb?tTm7|3Q)!$QlU z#Sj{@gX|$IXf|X5nL>Kd488=?gkbzn{O9~T5Fg5cI8Y1}4DEzUp#9KxXd_e&9fwXr zO;9sb1Jy#Ofiyygp%Xw3Lq|YwHB|-D-gaDqoK&zoZekEVX7xQoPJNWnb?;tg34)9DFjCO%Sp-9LK zcpeVXpt#9;GL#9ipb&@(Wk6dXfk0cJDo_*5637Z%1P%flAclej0bURySSMI02o?AV zyg;caa1|^Om}@z=l~_kF$rmDjfp~*!)lGxohXHxZUn0#w8Oa4W+NjuIxRuH)+qdOX~hp*&ZS#_o4D(RX-yW+KkYn|;CVw(8LwL8~g*B{*Qy0KR5a&!63`kQBO zUbueZ*0MYL9Vwj$x`J;vb`*6nx^Lgz)M3{(vwQU3_4`Skm%0z#6Wo7zkJ%l5&*}d5 z2Z8s-?!LS4{Xp%(?fVIRVb6pw?FK!Dn)|$;)xOdm+B7`Thk3UD)%0QV$V5NpdHHLl z;cY|qPjD};zFs#pFu3vQ#(`O{DunWozJ`&;sFH>1!&zIj zR%*|lu}#%&=11)#I%}0as~yw8>8NWe!K&56HDTH%8rxKZW-4mz)Hc=7pL4)q9iq}$ z*;q&Csy^2UWdfPxYrWAuXt>@~+4S^mwElq6o_QWd`}GtJn@wWpjTo<(`@ksMRL!i| zcoZ3B)ieJC%5eTsGdBw%%6UNu>V+xNywb{aL4~!unTbU?%E)FvNn-`2X$&enQ| zmGgoNXzap^Xw!v0i#ZPS92^$yvwy$X$Z@sfSBGf((nZ%CJ}sHI#BdSAp#pPzX}V*n z!ye4FrEHfIPS-KQrAaP_UHhHKUCNi^T&KA@Vz;}vU?*J9yWy9&IQL@rx`w;Yb^C&C zbh_DpmwcO7-F^?dA=~m_BzL$~ja^LaQ#I;*}x2^VAQ@ie-|HNAPz{o)6dTvmE&Z~#m&nBtH^Y~PeF!@qMhtlx85jwVxE%gAr~~&FR};A^RY>>M);IC1o1Xe5Zi<+Be6-yxJaTp z?rr?_c!juAL_)L&`2_hRC6j7QaZ5awC?PwMMTr9mM-tv8ViF+|HnEkgNM1#rO*}`@ zr^HZDrg;7QCqcCXu>6>X6lfNczWqPp#*(t31Ojmj?a~Jy%`w9!4ET-RQ4zMNk zgR}#T6Reh$23j-yHnWf2&8eq1G38jFQ@T>?7>%6!T#eMlDVtbp*!MXasjE2*_7diO zwr0v|PA=mdi^Z-_8BR5)zhft-9ODdd4H&Od5;>Kry=mIazRaY|iS&o5)$H7KtMsOf zm1$<28L7{?j5PVQ?QB@;ZZ490CXL4mPTkHm<_@O4Wwobn;RdFKrxVyqlgrE+UP7Yz zwN{xE^Y;%Lp1*nwNzs|T?rwh#;oymYmT#ID6AGBSmFLBYS5`mg$<@C5q+5St8>!gp z>e2tFe*sUu)0|>pih(HxrWlxFV2Xh$2BsL8Vql7aDF&t(01OOm0~mr(Ud{rXHcbQvIX)J@pp#gX#sw zD~vsj-9heaTs$dzj1L$O8oxFkG9ELQm^cH$nz);Io9r~%VNzsLVzSp{(Bu`su*n;w z3vvw-kEA0vAd$$$NDpKHG6o4l&PJLdZIMpqqvoHHa!3`Vrg^)0r};hee)DJMhs}?f z*P1t&pGNc=%`jSM6k)U(k%KsnxQqB^WQ>@H@IoXa@)5O$Lc=ErSp)*H)NrjK%`n&S zu%Xc4iNSY6ZNmkIp884pf%?nz4fMwK2K8?0HRx^AOVW$fbJd%tr=a^p_m1u<-6CD4 zuAeSecb@L_IRkU9%&DETa}Ik>=o}~b7Pv2bhT2uN95@q>QTwjernVVQhdaTQ)H>BF z;S9JNTu1G-+C}&dI05LoYUA)z@NIA!JOGYSN>Cyw;Q&a=vP$2Sz9@|=D=HT#=PKtZ z7b@?C8Ng=4^kK#@G;BYt1Xck%0=uAMpfXp*OvO%Rg{-kGTUIQmCl@b!LQYQ3Pi~*w zOF5KWrrcHeS@LV;YUDo2FO!Gl&&q4dcyEE;|_qBLSP z;x*hhJT#&{vT$Vm@QvY5BgP{uMkvE~N2ZV1jYN!W7=AsXH{vsrGEzSLdBkP}H&Qs# z{37Q?{)^o&4!>xAaqmUXix)564ag6`2WAfF3>Xhs4`2tJ2bK@490(Zj9`G1&1KR2V zAJFDBfEjRp?D^R2v25?n-jd#~-qXDWy-~duy{miedewURdJgwo?5XM5&=cBY(qrB; z>(QG>S05EW+Vm*pQShV1j}*Iab?@zFbbEBq?M8IH>^j-S>sr&r>2m6v=)BaqwewMD zQKw($%#Lo5!#Y|!QacuPJm}cifw?n$=Wu&N`>u9gdrCXK{c8L1_LBDPKt8lTY!|nS z+S|oeVz_u(`-E6o%oK-+{lu%pVd6INQSl*hg}AP5N85q6+P1T8zqP$@8yC$Gsfg4? zZXzGiYSCJeuc%N|B-#P;O`^-9tD^IwCQ*&(v*@j8Ky*)ZMR-a0K=?{15snK_3a<QkjSJ*7<0!Li|GB2xrjt!gO?R4J zHwl}rG<7zOHLEq(#xgYpmN`?^&-^_q^^D(46bl>iX*%fQG4u)jb2fJL~O$_Pp+5 z-M*?f)uz?K)my9EsxDW3suS|zUfUTs?)TScqdS9PmeuG*mrTSW%#H>#&q z&8)&yk*aoA-74EswzF(+SxH$@SyWkU8L2F~EV#_7%(`rTnQ56$>4(x!rDLVfOS?+X z0jVuLRJyY?t2DEeUK(4vKHoDxIzKc2cz$0#lwX?Pnm=4%lz%b5r$Dw~K|yrEv;xBd zkAmcaLj|h};tIADoGEx(P+U-7&|UDQ(2RGGm%$6-p}^ZkA+MCj}!PfKKCi z^5MK;-X&fcFPZPgH{pHbb@C4LQ~9C%g?v@s)13I6*qoT0sGP7I&m7krmmH^@#W@B! zx;Z*InmIGG-(`46rFqKRT9(5_H4Ui7-?Q_=gP;i9KSx5^inFDrK| zcP)1*Pbg=Urskd>qL8iX|1U70wk)E6gi0EBF;@71WC0 zdRl#H{e}9D`o8)?P(G{w+#p}yQa{|F+F;yZQa{*W(BRPE+u+t<+OWPMx`Ec<-w@W2 z)v&c8ry;J*t}V7LzinSzW!svzO>K2;7u$qwIRKB_M%x~O=$a5|iVWMniBv@^Kzw}c#b-*KknKu1=`zK&xZwLseoqP(-?QOEU;-#Ugm;2D1aKylb>?*j_s#Ds>Ko}t^#?xw(nss->No25dHlJL)z{l^-M_j|wlBNy zRlj?`f1gv|g?`ok@ct!zoW23jx1!&4v|!9&Y}eTHH@c(Du~}nKUyS9C-F<@`dpM>$RzCK8Y+$VT`@-)}$5Y0;CN#gJzrPvJ9se+a z`W`<%fBgIeYQpvV@p0z(#|hFz#Q4nd?g`_G%n7~m#0l96kBMdDca@x!WtES>G+?1h zMoOQQD`5y&lv1FwwQ@Ji505t?tuAe{_8aZTS{p!lS;t0?j3YIAIkI2Us`4(R+4a~nFhb{J7YFX+br&}mn9J6!)nzzLo zi#AI~i}T1Vi*So;7M&J>He%cD;5W7D=zTV6wvlKhbRv3_O^z)SZG_$mw7s_b&=~YN zbfL{x+wW*R`Z{{CtruD!%|%meI~>{^cVj#;WtgiD>5d5)1SS`A++myJCd?Ad3Ct0P z(~jpbp_s>*dPhTyIVKPD5fkXh!K}s{#S}Zfb>8Z-+GPyujE!@4a+&3F8motOcJ_2J zacRTaVC|g=F5WH=u>shH&fPApSPd-GrQgNb*a^Hg+Kbbsej21g zjc=8gz`N0>-k0Rt;??5)#pko{S>JwdICv}Mu}XK9iuXld72kua0(@(H5_}VVKl;A) z#jZKIc68mCUxwe+HQ8%luKNbw{(N22vu>qdl>a<`r?sorwfkN1uk;tJGxOW)zuf<; z|Dtv8{Wbk-{44zK25b&Ix_(!Xa?rAXguvSM)j`fd)POtd4T7|S&IFweT)+N!P)_jj zVAb{RLB+vygHHuL4YCYI2Ja8%2OSQM2uTd}3%ws^6}&El6G{wI4)YJb5b`SYR#<9S zVaS?LZkR{d_pqK&moQ>je>fwoA#6eTjBrNys<7;^{O}tQFC&jfEe&IYizCJ&JEGQy zJ&Zs{u8Ojc5`+^Y&PAS!s)=fe@QEymT82}MB1hUqp`zNNL{ami%yDVa$MIP)$S5^j zO7t=Oo|ySjOk59M4nG!y#mPn!@bNLhG4Z%|yd}OfW-op`dK>;gjB|_!ejh$4=3cA~ z(SX>2_lbEH>rC_{zQDH-R>kInw^)NQL4-%KZ;4~X?*z-(LqxAQePSHpD^ZhpjCeS9 zB-V|1n`9r4j(;1wnAl5N5syz$jXfCWN-B=8OVA)Li4&7PCa5Nu#LXe?j}K13#^=X< zjW>+H7Jn;VJMK&TmW1KNVRAKTdVEWwHF+&rLds5XN~|P{DOT}j2@l9dlryo91iF(FeDHj(uyOloG*k_0h1kE)pDkn}V$jp9OWN&1+KBQK|_CQ*_b zle;M9)K^Iz$%DzR)M-f}$#%&hNi}2qC$O;jHew1KeauRlt(kAa~RX8 zjN|~?U4{oEH_4l3M7Ls)=tIf+vO4jh>vqcV)Hp7K*~os*dChH1+s!)7QRKqX<%Xgd`lSXCraTK_6Y2x%F?6;{)(w1bPGR~(obLG=^WUS3l<+yTf z(vGHwq^q*@+1EMF;0+#+btC0WYFwI3+8g#u&ItETI+Wp?BH_+Y*UR{p332vtJJaJb zXJ_2yjB+2Phh@x4zsjDIQpgdeKH-+IFLU@@nKV+mP0Hoenlwg+Ohz5&5LcXDmzk3} Z!TFwMp5C6x%5>v=;r6FfGaS>${|i;rC}{uy delta 840 zcmcc9#JEFpg0dhZ0|Ns`PGU)BNoh(dkj*sFBwR$>$iT=**U(Vc&_uz=$jZdf%G7k? zd`F;&iLQ~6iGjX>nSOk3Vo|Yfd1?}vE+|dP$xPNQE=$%;$|+6NH83!o%*Uvt8)Q-D zTvn==UX+=npqsCtpORRTs9&61lvt9Sp%1df5N>9Y!Q@y*aaOQFldBl@1sp^D!pbc4 z@{$#FOG`3yQz!3cEa7AUx)BK2e@wPuO6TN&^8QR-!4%HOGKqN>6UUdybC_+}b49M( zweFrc`2n*O8|(SI;*oBXSSE2Xe1K@+_%V47iyfP)gc?f`@8kz8Qf#WfMYM1JnZ&w} zAEt)$1yC(x)#QaN<{b7V+<%n6WZwlEC&@mcXk}ePb>k!!|H;*C@o@EAH$du3AnK#T zPhU@N(*~>0+a+$s%cVYvC1COl_9!u!`P?TCO<-VfR5mttRA6A3tikbxrGcsb|KtSD zx07|nW!Tvp*c;dy*e36i)U9W3fIy}OrUq6ZWC22uyZ{htHfS_tHKa5eG^#auHrX`o zZ#vM#(Ig9Gr!{9ZN3^uIEa*<}wU`tz^~d|2HrJ9sZ{N&D}Sf1o;KsRrFO_HOn+Z6jvzP$OK9s5D*fO z=kDN?^+%5!{C)7{zAyVe?PlJs zxbe=$>&ve%UpeR5oR$5P`)>jLuG2Q5EwzWa`_#0f(+(_ZS#)AE`)1>#d`F$H=wFF@ z82_*^>tp!GqW?4gU*x{b!y|4j-mO@sxK``CHjD8D Date: Fri, 9 Dec 2022 22:23:10 -0600 Subject: [PATCH 10/82] fix changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5490b032..e19ad08a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.0] +RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` +and `tropo_delay` functions, as well as anciliarry functions needed for defining AOIs, look vectors, etc. + +### New/Updated Features ++ Python library access to main functions for accessing weather model data and calculating delays ++ Slant delay calculation through projection should be supported for all workflows + ## [0.2.1] * Upgrade dem-stitcher to [`>=2.3.1`](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/blob/dev/CHANGELOG.md#231) so that the updated urls for the GLO-30 DEM are used. From 9c58c758a0705dd21325094f22ae397fcff1de20 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 13:16:26 -0600 Subject: [PATCH 11/82] fix import in circleci and simplify type checking in llreader --- .circleci/config.yml | 2 +- tools/RAiDER/llreader.py | 33 +++++++++------------------------ 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 40a247c06..21ee62ba7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -43,7 +43,7 @@ jobs: eval "$($HOME/bin/micromamba shell hook -s posix)" micromamba activate RAiDER python -m pip install . - python -c "import RAiDER; from RAiDER.delay import main" + python -c "import RAiDER; from RAiDER.delay import tropo_delay" python -c "import RAiDER; from RAiDER.interpolator import interp_along_axis" - run: name: Run unit tests diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index 9a4865c94..9dea3930c 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -25,7 +25,9 @@ class AOI(object): def __init__(self): self._bounding_box = None self._proj = CRS.from_epsg(4326) - + + def type(self): + return self._type def bounds(self): return self._bounding_box @@ -57,17 +59,12 @@ def __init__(self, station_file): AOI.__init__(self) self._filename = station_file self._bounding_box = bounds_from_csv(station_file) - - - def type(self): - return 'station_file' - + self._type = 'station_file' def readLL(self): df = pd.read_csv(self._filename).drop_duplicates(subset=["Lat", "Lon"]) return df['Lat'].values, df['Lon'].values - def readZ(self): df = pd.read_csv(self._filename) if 'Hgt_m' in df.columns: @@ -85,6 +82,7 @@ def readZ(self): class RasterRDR(AOI): def __init__(self, lat_file, lon_file=None, hgt_file=None, convention='isce'): AOI.__init__(self) + self._type = 'radar_rasters' # allow for 2-band lat/lon raster if (lon_file is None): self._file = lat_file @@ -97,11 +95,6 @@ def __init__(self, lat_file, lon_file=None, hgt_file=None, convention='isce'): self._hgtfile = hgt_file self._convention = convention - - def type(self): - return 'radar_rasters' - - def readLL(self): if self._latfile is not None: return rio_open(self._latfile), rio_open(self._lonfile) @@ -130,10 +123,7 @@ class BoundingBox(AOI): def __init__(self, bbox): AOI.__init__(self) self._bounding_box = bbox - - def type(self): - return 'bounding_box' - + self._type = 'bounding_box' class GeocodedFile(AOI): '''Parse a Geocoded file for coordinates''' @@ -144,10 +134,7 @@ def __init__(self, filename, is_dem=False): self._bounding_box = rio_extents(self.p) self._is_dem = is_dem _, self._proj, self._gt = rio_stats(filename) - - - def type(self): - return 'geocoded_file' + self._type = 'geocoded_file' def readLL(self): @@ -181,13 +168,11 @@ class Geocube(AOI): '''Parse a georeferenced data cube''' def __init__(self): AOI.__init__(self) + self._type = 'geocube' raise NotImplementedError - def type(self): - return 'geocube' - def readLL(self): - raise NotImplementedError + return None def bounds_from_latlon_rasters(latfile, lonfile): From f4d85e7338ad2a2191a04371c5c3aaa863190b98 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 16:45:16 -0600 Subject: [PATCH 12/82] Fix bugs and update unit tests --- test/scenario_3/raider_example_3.yaml | 2 +- test/test_scenario_3.py | 27 ++++++++++++++++++++++++ test/test_scenarios.py | 30 +-------------------------- tools/RAiDER/cli/raider.py | 12 +++++++---- tools/RAiDER/delay.py | 18 ++++++++++------ tools/RAiDER/delayFcns.py | 4 ++-- tools/RAiDER/losreader.py | 4 +--- 7 files changed, 52 insertions(+), 45 deletions(-) create mode 100644 test/test_scenario_3.py diff --git a/test/scenario_3/raider_example_3.yaml b/test/scenario_3/raider_example_3.yaml index b336ab452..94d298c8c 100644 --- a/test/scenario_3/raider_example_3.yaml +++ b/test/scenario_3/raider_example_3.yaml @@ -11,7 +11,7 @@ aoi_group: height_group: height_levels: 0 100 500 1000 los_group: - ray_trace: True # Use projected slant delay by default + ray_trace: True orbit_file: test/scenario_3/S1A_OPER_AUX_POEORB_OPOD_20181203T120749_V20181112T225942_20181114T005942.EOF runtime_group: output_projection: 4326 diff --git a/test/test_scenario_3.py b/test/test_scenario_3.py new file mode 100644 index 000000000..e1d9e18e0 --- /dev/null +++ b/test/test_scenario_3.py @@ -0,0 +1,27 @@ +import os +import pytest +import subprocess + +from test import TEST_DIR + +import numpy as np +import xarray as xr + + + +def test_scenario_3(): + SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_3") + + test_path = os.path.join(SCENARIO_DIR, 'raider_example_3.yaml') + process = subprocess.run(['raider.py', test_path],stdout=subprocess.PIPE, universal_newlines=True) + assert process.returncode == 0 + + new_data = xr.load_dataset('HRRR_tropo_20181113T230000_std.nc') + golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20181113T230000_std.nc')) + + assert np.allclose(golden_data['wet'], new_data['wet']) + assert np.allclose(golden_data['hydro'], new_data['hydro']) + + # Clean up files + subprocess.run(['rm', '-f', './HRRR*']) + subprocess.run(['rm', '-rf', './weather_files']) \ No newline at end of file diff --git a/test/test_scenarios.py b/test/test_scenarios.py index 42adde142..26ce04511 100644 --- a/test/test_scenarios.py +++ b/test/test_scenarios.py @@ -17,6 +17,7 @@ def test_scenario_1(): new_data = xr.load_dataset('HRRR_tropo_20200101T120000_ztd.nc') golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20200101T120000_ztd.nc')) + assert np.allclose(golden_data['wet'], new_data['wet']) assert np.allclose(golden_data['hydro'], new_data['hydro']) @@ -25,32 +26,3 @@ def test_scenario_1(): subprocess.run(['rm', '-f', './HRRR*']) subprocess.run(['rm', '-rf', './weather_files']) - -#TODO: Next release include GNSS station test -# @pytest.mark.long -# def test_scenario_2(): -# test_path = os.path.join(TEST_DIR, "scenario_2", 'raider_example_2.yaml') -# process = subprocess.run(['raider.py', test_path],stdout=subprocess.PIPE, universal_newlines=True) -# assert process.returncode == 0 - -# # Clean up files -# subprocess.run(['rm', '-f', './HRRR*']) -# subprocess.run(['rm', '-rf', './weather_files']) - - -def test_scenario_3(): - SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_3") - - test_path = os.path.join(SCENARIO_DIR, 'raider_example_3.yaml') - process = subprocess.run(['raider.py', test_path],stdout=subprocess.PIPE, universal_newlines=True) - assert process.returncode == 0 - - new_data = xr.load_dataset('HRRR_tropo_20181113T230000_std.nc') - golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20181113T230000_std.nc')) - - assert np.allclose(golden_data['wet'], new_data['wet']) - assert np.allclose(golden_data['hydro'], new_data['hydro']) - - # Clean up files - subprocess.run(['rm', '-f', './HRRR*']) - subprocess.run(['rm', '-rf', './weather_files']) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 7f9a28bad..85346f638 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -300,10 +300,14 @@ def main(iargs=None): # Write the delays to file # Different options depending on the inputs - if not los.ray_trace() and not los.is_Zenith(): + if los.is_Projected(): out_filename = w.replace("_ztd", "_std") - else: + f = f.replace("_ztd", "_std") + elif los.ray_trace(): out_filename = w.replace("_ztd", "_ray") + f = f.replace("_ztd", "_ray") + else: + out_filename = w if hydro_delay is None: # means that a dataset was returned @@ -321,9 +325,9 @@ def main(iargs=None): else: if aoi.type() == 'station_file': - w = f'{os.path.splitext(w)[0]}.csv' + w = f'{os.path.splitext(out_filename)[0]}.csv' if aoi.type() in ['station_file', 'radar_rasters']: - writeDelays(aoi, wet_delay, hydro_delay, w, f, outformat=params['raster_format']) + writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 9757e953b..5b5446913 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -45,6 +45,10 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 out_proj: int,str - (optional) EPSG code for output projection look_dir: str - (optional) Satellite look direction. Only needed for slant delay calculation cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes + + Returns + ------- + xarray Dataset """ # get heights if height_levels is None: @@ -96,10 +100,11 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 # If output is desired in degrees use_weather_model_cube = False - if (cube_spacing_m is not None) or (crs != CRS(4326)): + if (cube_spacing_m is None) and (crs == CRS(4326)): use_weather_model_cube = True - + + else: #TODO handle this better if cube_spacing_m is None: cube_spacing_m = 5000 @@ -111,8 +116,9 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 out_spacing = cube_spacing_m out_snwe = clip_bbox(out_snwe, out_spacing) - logger.debug(f"Output SNWE: {out_snwe}") - logger.debug(f"Output cube spacing: {out_spacing}") + logger.debug(f"Output SNWE: {out_snwe}") + logger.debug(f"Output cube spacing: {out_spacing}") + # Load CRS from weather model file wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] @@ -124,7 +130,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 # Build the output grid zpts = np.array(heights) - if use_weather_model_cube: + if not use_weather_model_cube: xpts = np.arange(out_snwe[2], out_snwe[3] + out_spacing, out_spacing) ypts = np.arange(out_snwe[1], out_snwe[0] - out_spacing, -out_spacing) else: @@ -134,7 +140,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 # If no orbit is provided # Build zenith delay cube - if los.is_Zenith() or los.is_Projected: + if los.is_Zenith() or los.is_Projected(): out_type = ["zenith" if los.is_Zenith() else 'slant - projected'][0] # Get ZTD interpolators diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index bb2c87a91..02b2610f6 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -98,14 +98,14 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): ys_wm = np.array(ds.variables['y'][:]) zs_wm = np.array(ds.variables['z'][:]) wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] - hydro = ds.variables['hydro_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] else: ds = wm_file xs_wm = np.array(ds.variables['x'][:]) ys_wm = np.array(ds.variables['y'][:]) zs_wm = np.array(ds.variables['z'][:]) wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] - hydro = ds.variables['hydro_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] wet = np.array(wet).transpose(1, 2, 0) hydro = np.array(hydro).transpose(1, 2, 0) diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 83ab06405..3bfad9ef4 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -160,9 +160,7 @@ class Raytracing(LOS): -------- >>> from RAiDER.losreader import Raytracing >>> import numpy as np - ->>> #TODO - >>> # + >>> TODO """ def __init__(self, filename=None, los_convention='isce', time=None, pad=None): From ce4b90b908881267f525fb2a1ce88f18fef4dcaa Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 22:41:50 -0600 Subject: [PATCH 13/82] kludgy fix for now for interpolating to cubes --- tools/RAiDER/delay.py | 17 +++++++++++------ tools/RAiDER/delayFcns.py | 12 +++++++----- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 5b5446913..afb7c4c15 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -10,6 +10,7 @@ import datetime import h5py import numpy as np +import pyproj import xarray from netCDF4 import Dataset @@ -57,18 +58,23 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 #TODO: expose this as library function ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, los, out_proj = out_proj, cube_spacing_m = cube_spacing_m, look_dir = look_dir) + ds = ds.rename({'x': 'y', 'y': 'x'}) if (aoi.type() == 'bounding_box') or (aoi.type() == 'Geocube'): return ds, None else: - with xarray.load_dataset(weather_model_file) as ds2: - wm_proj = CRS.from_wkt(ds2.CRS.attrs['crs_wkt']) + # CRS can be an int, str, or CRS object + try: + out_proj = CRS.from_epsg(out_proj) + except pyproj.exceptions.CRSError: + out_proj = out_proj + pnt_proj = CRS.from_epsg(4326) - lats, lons = aoi.readLL() + lons, lats = aoi.readLL() hgts = aoi.readZ() - pnts = transformPoints(lats, lons, hgts, pnt_proj, wm_proj).T - ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro + pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj).T + ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) @@ -81,7 +87,6 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 return wetDelay, hydroDelay - def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4326, cube_spacing_m=None, look_dir='right', nproc=1): """ raider cube generation function. diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index 02b2610f6..321646763 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -12,10 +12,9 @@ from netCDF4 import Dataset import numpy as np from pyproj import CRS, Transformer -from scipy.interpolate import RegularGridInterpolator +from scipy.interpolate import RegularGridInterpolator as Interpolator from RAiDER.constants import _STEP -from RAiDER.interpolator import RegularGridInterpolator as Interpolator from RAiDER.makePoints import makePoints1D @@ -119,9 +118,12 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): wet = make_shared_raw(wet) hydro = make_shared_raw(hydro) - - ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet, fill_value=np.nan) - ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro, fill_value=np.nan) + try: + ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet, fill_value=np.nan, bounds_error = False) + ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro, fill_value=np.nan, bounds_error = False) + except ValueError: + ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet.transpose(1,0,2), fill_value=np.nan, bounds_error = False) + ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro.transpose(1,0,2), fill_value=np.nan, bounds_error = False) return ifWet, ifHydro From 611821029eaa8de4986baa366316da477b24bcff Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 22:51:07 -0600 Subject: [PATCH 14/82] undo file changes --- test/test_geom/test_era5.nc | Bin 6456 -> 2216 bytes test/test_geom/test_era5t.nc | Bin 6456 -> 2216 bytes test/test_geom/test_erai.nc | Bin 16732 -> 3256 bytes 3 files changed, 0 insertions(+), 0 deletions(-) diff --git a/test/test_geom/test_era5.nc b/test/test_geom/test_era5.nc index b5ca6699a0444ff24963a68c4e16d82e023b7fba..89d77b7a5257e4c84aa39e14c96d97655ce37369 100644 GIT binary patch delta 621 zcmdmCv_f!#vLGV^0|Q4+Vo7F6X-X=P%{b8{T*Su6z{p6~&`{UVM8U|&%EZvh*mC0h zx_UDU6EjmYOA9>t97wP3D78UDFp3TUu2yuhC zrJlKo0mv?h6O1f$foAHZ7iA_*KEYVb$THc6DW8)C=pi6r*OgQfuyRG5! ts)s*+UE`OMUZ!!>&fEQ=|EJ(vkssoWlWfzxbIl6fO3zj4)mDJa0suP!+baM7 literal 6456 zcmb_g2~<d1U5F5q-LXCn7)CYo+7C{*dAwVF6Apt}g&P@Or4OScw z4bOsoj#vevR;v~hp(-c@p;jpb=Q`kkpwYtH=OX>B%l^x^R`196?ctt#&b|BW8v=`$ zcxY=49b>_a1didy2_ku-q;T#~9_RGgKlf(h(j&iD4tLhmzauu7(c?r&c@D;Zfi%kaWb`acCoW} zVK|t2F8j(fEY&nJkrx&!iVGERMV#;mx}BAsgO#m|sdZeu$ePQEO^L9MWhV;h96k>g zv<~A5tp9q26{Jk*EYo;)B0JWU9%pLJVRNFm*5Pas+sT?8&W?vWvBGm$ao38ti2^pC zZtLh|!(iCj*fFdeoot=#Y@O(~&hs1`=uY%7XZv~f97h|vb40iuongme)1BE|5JyK_ zr-%slJZC2xD*GPEc9;oQhXE>{@JWeY_xj=Vlpz$P~oV!W5{&xw1$00@WfY>;>R z1F!dPAH2Wb_~G>gafzZRJjI9Ce0NT~o)I}xOP2xm-TuRC;d}hM=Nl8t&U~V|R%@X{v?C|i=xQGZL7k9zi^KSay zXYSOk^~C2xJ)n^OYK~p@FxhH52MBlMQmJ>@b`LCKk~2R;Q2>2kh5y(c7Yv{00IZD*Z)+T z1>d0W=WZ8Yfa^Y3n}uQ`@@jo+BtHKS)@Dphq-jhfl*Se&hT}uEF=WMaIlKrS$22M_ zmKV+wk)NV}Un4`|+wq})9sf?P#_xr-MhtEJ|6Z$wLCmaqh5NpPIX+k`S3#4vf$V7q zKK~EaYJ?X|2*-zNH0~>IY&3D zEA?)4n^U^OjRu?vGyzU6-2wAM&yDDX7V`st_%jLj8~*IV?;rmCY6P4P#9i+J330;JH^SAqC69b02$yRU)*6b)~4d@Yy8O5CBY*38PD6=UvAhuXBW>IEo&CoK14!|%4 ze>xBYM;J4-W@*jDUXOQ=55Tv_3y6W^<$z`2V~fu%?^$u{$gOG;hi@GPd-PV-t?xmP zfISTQJy?vx*BY-iUAuJcBG}7-Ccu?z&A`iVE%qVP@~EY!v|2~)fuxh z=0FS!fH}b8%p8()&RBwHIfRqmCzk;+e89uL59wj`B}r;cSWP%EtcG2~0bm37au4-RD2MKEye+es=(qLwueChMfpWVMTLMou-MB7 z4rPjpHi9Pw1r&}g9ABsp)B|E~EE(yON`cITLJDAfeo6ke{H?%mfZOu7K#Zd;q~bH% z0bujY^DOh`0L_4wc@_}Q0SqaQ=aT0wa-wpgfxMguU=(NsD7KMrW1Nl5iO4p}{v>-6 z=)~;Fq#A-w$~Gi(*y3nnwxOg!^0VYTs1j^F=sD2yph{4sL_ty^QHXQISzeZPzY@_NYDsl@W#M=_Jj#BMO*;L5eJF)iIrlx zxI#jeFvL&9osyFhf5}8~cb1N1r(}Zo`Ye|>k9%bLn$Nt(vLb&tf#x;-3Vr>c?Ot)c#}eF>~QDANvd+vnC#c`-1F< zx86{nX#d$U!=0m#^wD_0u*E*c1n4&kSOf5|?}(g`3;F^%fOQ7Xfy_7|d&B_W0ojq( z5$!@Uz%J63qn%*!*MyyrBflA|;ZeDzMGyCsYmBbfHFc7S2CKzkzp}m_Up|1Lz4c3cCFb>LErGI_MR8 z2pzA5`p^SHAL1A20nD-&?LgaM)?H{5+6*h_q8vam+JLfA2FgSDNG5Z+h(LvqDTL@d z^ex&7SIE&JxclAc7t{oIEGO0wA;fZcPxA?ah#&%pC&X=HBatqaiYR0Hay5N%HEyrSg4&pE!TU)Z;}AXY>B-jOd^A~ z@wsHWg@juY^N8bfrj^ran>}-He}2l`%gn;U%3_=Gc@vG> z8|Q;g^K2VjeqOZJEs|N`yu*#lEOB&q@OQe&m`HzN8)IK%Hfbhxk*{Nejf6hyi#$*J zCI5EsV*a*N*2Q###oS5x|IHR7Lnj?c*g!9pv7R67beXDI=Ff z`ST`6{StM8XURLkJIc$89vQua*GBu@OvlW~%#v14`+;_erlxUe*|c4>A89@`A#EqE zo)$*iOlvUHGW@|X)zHmwqG73Fu%Vg3Q$wDiy}@gP6NYmP`V5X6L>s6Kb{Yg2XumoB zN5MeM;1_?aA6PV~J9v7)d*I$+u9hC?s=?D*lLnmzg@aXtjh#iEzjn><=C}PZP_)mt5~h4&g9pWmyvu3W0DJMW^XRE|?7DBGdtkCY3Q8*55x5^8L!TWV+4 z&Zsr2xm}xEd$hK(_FAn{PLZ#WquN_?4|%D)Q^u8Tlhw-VWy!LWvPb2kWofdea>H^r znNX%IpH%KAi;(S?^_Dvo_7_EOoGH~SbSTUyyjzrCz%Mvjs8h(zFUwaKI207->*g>)+M+8C_kv36HBoO0JJRu;g(QSkXJ~0=mk(h`gmJ$?nLRu!3Ne@d~r01k{(u>kt z(o@pw(ig>s(sGDi7n@7lq+L=?@svuN%E-#f%C<_AN>=5j%AYG9R+?4@SC&@nskmM_ zq0+M=y<$_vk&0{b+wx9%ll-v!9(Xt8$B${td*wayUym6c(>taoAC$j1HsP4bF>6(Z z3aN5bV%0v?R#mBLr|PasrMjq6v>LWjRIgNbTj{NHTc@^C)KYblI#HdhzM`&HA68d& zk5<1_zfeEuHtV+RrgzWkp5Of?;0ui_z)j0((KV3(KKo*G^aFIG`*T? z%|*@a*W)PnC=HbFDeEW~C>@l`6dCvp6gg!J4%n?=0EUS7HvmKlgagZN1sQCN2y1?N49&md!GAd_eS3|-(|i(`_}vN zz^?IW_l@>l>}%~)=WF0|(C4~O%d&2tX+GgTE5CAPsaXB2Jzq^{#jtW%^I3>x$?|70 z{Eqvb@q6gk>$leLf?q#tbilp9uLAc5ngrAb)CUFyP7g2+NDZ79m>OUeusz^@U}S(I zP$$qO@X+dnppIbgpsTBUgV@3IgFjh)BPb}?CwNa#T+qE>kKpcLeaO~_ppcF=M(jmw z&rq#3)7T6)J+ysIEPDa_@|rgGRQ92;jqE`7?yv~XsIau~DcmL8=&%sZ0N0E=gF}Qb z=FZ{fgkR@6aDR&k0+H}+(#f9!67pJ1baDG&%^1-}SR3K|4=<1WRO#XH0&#ka>NCoD`zN{CGumq1Cd zOZ+rZk$60jmKd9ONO)8z5#AN1iUdiYCE1Ijld6*)l8z?XidKj&C&#BONnV*8oxDG} zHl-t_KczRtB9)SQDYbfSX=>{_l=?i?Amvm_W=i|oAJ%f-KaF%v~lZKt!qtd zOzTJ+OdFYYH{CjIVBL%L_UoP3m!)mXFv)O9Uz{-w{!>w(@icRG#t97wP3D78UDFp3TUu2yuhC zrJlKo0mv?h6O1f$foAHZ7iA_*KEYVb$THc6DW8)C=pi6r*OgQfuyRG5! ts)s*+UE`OMUZ!!>&fEQ=|EJ(vkssoWlWfzxbIl6fO3zj4)mDJa0suP!+baM7 literal 6456 zcmb_g2~<d1U5F5q-LXCn7)CYo+7C{*dAwVF6Apt}g&P@Or4OScw z4bOsoj#vevR;v~hp(-c@p;jpb=Q`kkpwYtH=OX>B%l^x^R`196?ctt#&b|BW8v=`$ zcxY=49b>_a1didy2_ku-q;T#~9_RGgKlf(h(j&iD4tLhmzauu7(c?r&c@D;Zfi%kaWb`acCoW} zVK|t2F8j(fEY&nJkrx&!iVGERMV#;mx}BAsgO#m|sdZeu$ePQEO^L9MWhV;h96k>g zv<~A5tp9q26{Jk*EYo;)B0JWU9%pLJVRNFm*5Pas+sT?8&W?vWvBGm$ao38ti2^pC zZtLh|!(iCj*fFdeoot=#Y@O(~&hs1`=uY%7XZv~f97h|vb40iuongme)1BE|5JyK_ zr-%slJZC2xD*GPEc9;oQhXE>{@JWeY_xj=Vlpz$P~oV!W5{&xw1$00@WfY>;>R z1F!dPAH2Wb_~G>gafzZRJjI9Ce0NT~o)I}xOP2xm-TuRC;d}hM=Nl8t&U~V|R%@X{v?C|i=xQGZL7k9zi^KSay zXYSOk^~C2xJ)n^OYK~p@FxhH52MBlMQmJ>@b`LCKk~2R;Q2>2kh5y(c7Yv{00IZD*Z)+T z1>d0W=WZ8Yfa^Y3n}uQ`@@jo+BtHKS)@Dphq-jhfl*Se&hT}uEF=WMaIlKrS$22M_ zmKV+wk)NV}Un4`|+wq})9sf?P#_xr-MhtEJ|6Z$wLCmaqh5NpPIX+k`S3#4vf$V7q zKK~EaYJ?X|2*-zNH0~>IY&3D zEA?)4n^U^OjRu?vGyzU6-2wAM&yDDX7V`st_%jLj8~*IV?;rmCY6P4P#9i+J330;JH^SAqC69b02$yRU)*6b)~4d@Yy8O5CBY*38PD6=UvAhuXBW>IEo&CoK14!|%4 ze>xBYM;J4-W@*jDUXOQ=55Tv_3y6W^<$z`2V~fu%?^$u{$gOG;hi@GPd-PV-t?xmP zfISTQJy?vx*BY-iUAuJcBG}7-Ccu?z&A`iVE%qVP@~EY!v|2~)fuxh z=0FS!fH}b8%p8()&RBwHIfRqmCzk;+e89uL59wj`B}r;cSWP%EtcG2~0bm37au4-RD2MKEye+es=(qLwueChMfpWVMTLMou-MB7 z4rPjpHi9Pw1r&}g9ABsp)B|E~EE(yON`cITLJDAfeo6ke{H?%mfZOu7K#Zd;q~bH% z0bujY^DOh`0L_4wc@_}Q0SqaQ=aT0wa-wpgfxMguU=(NsD7KMrW1Nl5iO4p}{v>-6 z=)~;Fq#A-w$~Gi(*y3nnwxOg!^0VYTs1j^F=sD2yph{4sL_ty^QHXQISzeZPzY@_NYDsl@W#M=_Jj#BMO*;L5eJF)iIrlx zxI#jeFvL&9osyFhf5}8~cb1N1r(}Zo`Ye|>k9%bLn$Nt(vLb&tf#x;-3Vr>c?Ot)c#}eF>~QDANvd+vnC#c`-1F< zx86{nX#d$U!=0m#^wD_0u*E*c1n4&kSOf5|?}(g`3;F^%fOQ7Xfy_7|d&B_W0ojq( z5$!@Uz%J63qn%*!*MyyrBflA|;ZeDzMGyCsYmBbfHFc7S2CKzkzp}m_Up|1Lz4c3cCFb>LErGI_MR8 z2pzA5`p^SHAL1A20nD-&?LgaM)?H{5+6*h_q8vam+JLfA2FgSDNG5Z+h(LvqDTL@d z^ex&7SIE&JxclAc7t{oIEGO0wA;fZcPxA?ah#&%pC&X=HBatqaiYR0Hay5N%HEyrSg4&pE!TU)Z;}AXY>B-jOd^A~ z@wsHWg@juY^N8bfrj^ran>}-He}2l`%gn;U%3_=Gc@vG> z8|Q;g^K2VjeqOZJEs|N`yu*#lEOB&q@OQe&m`HzN8)IK%Hfbhxk*{Nejf6hyi#$*J zCI5EsV*a*N*2Q###oS5x|IHR7Lnj?c*g!9pv7R67beXDI=Ff z`ST`6{StM8XURLkJIc$89vQua*GBu@OvlW~%#v14`+;_erlxUe*|c4>A89@`A#EqE zo)$*iOlvUHGW@|X)zHmwqG73Fu%Vg3Q$wDiy}@gP6NYmP`V5X6L>s6Kb{Yg2XumoB zN5MeM;1_?aA6PV~J9v7)d*I$+u9hC?s=?D*lLnmzg@aXtjh#iEzjn><=C}PZP_)mt5~h4&g9pWmyvu3W0DJMW^XRE|?7DBGdtkCY3Q8*55x5^8L!TWV+4 z&Zsr2xm}xEd$hK(_FAn{PLZ#WquN_?4|%D)Q^u8Tlhw-VWy!LWvPb2kWofdea>H^r znNX%IpH%KAi;(S?^_Dvo_7_EOoGH~SbSTUyyjzrCz%Mvjs8h(zFUwaKI207->*g>)+M+8C_kv36HBoO0JJRu;g(QSkXJ~0=mk(h`gmJ$?nLRu!3Ne@d~r01k{(u>kt z(o@pw(ig>s(sGDi7n@7lq+L=?@svuN%E-#f%C<_AN>=5j%AYG9R+?4@SC&@nskmM_ zq0+M=y<$_vk&0{b+wx9%ll-v!9(Xt8$B${td*wayUym6c(>taoAC$j1HsP4bF>6(Z z3aN5bV%0v?R#mBLr|PasrMjq6v>LWjRIgNbTj{NHTc@^C)KYblI#HdhzM`&HA68d& zk5<1_zfeEuHtV+RrgzWkp5Of?;0ui_z)j0((KV3(KKo*G^aFIG`*T? z%|*@a*W)PnC=HbFDeEW~C>@l`6dCvp6gg!J4%n?=0EUS7HvmKlgagZN1sQCN2y1?N49&md!GAd_eS3|-(|i(`_}vN zz^?IW_l@>l>}%~)=WF0|(C4~O%d&2tX+GgTE5CAPsaXB2Jzq^{#jtW%^I3>x$?|70 z{Eqvb@q6gk>$leLf?q#tbilp9uLAc5ngrAb)CUFyP7g2+NDZ79m>OUeusz^@U}S(I zP$$qO@X+dnppIbgpsTBUgV@3IgFjh)BPb}?CwNa#T+qE>kKpcLeaO~_ppcF=M(jmw z&rq#3)7T6)J+ysIEPDa_@|rgGRQ92;jqE`7?yv~XsIau~DcmL8=&%sZ0N0E=gF}Qb z=FZ{fgkR@6aDR&k0+H}+(#f9!67pJ1baDG&%^1-}SR3K|4=<1WRO#XH0&#ka>NCoD`zN{CGumq1Cd zOZ+rZk$60jmKd9ONO)8z5#AN1iUdiYCE1Ijld6*)l8z?XidKj&C&#BONnV*8oxDG} zHl-t_KczRtB9)SQDYbfSX=>{_l=?i?Amvm_W=i|oAJ%f-KaF%v~lZKt!qtd zOzTJ+OdFYYH{CjIVBL%L_UoP3m!)mXFv)O9Uz{-w{!>w(@icRG#$iT=**U(Vc&_uz=$jZdf%G7k? zd`F;&iLQ~6iGjX>nSOk3Vo|Yfd1?}vE+|dP$xPNQE=$%;$|+6NH83!o%*Uvt8)Q-D zTvn==UX+=npqsCtpORRTs9&61lvt9Sp%1df5N>9Y!Q@y*aaOQFldBl@1sp^D!pbc4 z@{$#FOG`3yQz!3cEa7AUx)BK2e@wPuO6TN&^8QR-!4%HOGKqN>6UUdybC_+}b49M( zweFrc`2n*O8|(SI;*oBXSSE2Xe1K@+_%V47iyfP)gc?f`@8kz8Qf#WfMYM1JnZ&w} zAEt)$1yC(x)#QaN<{b7V+<%n6WZwlEC&@mcXk}ePb>k!!|H;*C@o@EAH$du3AnK#T zPhU@N(*~>0+a+$s%cVYvC1COl_9!u!`P?TCO<-VfR5mttRA6A3tikbxrGcsb|KtSD zx07|nW!Tvp*c;dy*e36i)U9W3fIy}OrUq6ZWC22uyZ{htHfS_tHKa5eG^#auHrX`o zZ#vM#(Ig9Gr!{9ZN3^uIEa*<}wU`tz^~d|2HrJ9sZ{N&D}Sf1o;KsRrFO_HOn+Z6jvzP$OK9s5D*fO z=kDN?^+%5!{C)7{zAyVe?PlJs zxbe=$>&ve%UpeR5oR$5P`)>jLuG2Q5EwzWa`_#0f(+(_ZS#)AE`)1>#d`F$H=wFF@ z82_*^>tp!GqW?4gU*x{b!y|4j-mO@sxK``CHjD8DF8mYpzIWeU-uKr3)?0sBi`n^R_LMVwpKoT*37i#IHxD@( z85vmsWdH>Lr9>))NMbN!2%rMeG=Le2IL7Zyiqf`30*jC+?Uj*n0#IO($Ule6sDQe3 zPM9l|!Xi)@Br1gts*`Pst{y1c1&gFrndxyPI)h5%NOdq?7KOG!qfxeKl+9uU+7^Sh z!Pq+@JbnEU(HsPkMv9JPP$MY>20kVhg@Qr)Gq8432@brsHWi20qSOI-M=r zW-&^dTieH2N0MZAK+#!vRCFSffU>c1L@kOXEX3pRwm7>*b_Bfb zLiEDeh4yy#cF~LQc*ocn2h?(!mu~`Rfz%%aiuzA25*AuVO2?spI{i-!FZ$PpeHaOj zfyoOfcm#_1cd}uCyBPp~UNh->O4ILaF^x$fF(y~x$2zFQ5QsDafgVY~(HUUeq-}@AfEE8Y2!21nE~c|2U`A-t5VdGBeac z`NuscwI%gYYEP>FYOYyiY78MUa&o6eQkdju0u7A(vB$?IHJKSmY5gai(s|?oq;pEk zKXYajM`hCJ2s(*^Cm?JcZER3Bw%~)XvB6A!7TAD`K>C$0zuTCB$0ZUdF*vCwlN^Ip zAW~^09A)y-$@r7TAV^Bmdsj%L(37MkfYc}HHBvvNN$RIG7FFodNh4!%co0oL+hmtj zhW)0!a`_`rSH{J}L{ej8=>+LY|F}lh#auQ*^Y&h*v_H~=l$aQZ12Hh!FSR4f`(Z_u zOrnDuCPuRU);;OR@BM$~NH$|~ieGXt*M~~P(MXIqGJ+b5pfhQ}V*(2$tF+ge|i7YLHDoke><&EhAk_d_LukXETAVotC`p&?f(znzfuAbkwCPx zMA`oE`&WBmre6{PPl_es5phg1DTc(DJOO27ROt+ybZf-?DGq1Se~yp*$MJvQN@qSW zC!6}i=3niVzvRmK*fYnY#}uXgzvRlLbX`-$B|krD|9_AxWwNCu8bJqSp(h>x6;~>r z1ZonMA$^MA62Yjy;z)(~kK_Kp%{=MzN%qIq|L^Z3=g?}5!|@tvto@oBis+I1o8(Su z|9_AhS#K~QNWbC)=1(9e5x_Iv47yj&gWu*`1-|y*w~8&GBOwQ zcb8nux3j}q>@LBY#l~VyfR2!n!Rm{zW9RHH0lr921nKz$=9uJBdOl0fAL+iCdQm8&~44E3x8H%kwEg!LT1=Tl}a-U~+ zWW7P=#46PEWxjdj?zoK$4S3*$oRTR9rWlxFV2Xh$2BsL8Vql7aDF&t(m||dxf&Zrr z%-~JQ|8)kY^ZtEN;5$hKSoHI z_`CLlmw*4%n9fuGrvm?5FP!(gsQpcw$NOE(c;-I^$wTriCgn$qv@RWG2E?3a!ZQUT zt%v_jqIvj9iRQ)q6e2HrQet>9yig!vyhvUY&yN?tTh9yOh4N?f=KyH&XYvtzW4;N1 z0Uyb?tTm7|3Q)!$QlU z#Sj{@gX|$IXf|X5nL>Kd488=?gkbzn{O9~T5Fg5cI8Y1}4DEzUp#9KxXd_e&9fwXr zO;9sb1Jy#Ofiyygp%Xw3Lq|YwHB|-D-gaDqoK&zoZekEVX7xQoPJNWnb?;tg34)9DFjCO%Sp-9LK zcpeVXpt#9;GL#9ipb&@(Wk6dXfk0cJDo_*5637Z%1P%flAclej0bURySSMI02o?AV zyg;caa1|^Om}@z=l~_kF$rmDjfp~*!)lGxohXHxZUn0#w8Oa4W+NjuIxRuH)+qdOX~hp*&ZS#_o4D(RX-yW+KkYn|;CVw(8LwL8~g*B{*Qy0KR5a&!63`kQBO zUbueZ*0MYL9Vwj$x`J;vb`*6nx^Lgz)M3{(vwQU3_4`Skm%0z#6Wo7zkJ%l5&*}d5 z2Z8s-?!LS4{Xp%(?fVIRVb6pw?FK!Dn)|$;)xOdm+B7`Thk3UD)%0QV$V5NpdHHLl z;cY|qPjD};zFs#pFu3vQ#(`O{DunWozJ`&;sFH>1!&zIj zR%*|lu}#%&=11)#I%}0as~yw8>8NWe!K&56HDTH%8rxKZW-4mz)Hc=7pL4)q9iq}$ z*;q&Csy^2UWdfPxYrWAuXt>@~+4S^mwElq6o_QWd`}GtJn@wWpjTo<(`@ksMRL!i| zcoZ3B)ieJC%5eTsGdBw%%6UNu>V+xNywb{aL4~!unTbU?%E)FvNn-`2X$&enQ| zmGgoNXzap^Xw!v0i#ZPS92^$yvwy$X$Z@sfSBGf((nZ%CJ}sHI#BdSAp#pPzX}V*n z!ye4FrEHfIPS-KQrAaP_UHhHKUCNi^T&KA@Vz;}vU?*J9yWy9&IQL@rx`w;Yb^C&C zbh_DpmwcO7-F^?dA=~m_BzL$~ja^LaQ#I;*}x2^VAQ@ie-|HNAPz{o)6dTvmE&Z~#m&nBtH^Y~PeF!@qMhtlx85jwVxE%gAr~~&FR};A^RY>>M);IC1o1Xe5Zi<+Be6-yxJaTp z?rr?_c!juAL_)L&`2_hRC6j7QaZ5awC?PwMMTr9mM-tv8ViF+|HnEkgNM1#rO*}`@ zr^HZDrg;7QCqcCXu>6>X6lfNczWqPp#*(t31Ojmj?a~Jy%`w9!4ET-RQ4zMNk zgR}#T6Reh$23j-yHnWf2&8eq1G38jFQ@T>?7>%6!T#eMlDVtbp*!MXasjE2*_7diO zwr0v|PA=mdi^Z-_8BR5)zhft-9ODdd4H&Od5;>Kry=mIazRaY|iS&o5)$H7KtMsOf zm1$<28L7{?j5PVQ?QB@;ZZ490CXL4mPTkHm<_@O4Wwobn;RdFKrxVyqlgrE+UP7Yz zwN{xE^Y;%Lp1*nwNzs|T?rwh#;oymYmT#ID6AGBSmFLBYS5`mg$<@C5q+5St8>!gp z>e2tFe*sUu)0|>pih(HxrWlxFV2Xh$2BsL8Vql7aDF&t(01OOm0~mr(Ud{rXHcbQvIX)J@pp#gX#sw zD~vsj-9heaTs$dzj1L$O8oxFkG9ELQm^cH$nz);Io9r~%VNzsLVzSp{(Bu`su*n;w z3vvw-kEA0vAd$$$NDpKHG6o4l&PJLdZIMpqqvoHHa!3`Vrg^)0r};hee)DJMhs}?f z*P1t&pGNc=%`jSM6k)U(k%KsnxQqB^WQ>@H@IoXa@)5O$Lc=ErSp)*H)NrjK%`n&S zu%Xc4iNSY6ZNmkIp884pf%?nz4fMwK2K8?0HRx^AOVW$fbJd%tr=a^p_m1u<-6CD4 zuAeSecb@L_IRkU9%&DETa}Ik>=o}~b7Pv2bhT2uN95@q>QTwjernVVQhdaTQ)H>BF z;S9JNTu1G-+C}&dI05LoYUA)z@NIA!JOGYSN>Cyw;Q&a=vP$2Sz9@|=D=HT#=PKtZ z7b@?C8Ng=4^kK#@G;BYt1Xck%0=uAMpfXp*OvO%Rg{-kGTUIQmCl@b!LQYQ3Pi~*w zOF5KWrrcHeS@LV;YUDo2FO!Gl&&q4dcyEE;|_qBLSP z;x*hhJT#&{vT$Vm@QvY5BgP{uMkvE~N2ZV1jYN!W7=AsXH{vsrGEzSLdBkP}H&Qs# z{37Q?{)^o&4!>xAaqmUXix)564ag6`2WAfF3>Xhs4`2tJ2bK@490(Zj9`G1&1KR2V zAJFDBfEjRp?D^R2v25?n-jd#~-qXDWy-~duy{miedewURdJgwo?5XM5&=cBY(qrB; z>(QG>S05EW+Vm*pQShV1j}*Iab?@zFbbEBq?M8IH>^j-S>sr&r>2m6v=)BaqwewMD zQKw($%#Lo5!#Y|!QacuPJm}cifw?n$=Wu&N`>u9gdrCXK{c8L1_LBDPKt8lTY!|nS z+S|oeVz_u(`-E6o%oK-+{lu%pVd6INQSl*hg}AP5N85q6+P1T8zqP$@8yC$Gsfg4? zZXzGiYSCJeuc%N|B-#P;O`^-9tD^IwCQ*&(v*@j8Ky*)ZMR-a0K=?{15snK_3a<QkjSJ*7<0!Li|GB2xrjt!gO?R4J zHwl}rG<7zOHLEq(#xgYpmN`?^&-^_q^^D(46bl>iX*%fQG4u)jb2fJL~O$_Pp+5 z-M*?f)uz?K)my9EsxDW3suS|zUfUTs?)TScqdS9PmeuG*mrTSW%#H>#&q z&8)&yk*aoA-74EswzF(+SxH$@SyWkU8L2F~EV#_7%(`rTnQ56$>4(x!rDLVfOS?+X z0jVuLRJyY?t2DEeUK(4vKHoDxIzKc2cz$0#lwX?Pnm=4%lz%b5r$Dw~K|yrEv;xBd zkAmcaLj|h};tIADoGEx(P+U-7&|UDQ(2RGGm%$6-p}^ZkA+MCj}!PfKKCi z^5MK;-X&fcFPZPgH{pHbb@C4LQ~9C%g?v@s)13I6*qoT0sGP7I&m7krmmH^@#W@B! zx;Z*InmIGG-(`46rFqKRT9(5_H4Ui7-?Q_=gP;i9KSx5^inFDrK| zcP)1*Pbg=Urskd>qL8iX|1U70wk)E6gi0EBF;@71WC0 zdRl#H{e}9D`o8)?P(G{w+#p}yQa{|F+F;yZQa{*W(BRPE+u+t<+OWPMx`Ec<-w@W2 z)v&c8ry;J*t}V7LzinSzW!svzO>K2;7u$qwIRKB_M%x~O=$a5|iVWMniBv@^Kzw}c#b-*KknKu1=`zK&xZwLseoqP(-?QOEU;-#Ugm;2D1aKylb>?*j_s#Ds>Ko}t^#?xw(nss->No25dHlJL)z{l^-M_j|wlBNy zRlj?`f1gv|g?`ok@ct!zoW23jx1!&4v|!9&Y}eTHH@c(Du~}nKUyS9C-F<@`dpM>$RzCK8Y+$VT`@-)}$5Y0;CN#gJzrPvJ9se+a z`W`<%fBgIeYQpvV@p0z(#|hFz#Q4nd?g`_G%n7~m#0l96kBMdDca@x!WtES>G+?1h zMoOQQD`5y&lv1FwwQ@Ji505t?tuAe{_8aZTS{p!lS;t0?j3YIAIkI2Us`4(R+4a~nFhb{J7YFX+br&}mn9J6!)nzzLo zi#AI~i}T1Vi*So;7M&J>He%cD;5W7D=zTV6wvlKhbRv3_O^z)SZG_$mw7s_b&=~YN zbfL{x+wW*R`Z{{CtruD!%|%meI~>{^cVj#;WtgiD>5d5)1SS`A++myJCd?Ad3Ct0P z(~jpbp_s>*dPhTyIVKPD5fkXh!K}s{#S}Zfb>8Z-+GPyujE!@4a+&3F8motOcJ_2J zacRTaVC|g=F5WH=u>shH&fPApSPd-GrQgNb*a^Hg+Kbbsej21g zjc=8gz`N0>-k0Rt;??5)#pko{S>JwdICv}Mu}XK9iuXld72kua0(@(H5_}VVKl;A) z#jZKIc68mCUxwe+HQ8%luKNbw{(N22vu>qdl>a<`r?sorwfkN1uk;tJGxOW)zuf<; z|Dtv8{Wbk-{44zK25b&Ix_(!Xa?rAXguvSM)j`fd)POtd4T7|S&IFweT)+N!P)_jj zVAb{RLB+vygHHuL4YCYI2Ja8%2OSQM2uTd}3%ws^6}&El6G{wI4)YJb5b`SYR#<9S zVaS?LZkR{d_pqK&moQ>je>fwoA#6eTjBrNys<7;^{O}tQFC&jfEe&IYizCJ&JEGQy zJ&Zs{u8Ojc5`+^Y&PAS!s)=fe@QEymT82}MB1hUqp`zNNL{ami%yDVa$MIP)$S5^j zO7t=Oo|ySjOk59M4nG!y#mPn!@bNLhG4Z%|yd}OfW-op`dK>;gjB|_!ejh$4=3cA~ z(SX>2_lbEH>rC_{zQDH-R>kInw^)NQL4-%KZ;4~X?*z-(LqxAQePSHpD^ZhpjCeS9 zB-V|1n`9r4j(;1wnAl5N5syz$jXfCWN-B=8OVA)Li4&7PCa5Nu#LXe?j}K13#^=X< zjW>+H7Jn;VJMK&TmW1KNVRAKTdVEWwHF+&rLds5XN~|P{DOT}j2@l9dlryo91iF(FeDHj(uyOloG*k_0h1kE)pDkn}V$jp9OWN&1+KBQK|_CQ*_b zle;M9)K^Iz$%DzR)M-f}$#%&hNi}2qC$O;jHew1KeauRlt(kAa~RX8 zjN|~?U4{oEH_4l3M7Ls)=tIf+vO4jh>vqcV)Hp7K*~os*dChH1+s!)7QRKqX<%Xgd`lSXCraTK_6Y2x%F?6;{)(w1bPGR~(obLG=^WUS3l<+yTf z(vGHwq^q*@+1EMF;0+#+btC0WYFwI3+8g#u&ItETI+Wp?BH_+Y*UR{p332vtJJaJb zXJ_2yjB+2Phh@x4zsjDIQpgdeKH-+IFLU@@nKV+mP0Hoenlwg+Ohz5&5LcXDmzk3} Z!TFwMp5C6x%5>v=;r6FfGaS>${|i;rC}{uy From 4453f4ec9759db57061dccfa31262685fff3ade7 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 8 Dec 2022 23:00:33 -0600 Subject: [PATCH 15/82] update docstrings changelog --- CHANGELOG.md | 2 +- tools/RAiDER/delay.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e19ad08a8..541e480d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and `tropo_delay` functions, as well as anciliarry functions needed for defining ### New/Updated Features + Python library access to main functions for accessing weather model data and calculating delays -+ Slant delay calculation through projection should be supported for all workflows ++ Slant delay calculation through projection is supported for cubes with orbit files ## [0.2.1] diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index afb7c4c15..077a4fc84 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -34,7 +34,7 @@ ############################################################################### def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4326, look_dir='right', cube_spacing_m=None): """ - raider main function for calculating delays. + Calculate integrated delays on query points. Parameterss ---------- @@ -49,7 +49,8 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 Returns ------- - xarray Dataset + xarray Dataset or wet and hydrostatic delays at the query points. The dataset will contain fields + 'wet' and 'hydro' which are the total (integrated) wet and hydrostatic delays """ # get heights if height_levels is None: From d7736801d580c512db3d58912b1d54f94c610004 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Thu, 8 Dec 2022 16:34:31 -0800 Subject: [PATCH 16/82] correct naming of ray tracing delays --- tools/RAiDER/cli/raider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 85346f638..9b232539e 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -304,8 +304,8 @@ def main(iargs=None): out_filename = w.replace("_ztd", "_std") f = f.replace("_ztd", "_std") elif los.ray_trace(): - out_filename = w.replace("_ztd", "_ray") - f = f.replace("_ztd", "_ray") + out_filename = w.replace("_std", "_ray") + f = f.replace("_std", "_ray") else: out_filename = w From c0aa76cfbaacf55da19b68f97cc5802caa172296 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Thu, 8 Dec 2022 16:56:39 -0800 Subject: [PATCH 17/82] reuse DEM if it exists --- tools/RAiDER/cli/validators.py | 6 +++- tools/RAiDER/dem.py | 50 ++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 25 deletions(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 02793f122..55a87592e 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -44,6 +44,7 @@ def get_los(args): else: los = Conventional(args.los_file, args.los_convention) elif ('los_cube' in args.keys()) and (args['los_cube'] is not None): + raise NotImplementedError('LOS_cube is not yet implemented') # if args.ray_trace: # los = Raytracing(args.los_cube) @@ -149,7 +150,10 @@ def enforce_bbox(bbox): """ Enforce a valid bounding box """ - bbox = [float(d) for d in bbox.strip().split()] + if isinstance(bbox, str): + bbox = [float(d) for d in bbox.strip().split()] + else: + bbox = [float(d) for d in bbox] # Check the bbox if len(bbox) != 4: diff --git a/tools/RAiDER/dem.py b/tools/RAiDER/dem.py index cd15939d8..bfd936e89 100644 --- a/tools/RAiDER/dem.py +++ b/tools/RAiDER/dem.py @@ -30,7 +30,7 @@ def getHeights(ll_bounds, dem_type, dem_file, lats=None, lons=None): if dem_type == 'hgt': htinfo = get_file_and_band(dem_file) hts = rio_open(htinfo[0], band=htinfo[1]) - + elif dem_type == 'csv': # Heights are in the .csv file hts = pd.read_csv(dem_file)['Hgt_m'].values @@ -42,7 +42,7 @@ def getHeights(ll_bounds, dem_type, dem_file, lats=None, lons=None): elif (dem_type == 'download') or (dem_type == 'dem'): if ~os.path.exists(dem_file): download_dem( - ll_bounds, + ll_bounds, writeDEM = True, outName=dem_file, ) @@ -50,7 +50,7 @@ def getHeights(ll_bounds, dem_type, dem_file, lats=None, lons=None): # Interpolate to the query points hts = interpolateDEM( dem_file, - lats, + lats, lons, ) @@ -62,28 +62,30 @@ def download_dem( save_flag='new', writeDEM=False, outName='warpedDEM', - buf=0.02 + buf=0.02, + overwrite=False, ): - ''' Download a DEM if one is not already present. ''' - folder = os.path.dirname(outName) - # inExtent is SNWE - # dem-stitcher wants WSEN - bounds = [ - np.floor(ll_bounds[2]) - buf, np.floor(ll_bounds[0]) - buf, - np.ceil(ll_bounds[3]) + buf, np.ceil(ll_bounds[1]) + buf - ] + """ Download a DEM if one is not already present. """ + if os.path.exists(outName) and not overwrite: + zvals, metadata = rio_open(outName, returnProj=True) - zvals, metadata = stitch_dem( - bounds, - dem_name='glo_30', - dst_ellipsoidal_height=True, - dst_area_or_point='Area', - ) - if writeDEM: - dem_file = os.path.join(folder, 'GLO30_fullres_dem.tif') - with rasterio.open(dem_file, 'w', **metadata) as ds: - ds.write(zvals, 1) - ds.update_tags(AREA_OR_POINT='Point') + else: + # inExtent is SNWE + # dem-stitcher wants WSEN + bounds = [ + np.floor(ll_bounds[2]) - buf, np.floor(ll_bounds[0]) - buf, + np.ceil(ll_bounds[3]) + buf, np.ceil(ll_bounds[1]) + buf + ] - return zvals, metadata + zvals, metadata = stitch_dem( + bounds, + dem_name='glo_30', + dst_ellipsoidal_height=True, + dst_area_or_point='Area', + ) + if writeDEM: + with rasterio.open(outName, 'w', **metadata) as ds: + ds.write(zvals, 1) + ds.update_tags(AREA_OR_POINT='Point') + return zvals, metadata From 7f21ab80d9a7cd2ee014daa1005cb379db35e187 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Thu, 8 Dec 2022 17:50:07 -0800 Subject: [PATCH 18/82] tweak to validators --- tools/RAiDER/cli/validators.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 55a87592e..6cbcb2497 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -33,18 +33,18 @@ def enforce_wm(value): def get_los(args): - if ('orbit_file' in args.keys()) and (args['orbit_file'] is not None): + if arg.get('orbit_file'): if args.ray_trace: los = Raytracing(args.orbit_file) else: los = Conventional(args.orbit_file) - elif ('los_file' in args.keys()) and (args['los_file'] is not None): + elif args.get('los_file'): if args.ray_trace: los = Raytracing(args.los_file, args.los_convention) else: los = Conventional(args.los_file, args.los_convention) - elif ('los_cube' in args.keys()) and (args['los_cube'] is not None): + elif args.get('los_cube'): raise NotImplementedError('LOS_cube is not yet implemented') # if args.ray_trace: # los = Raytracing(args.los_cube) @@ -132,7 +132,14 @@ def get_query_region(args): query = BoundingBox(bbox) elif 'geocoded_file' in args.keys(): - query = GeocodedFile(args.geocoded_file, is_dem=False) + gfile = os.path.basename(args.geocoded_file).upper() + if (gfile.startswith('SRTM') or gfile.startswith('GLO')): + logger.debug('Using user DEM: %s', gfile) + is_dem = True + else: + is_dem = False + + query = GeocodedFile(args.geocoded_file, is_dem=is_dem) ## untested elif 'los_cube' in args.keys(): From 5b07e7b05adca9fce14f3a60672e19d6a4d2b9f5 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 11:59:21 -0800 Subject: [PATCH 19/82] correct typo --- tools/RAiDER/cli/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 6cbcb2497..71ded110e 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -33,7 +33,7 @@ def enforce_wm(value): def get_los(args): - if arg.get('orbit_file'): + if args.get('orbit_file'): if args.ray_trace: los = Raytracing(args.orbit_file) else: From b5ab011b849752007d5ebdfa33406740cfda36ec Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 15:33:19 -0600 Subject: [PATCH 20/82] separate out second interp call to its own function for now --- tools/RAiDER/delay.py | 4 ++-- tools/RAiDER/delayFcns.py | 43 +++++++++++++++++++++++++-------------- 2 files changed, 30 insertions(+), 17 deletions(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 077a4fc84..1eafb765c 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -22,7 +22,7 @@ from RAiDER.constants import _STEP from RAiDER.delayFcns import ( - getInterpolators, + getInterpolators, getInterpolators2 ) from RAiDER.logger import logger, logging from RAiDER.losreader import Zenith, Conventional, Raytracing, get_sv, getTopOfAtmosphere @@ -75,7 +75,7 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 lons, lats = aoi.readLL() hgts = aoi.readZ() pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj).T - ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' + ifWet, ifHydro = getInterpolators2(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index 321646763..494e2b19a 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -91,15 +91,8 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): an interpolator ''' # Get the weather model data - if isinstance(wm_file, str): - with Dataset(wm_file, mode='r') as ds: - xs_wm = np.array(ds.variables['x'][:]) - ys_wm = np.array(ds.variables['y'][:]) - zs_wm = np.array(ds.variables['z'][:]) - wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] - hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] - else: - ds = wm_file + + with Dataset(wm_file, mode='r') as ds: xs_wm = np.array(ds.variables['x'][:]) ys_wm = np.array(ds.variables['y'][:]) zs_wm = np.array(ds.variables['z'][:]) @@ -118,16 +111,36 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): wet = make_shared_raw(wet) hydro = make_shared_raw(hydro) - try: - ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet, fill_value=np.nan, bounds_error = False) - ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro, fill_value=np.nan, bounds_error = False) - except ValueError: - ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet.transpose(1,0,2), fill_value=np.nan, bounds_error = False) - ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro.transpose(1,0,2), fill_value=np.nan, bounds_error = False) + ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet, fill_value=np.nan, bounds_error = False) + ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro, fill_value=np.nan, bounds_error = False) + + return ifWet, ifHydro + + +def getInterpolators2(ds, kind='pointwise', shared=False): + '''This replicates getInterpolators for handling the output from tropo_delay_cube''' + xs_wm = np.array(ds.variables['x'][:]) + ys_wm = np.array(ds.variables['y'][:]) + zs_wm = np.array(ds.variables['z'][:]) + wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] + + # If shared interpolators are requested + # The arrays are not modified - so turning off lock for performance + if shared: + xs_wm = make_shared_raw(xs_wm) + ys_wm = make_shared_raw(ys_wm) + zs_wm = make_shared_raw(zs_wm) + wet = make_shared_raw(wet) + hydro = make_shared_raw(hydro) + + ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet.transpose(1,0,2), fill_value=np.nan, bounds_error = False) + ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro.transpose(1,0,2), fill_value=np.nan, bounds_error = False) return ifWet, ifHydro + def make_shared_raw(inarr): """ Make numpy view array of mp.Array From 86b5ad72f2ceb84fdc61c6edaaea4ccf53adb229 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 12:47:21 -0800 Subject: [PATCH 21/82] bug fix for geocoded files; log on write --- tools/RAiDER/cli/raider.py | 22 +++++++++++----------- tools/RAiDER/delay.py | 19 +++++++++++-------- tools/RAiDER/utilFcns.py | 2 ++ 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 9b232539e..684e069d5 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -96,7 +96,7 @@ def create_parser(): help='generate default template (if it does not exist) and exit.' ) - + p.add_argument( '--download-only', action='store_true', @@ -282,12 +282,12 @@ def main(iargs=None): except RuntimeError: logger.exception("Date %s failed", t) continue - + # Now process the delays try: wet_delay, hydro_delay = tropo_delay( - t, weather_model_file, aoi, los, - params['height_levels'], + t, weather_model_file, aoi, los, + params['height_levels'], params['output_projection'], params['look_dir'], params['cube_spacing_in_m'] @@ -295,7 +295,7 @@ def main(iargs=None): except RuntimeError: logger.exception("Date %s failed", t) continue - + ########################################################### # Write the delays to file # Different options depending on the inputs @@ -308,14 +308,14 @@ def main(iargs=None): f = f.replace("_std", "_ray") else: out_filename = w - + if hydro_delay is None: # means that a dataset was returned ds = wet_delay - ext = os.path.splitext(out_filename) + ext = os.path.splitext(out_filename)[1] if ext not in ['.nc', '.h5']: out_filename = f'{os.path.splitext(out_filename)[0]}.nc' - + out_filename = out_filename.replace("wet", "tropo") if out_filename.endswith(".nc"): @@ -323,11 +323,11 @@ def main(iargs=None): elif out_filename.endswith(".h5"): ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) + logger.info('Wrote delays to: %s', out_filename) + else: if aoi.type() == 'station_file': w = f'{os.path.splitext(out_filename)[0]}.csv' - if aoi.type() in ['station_file', 'radar_rasters']: + if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) - - diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 1eafb765c..571cac17e 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -42,7 +42,7 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 weather_model_File: string - Name of the NETCDF file containing a pre-processed weather model aoi: AOI object - AOI object los: LOS object - LOS object - height_levels: list - (optional) list of height levels on which to calculate delays. Only needed for cube generation. + height_levels: list - (optional) list of height levels on which to calculate delays. Only needed for cube generation. out_proj: int,str - (optional) EPSG code for output projection look_dir: str - (optional) Satellite look direction. Only needed for slant delay calculation cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes @@ -109,7 +109,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 if (cube_spacing_m is None) and (crs == CRS(4326)): use_weather_model_cube = True - + else: #TODO handle this better if cube_spacing_m is None: @@ -286,15 +286,20 @@ def transformPoints(lats, lons, hgts, old_proj, new_proj): t = Transformer.from_crs(old_proj, new_proj) # Flags for flipping inputs or outputs - in_flip = old_proj.axis_info[0].direction == "east" - out_flip = new_proj.axis_info[0].direction == "east" + if not isinstance(new_proj, pyproj.CRS): + new_proj = CRS.from_epsg(new_proj.lstrip('EPSG:')) + if not isinstance(old_proj, pyproj.CRS): + old_proj = CRS.from_epsg(old_proj.lstrip('EPSG:')) + + in_flip = old_proj.axis_info[0].direction + out_flip = new_proj.axis_info[0].direction - if in_flip: + if in_flip == 'east': res = t.transform(lons, lats, hgts) else: res = t.transform(lats, lons, hgts) - if out_flip: + if out_flip == 'east': return np.stack((res[1], res[0], res[2]), axis=-1).T else: return np.stack(res, axis=-1).T @@ -550,5 +555,3 @@ def build_cube_ray( if output_created_here: return outputArrs - - diff --git a/tools/RAiDER/utilFcns.py b/tools/RAiDER/utilFcns.py index e5cfb86f2..6c4742aa2 100755 --- a/tools/RAiDER/utilFcns.py +++ b/tools/RAiDER/utilFcns.py @@ -267,6 +267,8 @@ def writeArrayToRaster(array, filename, noDataValue=0., fmt='ENVI', proj=None, g dtype=dtype, crs=proj, nodata=noDataValue, driver=fmt, transform=trans) as dst: dst.write(array, 1) + logger.info('Wrote: %s', filename) + return def writeArrayToFile(lats, lons, array, filename, noDataValue=-9999): From e10211e884b19bf1518f7196e36573257490ba27 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 12:58:32 -0800 Subject: [PATCH 22/82] add log to validators --- tools/RAiDER/cli/validators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 71ded110e..e3dce74f5 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -15,6 +15,7 @@ from RAiDER.llreader import BoundingBox, Geocube, RasterRDR, StationFile, GeocodedFile, Geocube from RAiDER.losreader import Zenith, Conventional, Raytracing from RAiDER.utilFcns import rio_extents, rio_profile +from RAiDER.logger import logger _BUFFER_SIZE = 0.2 # default buffer size in lat/lon degrees From a8732632a7074887fe2d41b2021eec11040c9b78 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 14:18:43 -0800 Subject: [PATCH 23/82] fix naming so test 3 passes --- ...000_std.nc => HRRR_tropo_20181113T230000_ray.nc} | Bin test/test_scenario_3.py | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) rename test/scenario_3/{HRRR_tropo_20181113T230000_std.nc => HRRR_tropo_20181113T230000_ray.nc} (100%) diff --git a/test/scenario_3/HRRR_tropo_20181113T230000_std.nc b/test/scenario_3/HRRR_tropo_20181113T230000_ray.nc similarity index 100% rename from test/scenario_3/HRRR_tropo_20181113T230000_std.nc rename to test/scenario_3/HRRR_tropo_20181113T230000_ray.nc diff --git a/test/test_scenario_3.py b/test/test_scenario_3.py index e1d9e18e0..b168508dd 100644 --- a/test/test_scenario_3.py +++ b/test/test_scenario_3.py @@ -16,12 +16,12 @@ def test_scenario_3(): process = subprocess.run(['raider.py', test_path],stdout=subprocess.PIPE, universal_newlines=True) assert process.returncode == 0 - new_data = xr.load_dataset('HRRR_tropo_20181113T230000_std.nc') - golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20181113T230000_std.nc')) + new_data = xr.load_dataset('HRRR_tropo_20181113T230000_ray.nc') + golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20181113T230000_ray.nc')) assert np.allclose(golden_data['wet'], new_data['wet']) assert np.allclose(golden_data['hydro'], new_data['hydro']) # Clean up files subprocess.run(['rm', '-f', './HRRR*']) - subprocess.run(['rm', '-rf', './weather_files']) \ No newline at end of file + subprocess.run(['rm', '-rf', './weather_files']) From 7e725bf3eb8f545aae151927f4fa9c73ba5d6eff Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 14:54:10 -0800 Subject: [PATCH 24/82] bugfix for validators --- tools/RAiDER/cli/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index e3dce74f5..e763e643b 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -35,7 +35,7 @@ def enforce_wm(value): def get_los(args): if args.get('orbit_file'): - if args.ray_trace: + if args.get('ray_trace'): los = Raytracing(args.orbit_file) else: los = Conventional(args.orbit_file) From 7850dcfe092c368b868a72e81eb716b7eedb5605 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 16:20:16 -0800 Subject: [PATCH 25/82] better DEM handling --- tools/RAiDER/cli/validators.py | 7 +++--- tools/RAiDER/delay.py | 4 ++-- tools/RAiDER/dem.py | 10 ++++----- tools/RAiDER/llreader.py | 41 +++++++++++++++++++--------------- 4 files changed, 33 insertions(+), 29 deletions(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index e763e643b..5eba6a9d0 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -116,12 +116,13 @@ def get_query_region(args): ''' # Get bounds from the inputs # make sure this is first - if ('use_dem_latlon' in args.keys()) and args['use_dem_latlon']: + if args.get('use_dem_latlon'): query = GeocodedFile(args.dem, is_dem=True) elif 'lat_file' in args.keys(): - hgt_file = args.get('hgt_file_rdr', None) # only get it if exists - query = RasterRDR(args.lat_file, args.lon_file, hgt_file) + hgt_file = args.get('hgt_file_rdr') # only get it if exists + dem_file = args.get('dem') + query = RasterRDR(args.lat_file, args.lon_file, hgt_file, dem_file) elif 'station_file' in args.keys(): query = StationFile(args.station_file) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 571cac17e..1a8e2052a 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -93,7 +93,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 raider cube generation function. """ # For testing multiprocessing - # TODO - move this to configuration + # TODO - move this to configura0ion crs = CRS(out_proj) # Determine the output grid extent here @@ -103,7 +103,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 ) # Clip output grid to multiples of spacing - # If output is desired in degrees + # If output is desired in degreezMs use_weather_model_cube = False if (cube_spacing_m is None) and (crs == CRS(4326)): diff --git a/tools/RAiDER/dem.py b/tools/RAiDER/dem.py index bfd936e89..0327f4bab 100644 --- a/tools/RAiDER/dem.py +++ b/tools/RAiDER/dem.py @@ -41,11 +41,8 @@ def getHeights(ll_bounds, dem_type, dem_file, lats=None, lons=None): elif (dem_type == 'download') or (dem_type == 'dem'): if ~os.path.exists(dem_file): - download_dem( - ll_bounds, - writeDEM = True, - outName=dem_file, - ) + download_dem(ll_bounds, writeDEM=True, outName=dem_file) + #TODO: interpolate heights to query lats/lons # Interpolate to the query points hts = interpolateDEM( @@ -59,7 +56,6 @@ def getHeights(ll_bounds, dem_type, dem_file, lats=None, lons=None): def download_dem( ll_bounds, - save_flag='new', writeDEM=False, outName='warpedDEM', buf=0.02, @@ -67,6 +63,7 @@ def download_dem( ): """ Download a DEM if one is not already present. """ if os.path.exists(outName) and not overwrite: + logger.info('Using existing DEM: %s', outName) zvals, metadata = rio_open(outName, returnProj=True) else: @@ -87,5 +84,6 @@ def download_dem( with rasterio.open(outName, 'w', **metadata) as ds: ds.write(zvals, 1) ds.update_tags(AREA_OR_POINT='Point') + logger.info('Wrote DEM: %s', outName) return zvals, metadata diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index 9dea3930c..a664b7999 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -16,6 +16,7 @@ from RAiDER.dem import download_dem from RAiDER.interpolator import interpolateDEM from RAiDER.utilFcns import rio_extents, rio_open, rio_profile, rio_stats, get_file_and_band +from RAiDER.logger import logger class AOI(object): @@ -25,10 +26,12 @@ class AOI(object): def __init__(self): self._bounding_box = None self._proj = CRS.from_epsg(4326) - + + def type(self): return self._type + def bounds(self): return self._bounding_box @@ -61,10 +64,12 @@ def __init__(self, station_file): self._bounding_box = bounds_from_csv(station_file) self._type = 'station_file' + def readLL(self): df = pd.read_csv(self._filename).drop_duplicates(subset=["Lat", "Lon"]) return df['Lat'].values, df['Lon'].values + def readZ(self): df = pd.read_csv(self._filename) if 'Hgt_m' in df.columns: @@ -80,7 +85,7 @@ def readZ(self): class RasterRDR(AOI): - def __init__(self, lat_file, lon_file=None, hgt_file=None, convention='isce'): + def __init__(self, lat_file, lon_file=None, hgt_file=None, dem_file=None, convention='isce'): AOI.__init__(self) self._type = 'radar_rasters' # allow for 2-band lat/lon raster @@ -91,8 +96,9 @@ def __init__(self, lat_file, lon_file=None, hgt_file=None, convention='isce'): self._lonfile = lon_file self._proj, self._bounding_box, _ = bounds_from_latlon_rasters(lat_file, lon_file) - # keep track of the height file + # keep track of the height file, dem and convention self._hgtfile = hgt_file + self._demfile = dem_file self._convention = convention def readLL(self): @@ -105,13 +111,16 @@ def readLL(self): def readZ(self): - if self._hgtfile is not None: + demFile = 'GLO30_fullres_dem.tif' if self._demfile is None else self._demfile + if self._hgtfile is not None and os.path.exists(self._hgtfile): + logger.info('Using existing heights at: %s', self._hgtfile) return rio_open(self._hgtfile) + else: zvals, metadata = download_dem( self._bounding_box, - writeDEM = True, - outName = os.path.join('GLO30_fullres_dem.tif'), + writeDEM=True, + outName=os.path.join(demFile), ) z_bounds = get_bbox(metadata) z_out = interpolateDEM(zvals, z_bounds, self.readLL(), method='nearest') @@ -125,6 +134,7 @@ def __init__(self, bbox): self._bounding_box = bbox self._type = 'bounding_box' + class GeocodedFile(AOI): '''Parse a Geocoded file for coordinates''' def __init__(self, filename, is_dem=False): @@ -150,18 +160,12 @@ def readLL(self): def readZ(self): - if self._is_dem: - return rio_open(self._filename) - - else: - zvals, metadata = download_dem( - self._bounding_box, - writeDEM = True, - outName = os.path.join('GLO30_fullres_dem.tif'), - ) - z_bounds = get_bbox(metadata) - z_out = interpolateDEM(zvals, z_bounds, self.readLL(), method='nearest') - return z_out + demFile = self._filename if self._is_dem else 'GLO30_fullres_dem.tif' + bbox = self._bounding_box + zvals, metadata = download_dem(bbox, writeDEM=True, outName=demFile) + z_bounds = get_bbox(metadata) + z_out = interpolateDEM(zvals, z_bounds, self.readLL(), method='nearest') + return z_out class Geocube(AOI): @@ -171,6 +175,7 @@ def __init__(self): self._type = 'geocube' raise NotImplementedError + def readLL(self): return None From 158b5f7c9a0603d1d20b5dd9b6d4ccf3360243f7 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Fri, 9 Dec 2022 17:02:03 -0800 Subject: [PATCH 26/82] another bugfix in validator --- tools/RAiDER/cli/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 5eba6a9d0..1f1c05f94 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -120,7 +120,7 @@ def get_query_region(args): query = GeocodedFile(args.dem, is_dem=True) elif 'lat_file' in args.keys(): - hgt_file = args.get('hgt_file_rdr') # only get it if exists + hgt_file = args.get('height_file_rdr') # only get it if exists dem_file = args.get('dem') query = RasterRDR(args.lat_file, args.lon_file, hgt_file, dem_file) From 4c8b05af59ffeba1a8f32247da4bffb699494f2e Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 20:58:43 -0600 Subject: [PATCH 27/82] Update delay.py --- tools/RAiDER/delay.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 1a8e2052a..571cac17e 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -93,7 +93,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 raider cube generation function. """ # For testing multiprocessing - # TODO - move this to configura0ion + # TODO - move this to configuration crs = CRS(out_proj) # Determine the output grid extent here @@ -103,7 +103,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 ) # Clip output grid to multiples of spacing - # If output is desired in degreezMs + # If output is desired in degrees use_weather_model_cube = False if (cube_spacing_m is None) and (crs == CRS(4326)): From 70184c655b259c84f90cf892bbabc70bbf2bb1e5 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 20:55:41 -0600 Subject: [PATCH 28/82] convert interp dataarrays to arrays --- tools/RAiDER/delayFcns.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index 494e2b19a..68fc1c017 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -122,8 +122,8 @@ def getInterpolators2(ds, kind='pointwise', shared=False): xs_wm = np.array(ds.variables['x'][:]) ys_wm = np.array(ds.variables['y'][:]) zs_wm = np.array(ds.variables['z'][:]) - wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] - hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] + wet = np.array(ds.variables['wet_total' if kind=='total' else 'wet'][:]) + hydro = np.array(ds.variables['hydro_total' if kind=='total' else 'hydro'][:]) # If shared interpolators are requested # The arrays are not modified - so turning off lock for performance From f29ef75e5ef22acd5cbf20b657d78d5be856cee6 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 22:21:01 -0600 Subject: [PATCH 29/82] fix some ordering bugs and clean up --- tools/RAiDER/cli/raider.py | 17 ++++++------- tools/RAiDER/delay.py | 7 +++--- tools/RAiDER/delayFcns.py | 39 +++++++---------------------- tools/RAiDER/llreader.py | 2 +- tools/RAiDER/models/hrrr.py | 3 ++- tools/RAiDER/models/weatherModel.py | 2 +- 6 files changed, 23 insertions(+), 47 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 684e069d5..d3f0c90fc 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -195,12 +195,6 @@ def read_template_file(fname): template.update(enforce_time(AttributeDict(value))) if key == 'date_group': template['date_list'] = parse_dates(AttributeDict(value)) - if key == 'aoi_group': - ## in case a DEM is passed and should be used - dct_temp = {**AttributeDict(value), - **AttributeDict(params['height_group'])} - template['aoi'] = get_query_region(AttributeDict(dct_temp)) - if key == 'los_group': template['los'] = get_los(AttributeDict(value)) if key == 'look_dir': @@ -219,7 +213,12 @@ def read_template_file(fname): template['bounding_box'], ) ) - return AttributeDict(template) + + template.update(params['aoi_group']) + template = AttributeDict(template) + template['aoi'] = get_query_region(template) + + return template def drop_nans(d): @@ -244,9 +243,7 @@ def main(iargs=None): # Argument checking params = checkArgs(params) - - params['download_only'] = inps.download_only - + if not params.verbose: logger.setLevel(logging.INFO) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 571cac17e..750efd779 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -59,7 +59,6 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 #TODO: expose this as library function ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, los, out_proj = out_proj, cube_spacing_m = cube_spacing_m, look_dir = look_dir) - ds = ds.rename({'x': 'y', 'y': 'x'}) if (aoi.type() == 'bounding_box') or (aoi.type() == 'Geocube'): return ds, None @@ -72,10 +71,10 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 out_proj = out_proj pnt_proj = CRS.from_epsg(4326) - lons, lats = aoi.readLL() + lats, lons = aoi.readLL() hgts = aoi.readZ() - pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj).T - ifWet, ifHydro = getInterpolators2(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' + pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj).transpose(1,2,0) + ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index 68fc1c017..ab5f8b7b6 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -91,13 +91,16 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): an interpolator ''' # Get the weather model data + try: + ds = xarray.load_dataset(wm_file) + except: + ds = wm_file - with Dataset(wm_file, mode='r') as ds: - xs_wm = np.array(ds.variables['x'][:]) - ys_wm = np.array(ds.variables['y'][:]) - zs_wm = np.array(ds.variables['z'][:]) - wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] - hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] + xs_wm = np.array(ds.variables['x'][:]) + ys_wm = np.array(ds.variables['y'][:]) + zs_wm = np.array(ds.variables['z'][:]) + wet = ds.variables['wet_total' if kind=='total' else 'wet'][:] + hydro = ds.variables['hydro_total' if kind=='total' else 'hydro'][:] wet = np.array(wet).transpose(1, 2, 0) hydro = np.array(hydro).transpose(1, 2, 0) @@ -117,30 +120,6 @@ def getInterpolators(wm_file, kind='pointwise', shared=False): return ifWet, ifHydro -def getInterpolators2(ds, kind='pointwise', shared=False): - '''This replicates getInterpolators for handling the output from tropo_delay_cube''' - xs_wm = np.array(ds.variables['x'][:]) - ys_wm = np.array(ds.variables['y'][:]) - zs_wm = np.array(ds.variables['z'][:]) - wet = np.array(ds.variables['wet_total' if kind=='total' else 'wet'][:]) - hydro = np.array(ds.variables['hydro_total' if kind=='total' else 'hydro'][:]) - - # If shared interpolators are requested - # The arrays are not modified - so turning off lock for performance - if shared: - xs_wm = make_shared_raw(xs_wm) - ys_wm = make_shared_raw(ys_wm) - zs_wm = make_shared_raw(zs_wm) - wet = make_shared_raw(wet) - hydro = make_shared_raw(hydro) - - ifWet = Interpolator((ys_wm, xs_wm, zs_wm), wet.transpose(1,0,2), fill_value=np.nan, bounds_error = False) - ifHydro = Interpolator((ys_wm, xs_wm, zs_wm), hydro.transpose(1,0,2), fill_value=np.nan, bounds_error = False) - - return ifWet, ifHydro - - - def make_shared_raw(inarr): """ Make numpy view array of mp.Array diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index a664b7999..7c8a7c835 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -111,12 +111,12 @@ def readLL(self): def readZ(self): - demFile = 'GLO30_fullres_dem.tif' if self._demfile is None else self._demfile if self._hgtfile is not None and os.path.exists(self._hgtfile): logger.info('Using existing heights at: %s', self._hgtfile) return rio_open(self._hgtfile) else: + demFile = 'GLO30_fullres_dem.tif' if self._demfile is None else self._demfile zvals, metadata = download_dem( self._bounding_box, writeDEM=True, diff --git a/tools/RAiDER/models/hrrr.py b/tools/RAiDER/models/hrrr.py index f9e9c252a..385ff9d1a 100644 --- a/tools/RAiDER/models/hrrr.py +++ b/tools/RAiDER/models/hrrr.py @@ -88,7 +88,8 @@ def load_weather(self, *args, filename=None, **kwargs): filename = self.files # read data from grib file - ds = xarray.open_dataset(filename) + ds = xarray.open_dataset(filename, engine='cfgrib') + pl = np.array([self._convertmb2Pa(p) for p in ds.levels.values]) xArr = ds['x'].values yArr = ds['y'].values diff --git a/tools/RAiDER/models/weatherModel.py b/tools/RAiDER/models/weatherModel.py index 3f0ec1cf7..76b9811c7 100755 --- a/tools/RAiDER/models/weatherModel.py +++ b/tools/RAiDER/models/weatherModel.py @@ -198,7 +198,7 @@ def load( Calls the load_weather method. Each model class should define a load_weather method appropriate for that class. 'args' should be one or more filenames. ''' - self.set_latlon_bounds(ll_bounds, Nextra=0) + self.set_latlon_bounds(ll_bounds) # If the weather file has already been processed, do nothing self._out_name = self.out_file(outLoc) From a817c331fe5ce53a4a0f78906033c7bdd635dc18 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 22:21:32 -0600 Subject: [PATCH 30/82] ensure consistency with all aoi readLL --- tools/RAiDER/llreader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index 7c8a7c835..511fecf2f 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -177,7 +177,7 @@ def __init__(self): def readLL(self): - return None + return None, None def bounds_from_latlon_rasters(latfile, lonfile): From 55f083764a2e761d014bb59df8da151829088e8a Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 22:28:01 -0600 Subject: [PATCH 31/82] remove uneeded import --- tools/RAiDER/delay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 750efd779..5b2890e2d 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -22,7 +22,7 @@ from RAiDER.constants import _STEP from RAiDER.delayFcns import ( - getInterpolators, getInterpolators2 + getInterpolators ) from RAiDER.logger import logger, logging from RAiDER.losreader import Zenith, Conventional, Raytracing, get_sv, getTopOfAtmosphere From 5cb8b3e240457423c1b718b8a2a62a4ede6f8c3d Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Fri, 9 Dec 2022 23:02:18 -0600 Subject: [PATCH 32/82] need to handle cases where key is present but empty --- tools/RAiDER/cli/validators.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/RAiDER/cli/validators.py b/tools/RAiDER/cli/validators.py index 1f1c05f94..21352d312 100755 --- a/tools/RAiDER/cli/validators.py +++ b/tools/RAiDER/cli/validators.py @@ -119,21 +119,21 @@ def get_query_region(args): if args.get('use_dem_latlon'): query = GeocodedFile(args.dem, is_dem=True) - elif 'lat_file' in args.keys(): + elif args.get('lat_file'): hgt_file = args.get('height_file_rdr') # only get it if exists dem_file = args.get('dem') query = RasterRDR(args.lat_file, args.lon_file, hgt_file, dem_file) - elif 'station_file' in args.keys(): + elif args.get('station_file'): query = StationFile(args.station_file) - elif 'bounding_box' in args.keys(): + elif args.get('bounding_box'): bbox = enforce_bbox(args.bounding_box) if (np.min(bbox[0]) < -90) | (np.max(bbox[1]) > 90): raise ValueError('Lats are out of N/S bounds; are your lat/lon coordinates switched? Should be SNWE') query = BoundingBox(bbox) - elif 'geocoded_file' in args.keys(): + elif args.get('geocoded_file'): gfile = os.path.basename(args.geocoded_file).upper() if (gfile.startswith('SRTM') or gfile.startswith('GLO')): logger.debug('Using user DEM: %s', gfile) From 6a14afa28d7fb8375411c30dd3a54af629a81c3b Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 07:24:31 -0600 Subject: [PATCH 33/82] hrrr fix --- tools/RAiDER/models/hrrr.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tools/RAiDER/models/hrrr.py b/tools/RAiDER/models/hrrr.py index 385ff9d1a..94e09f75c 100644 --- a/tools/RAiDER/models/hrrr.py +++ b/tools/RAiDER/models/hrrr.py @@ -88,7 +88,10 @@ def load_weather(self, *args, filename=None, **kwargs): filename = self.files # read data from grib file - ds = xarray.open_dataset(filename, engine='cfgrib') + try: + ds = xarray.open_dataset(filename, engine='cfgrib') + except EOFError: + ds = xarray.open_dataset(filename, engine='netcdf4') pl = np.array([self._convertmb2Pa(p) for p in ds.levels.values]) xArr = ds['x'].values @@ -176,15 +179,6 @@ def _makeDataCubes(self, filename, out=None): lats = lats[y_min:y_max, x_min:x_max] lons = lons[y_min:y_max, x_min:x_max] - # TODO: Remove this block once we know we handle - # different flavors of HRRR correctly - # self._proj currently used but these are available - # as attrs of ds["t"] for example - # llhtolcc = Transformer.from_crs(4326, self._proj) - # res = llhtolcc.transform(lats, lons) - # print("ERROR in X: ", np.abs(res[0] - xArr[None, :]).max()) - # print("ERROR in Y: ", np.abs(res[1] - yArr[:, None]).max()) - # Data variables t = ds['t'][:, y_min:y_max, x_min:x_max].to_numpy() z = ds['gh'][:, y_min:y_max, x_min:x_max].to_numpy() @@ -258,7 +252,7 @@ def _makeDataCubes(self, filename, out=None): for k, v in self._proj.to_cf().items(): ds_new.proj.attrs[k] = v - ds_new.to_netcdf(out) + ds_new.to_netcdf(out, engine='netcdf4') def _download_hrrr_file(self, DATE, out, model='hrrr', product='prs', fxx=0, verbose=False): ''' From 8e6a8fd48fb52d7e431763bc1a35f655f39da139 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 09:14:23 -0600 Subject: [PATCH 34/82] add some units tests and bug fixes for llreader --- test/test_llreader.py | 14 ++++++++++++++ tools/RAiDER/llreader.py | 24 +++++++++++++----------- 2 files changed, 27 insertions(+), 11 deletions(-) diff --git a/test/test_llreader.py b/test/test_llreader.py index dff8681a6..31d7062bb 100644 --- a/test/test_llreader.py +++ b/test/test_llreader.py @@ -33,6 +33,14 @@ def llfiles(): return os.path.join(SCENARIO1_DIR, 'lat.dat'), os.path.join(SCENARIO1_DIR, 'lon.dat') +def test_latlon_reader_2(): + with pytest.raises(ValueError): + query = RasterRDR(lat_file=None, lon_file=None) + + with pytest.raises(ValueError): + query = RasterRDR(lat_file='doesnotexist.rdr', lon_file='doesnotexist.rdr') + + def test_latlon_reader(): latfile = os.path.join(GEOM_DIR, 'lat.rdr') lonfile = os.path.join(GEOM_DIR, 'lon.rdr') @@ -88,3 +96,9 @@ def test_bounds_from_csv(station_file): bounds_true = [33.746, 36.795, -118.312, -114.892] snwe = bounds_from_csv(station_file) assert all([np.allclose(b, t) for b, t in zip(snwe, bounds_true)]) + + +def test_readZ_sf(station_file): + aoi = StationFile(station_file) + assert np.allclose(aoi.readZ(), .1) + diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index 511fecf2f..da7cfc8e7 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -10,6 +10,7 @@ import numpy as np import pandas as pd +import rasterio from pyproj import CRS @@ -88,13 +89,15 @@ class RasterRDR(AOI): def __init__(self, lat_file, lon_file=None, hgt_file=None, dem_file=None, convention='isce'): AOI.__init__(self) self._type = 'radar_rasters' - # allow for 2-band lat/lon raster - if (lon_file is None): - self._file = lat_file - else: - self._latfile = lat_file - self._lonfile = lon_file + self._latfile = lat_file + self._lonfile = lon_file + if (self._latfile is None) and (self._lonfile is None): + raise ValueError('You need to specify a 2-band file or two single-band files') + + try: self._proj, self._bounding_box, _ = bounds_from_latlon_rasters(lat_file, lon_file) + except rasterio.errors.RasterioIOError: + raise ValueError('Could not open {}, does it exist?'.format(self._latfile)) # keep track of the height file, dem and convention self._hgtfile = hgt_file @@ -102,12 +105,11 @@ def __init__(self, lat_file, lon_file=None, hgt_file=None, dem_file=None, conven self._convention = convention def readLL(self): - if self._latfile is not None: - return rio_open(self._latfile), rio_open(self._lonfile) - elif self._file is not None: - return rio_open(self._file) + # allow for 2-band lat/lon raster + if self._lonfile is None: + return rio_open(self._latfile) else: - raise ValueError('lat/lon files are not defined') + return rio_open(self._latfile), rio_open(self._lonfile) def readZ(self): From 2d49ebb31cda796694bce18b8d28f52e79a26e5f Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Sat, 10 Dec 2022 12:23:39 -0800 Subject: [PATCH 35/82] fix proj calc with lat/lon/orbit files --- tools/RAiDER/delay.py | 6 ++++-- tools/RAiDER/losreader.py | 31 ++++++++++++++++++------------- 2 files changed, 22 insertions(+), 15 deletions(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 5b2890e2d..e23cbca4e 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -58,7 +58,8 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 height_levels = ds.z.values #TODO: expose this as library function - ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, los, out_proj = out_proj, cube_spacing_m = cube_spacing_m, look_dir = look_dir) + ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, + los, out_proj=out_proj, cube_spacing_m=cube_spacing_m, look_dir=look_dir) if (aoi.type() == 'bounding_box') or (aoi.type() == 'Geocube'): return ds, None @@ -80,8 +81,9 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 # return the delays (ZTD or STD) if los.is_Projected(): + los.setTime(dt) los.setPoints(lats, lons, hgts) - wetDelay = los(wetDelay) + wetDelay = los(wetDelay) hydroDelay = los(hydroDelay) return wetDelay, hydroDelay diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 3bfad9ef4..44ba3ad90 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -51,16 +51,16 @@ def setPoints(self, lats, lons=None, heights=None): self._lats = lats self._lons = lons self._heights = heights - + def setTime(self, dt): self._time = dt - + def is_Zenith(self): return self._is_zenith - + def is_Projected(self): return self._is_projected - + def ray_trace(self): return self._ray_trace @@ -90,13 +90,13 @@ class Conventional(LOS): """ def __init__(self, filename=None, los_convention='isce', time=None, pad=None): - LOS.__init__(self) + super().__init__() self._file = filename self._time = time - self._pad = pad + self._pad = 600 self._is_projected = True - self._convention = los_convention - if self._convention == 'hyp3': + self._convention = los_convention + if self._convention.lower() != 'isce': raise NotImplementedError() def __call__(self, delays): @@ -109,8 +109,11 @@ def __call__(self, delays): try: # if an ISCE-style los file is passed open it with GDAL LOS_enu = inc_hd_to_enu(*rio_open(self._file)) + except OSError: # Otherwise, treat it as an orbit / statevector file + self._pad = 600 + self._time = datetime.datetime(2018, 11, 13, 23, 0) svs = np.stack( get_sv(self._file, self._time, self._pad), axis=-1 ) @@ -163,16 +166,15 @@ class Raytracing(LOS): >>> TODO """ - def __init__(self, filename=None, los_convention='isce', time=None, pad=None): + def __init__(self, filename=None, los_convention='isce', time=None, pad=600): '''read in and parse a statevector file''' LOS.__init__(self) self._ray_trace = True self._file = filename self._time = time - self._pad = pad + self._pad = pad self._convention = los_convention - self._pad = 600 - if self._convention == 'hyp3': + if self._convention.lower() != 'isce': raise NotImplementedError() def getLookVectors(self, time, pad=3 * 60): @@ -213,6 +215,7 @@ def getLookVectors(self, time, pad=3 * 60): out="ecef" ) + def getIntersectionWithHeight(self, height): """ This function computes the intersection point of a ray at a height @@ -221,6 +224,7 @@ def getIntersectionWithHeight(self, height): # We just leverage the same code as finding top of atmosphere here return getTopOfAtmosphere(self._xyz, self._look_vecs, height) + def getIntersectionWithLevels(self, levels): """ This function returns the points at which rays intersect the @@ -246,6 +250,7 @@ def getIntersectionWithLevels(self, levels): return rays + def calculateDelays(self, delays): ''' Here "delays" is point-wise delays (i.e. refractivities), not @@ -580,7 +585,7 @@ def get_radar_pos(llh, orb, out="lookangle"): # Get xyz positions of targets here targ_xyz = np.stack( - lla2ecef(llh[:, 0], llh[:, 1], llh[:, 2]), axis=-1 + lla2ecef(llh[:, 1], llh[:, 0], llh[:, 2]), axis=-1 ) # Get some isce3 constants for this inversion From ebd767451c25c0a66f9cb484415f1f35b7683c6f Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 15:04:00 -0600 Subject: [PATCH 36/82] Update losreader.py --- tools/RAiDER/losreader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 44ba3ad90..bbb5b3e7a 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -89,11 +89,11 @@ class Conventional(LOS): be projected using the standard cos(inc) scaling. """ - def __init__(self, filename=None, los_convention='isce', time=None, pad=None): + def __init__(self, filename=None, los_convention='isce', time=None, pad=600): super().__init__() self._file = filename self._time = time - self._pad = 600 + self._pad = pad self._is_projected = True self._convention = los_convention if self._convention.lower() != 'isce': From 868781662572fdcf6f11f0808c919285bb45e0c9 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 15:06:55 -0600 Subject: [PATCH 37/82] Update losreader.py --- tools/RAiDER/losreader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index bbb5b3e7a..52a714468 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -112,8 +112,6 @@ def __call__(self, delays): except OSError: # Otherwise, treat it as an orbit / statevector file - self._pad = 600 - self._time = datetime.datetime(2018, 11, 13, 23, 0) svs = np.stack( get_sv(self._file, self._time, self._pad), axis=-1 ) From 051c5b1c67c65de4ae0b4224e64494299589ca9a Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 15:29:28 -0600 Subject: [PATCH 38/82] add projection end-to-end test --- .../HRRR_tropo_20181113T230000_std.nc | Bin 0 -> 21582 bytes test/scenario_3/raider_example_3_proj.yaml | 18 ++++++++++++ test/scenario_3/raider_example_4.yaml | 17 ----------- test/test_scenario_3_proj.py | 27 ++++++++++++++++++ 4 files changed, 45 insertions(+), 17 deletions(-) create mode 100644 test/scenario_3/HRRR_tropo_20181113T230000_std.nc create mode 100644 test/scenario_3/raider_example_3_proj.yaml delete mode 100644 test/scenario_3/raider_example_4.yaml create mode 100644 test/test_scenario_3_proj.py diff --git a/test/scenario_3/HRRR_tropo_20181113T230000_std.nc b/test/scenario_3/HRRR_tropo_20181113T230000_std.nc new file mode 100644 index 0000000000000000000000000000000000000000..063de4f1a3ab7951da7680e3622727f50f23ed93 GIT binary patch literal 21582 zcmeG^3wTpS){~~K^u=2Qq!KD@3zpI(ZPW6DH*K0eptL0^&%zf%bKAyElbWU!a9tkC zLqLB-1(jWX6m}Kx!6LG&EZPONrL+pG0=nRW3Mzt%@bPd_{&VJ$Gy!W_t?ajEp!a0X zoH;Xd&Y3y$y7#`k?3|d$zL6oFIuQ{r^H5mAFv&b&j6t`mC@UQu-drga@j3A%VdkkUd7so@s{VN#kVlV zP<_-##r4lB$;$?thj=$2wk2qXBWpq;;2#SMcdO5Pt{~H}L?2oFx2o}1>WhmPx-tv@ zuD%8jX8BDwGq|0{g}&;!zIYq2b3fZSzxyhmz7jgII>Y!%gV2|diDITh{Dkx=mfEeD zSG0`uW4VrS^NYR>-G-8MhCvp(r=?K#l-bHtvh7uN8|ex~u|BJ!Vm#zZ3Xt(B&y6k-oq8119P6HAcR0pd9kudRy$$*%G0q6c7+shX z4VHlJo=*>IopukKMTmA)MZ=JQF8q7kFSZNY5B0y&;i_!1!#XQk%^Y|PbGsF8yUkK% zt**8^EBt060poVGLwM`1*vRbsF~&l3eo>*NAm3a<`aqdDJGu$}Q1{qKuP`Bw0o!nx z4>^MZ;^<>rr!&VhZ+!ZuTtiU-6%(C&F)ASDP8^y-)y=hj6}+egw@4xePGRl>FZc+v z7kEK0hF*ae9a>n4zzaSiE}+130eI^#_(x)+AOzA2Ll$^JAI-?U^rAj70eDev=K#Fm zgA-2-g1?~W;`ZVNJ^By~f%J4X)Cw=?aSn$;(2IJ44bdR*qTCw;@PZHRp^zr%1s}9H z3<59s2)8cqf*vgugTM=VKCAQM1w9`rym&#+$80ZN)H9Z`_u>VgTLSQc5BF+bdcj8( zKri?V2*3;axB$GMzcm0a=m!Si1wHmm41w~&9*RNW1s~Q^eR#nqfsuRZMSHnd_u>T~ zSQybD_zON9O<=?ehM)w35(r8lD1o2^f)WTyASi*L1cDOyFPA`$DX*9=7DDI-18=R^ z#2&|&&CiQvNZ_)SFDJyR`-g-Dy8>ckfbK^Sk^r!G3CpJ-rDJeiPQ7tZ*B0sI#}_~7dvl9)>gnCj+TvTJedpf(?G{ao zWavA_QkK;sy?RgVox`myQm^+Pc>VLBDx)rNSTBL?2 z;%`lPd$+XWUGlX#tx1Ya=vx?OXp(aNzB7KEsY#ms&|@=Z*_xzD6|dUHLwxw&$xF|> zo1_)`$|;BDH%TWJ%ZBikP15J@ZA)15Vv}@W-KVR^Kh`8YIpf*rcQ!Xk3q~(}qi$D| zbo{>DM&rRIY1^3n2XFAS0X-XO&udw9W&lMT`^ z+jr^VdmAL}o=?&*eAplr-}dgyz0WpC)2CEk_4Cw=X8=5X@7p^| zI!EcrJr_DR$3*F0SrvL>1Hi}L+g^SsI!fQ?sm4ur157y=zh3GRrSJRU(TB}IKlJ3L zgChWLxu@^M2Z8>BnB_yu050uc{fC!PY>l2?o{k zkRE!=vHcqWPEX%i83oX+9kBTeptmk~?|2_gr2d8H`u*iappWmiEnyVEt4;g4i?h5&C6yC69az@WFixA5Q^z;l!g|zWh~$UX`2u=eq|- z=r=YW$bJgwpGi4(9ARy#wR=~9CH*fiQ2?y|>SBUsNQC}{L%VK%2k58FOp)yXk8T_A zaBqOy#^1SS570gS7=#FOeMLEhaC3c%7J|}T&qgqF{hQ4YE}80E5IVB-l&(BWPt&Jm z>0z*F6(;FvL_dea|7DOOgO8vDf)e;|lfd;k<1FA4n>l=BlomMJB z#Y{G=a+b~QA_JM;*$J(SpNoxZWemG|OuXcqxGDI_$8|a7%x?3P6VEhoib&g46~`ILZiHyrO{j>IpbzO=mLzJRBazM|dB9A+Z(;RoK}K|{7I zowtOnK?G#ME+9wW{t$`5EU-aAbz@)iC$bn3zrj`#((CN0?Svj~XF?1@S+H zn+a*{y86dT{PDNWytVe-H^a*o$lfMbxMjJ;PV$Kf!6*XJo7d9|5WVnbWi=-{6EYxy#9X?@@UKd%cGYp`UPs` z=WFs$>i>mmTbl;+|J<}lYlR=#`^t!)uc5(`t||fQ|HT_3lwxQ2;({OL|FKVdJ?W3} z|Fhcb|5=G0;{Wl?68)s`|4x_NQ`yD@pcAKez+Oxn{{Kp{j(7hkB^m}Y;Ei*15c~oq z!2SOf%YC|3hm8Vs)r)py>HHRjbp1acrPbzgKLITaDCiplxZJ-B5pB79l8qGbIwa0R za<|=kvHrO)?uDPh&`}cjN&P>!7viV*)++V00OyI@zU-)G1^Kp90@VMDH+yNGhNs>> z`J?R9QNuOm)(Xhe0TJc9|~uI`+N}%!k`4G1lW=B)-d*!05n!*H16beg&8VN zQVG%feHH*IAj}o7?5b2W&S?n|hrbczFn__!#a}3UM7&s*&5veS#^5t3fuIC}5(rA* znk8^3r*Q&bzxBfX{{ozM5irTJ$d$JNK#<4ZvKVs6?l2{zMhq2X}<)cBblLAC!Fi)}T;{j^G2n~9%^`03iuH2jqAr|`MTv9EVb zqk>%9Edl!Kfq3ad0e$%!+W8cBg^&ewp4BHY?k&2=6Z0`TBrM(GXjZ~uEB#J~9X{0M zu8}R}uxaU$o%sEWt9>2nYTxTvT;*9AyWXu=!iO*Dk*M$ERo-=u@6;2A8XR3Fe*FrM=8j=C1MO2&{E?RzU0Th4_f^>krsOvHxrFHcr$q2!j&%nI+)g|FLk@ zt6!h+e*O-vz3$f@diNp~h7YvgTw_tL!DKFtV|_I)5k7Kb9FtXG1dPk=u&Huomrcg| zrsi2S9=S@T&PY#D!HopD3O>FRmpBH9jixq<6H44xxWV8V95=OH>3E=|*JVvSmxfyBmwg8X80QGT{h_-R^Qx>~DC)MTV5X;O6>jW#t+ zn^<5h%q__qSD0T?8doKI+#q)KRLIXYm1!`HGi4ekm1>fb0bOi@FBOd`jq?tCaf!)^ znT5FpM#h1iZ~-zYS*=S+)oL;{sp^ajcn1YC8_cEN@!aZ2)J#pxoR|;R0b?e!0x{mH z5(=!K(L{PqmRMwhZ#b5!fx4aL1acMEXn?KD32gED4d;q=XeZZk=FD7Uso79e4Cb5X za@%LSoF1z~RpzRyb~)iAl`2=cN}H{!p{AS!I>RH!d+nt4W(WRSB1GsYioeV7F9R?{K*-bStsF?i4zn{*NUSGXogeL7Lab sXz!|iJztY1W@2B5>#lkES>F8^is@ZU0qG5E1`OggQUOQ2RXOPY0VSgH4*&oF literal 0 HcmV?d00001 diff --git a/test/scenario_3/raider_example_3_proj.yaml b/test/scenario_3/raider_example_3_proj.yaml new file mode 100644 index 000000000..5f26ffb2b --- /dev/null +++ b/test/scenario_3/raider_example_3_proj.yaml @@ -0,0 +1,18 @@ +# vim: set filetype=yaml: +look_dir: right +date_group: + date_list: [20181113] +time_group: + time: "23:00:00" + end_time: +weather_model: HRRR +aoi_group: + bounding_box: 36.8 36.85 -76.15 -76.05 +height_group: + height_levels: 0 100 500 1000 +los_group: + ray_trace: False # Use projected slant delay by default + orbit_file: test/scenario_3/S1A_OPER_AUX_POEORB_OPOD_20181203T120749_V20181112T225942_20181114T005942.EOF +runtime_group: + output_projection: 4326 + cube_spacing_in_m: 5000.0 diff --git a/test/scenario_3/raider_example_4.yaml b/test/scenario_3/raider_example_4.yaml deleted file mode 100644 index ce6dc46bf..000000000 --- a/test/scenario_3/raider_example_4.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# vim: set filetype=yaml: - look_dir: right - date_group: - date_list: [20181113] - time_group: - time: "23:00:00" - end_time: - weather_model: HRRR - aoi_group: - bounding_box: 36.8 36.85 -76.15 -76.05 - los_group: - ray_trace: False # Use projected slant delay by default - orbit_file: test/scenario_3/S1A_OPER_AUX_POEORB_OPOD_20181203T120749_V20181112T225942_20181114T005942.EOF - runtime_group: - output_projection: 4326 - cube_spacing_in_m: 250.0 - \ No newline at end of file diff --git a/test/test_scenario_3_proj.py b/test/test_scenario_3_proj.py new file mode 100644 index 000000000..5e3c07af1 --- /dev/null +++ b/test/test_scenario_3_proj.py @@ -0,0 +1,27 @@ +import os +import pytest +import subprocess + +from test import TEST_DIR + +import numpy as np +import xarray as xr + + +# @pytest.mark.skip +def test_scenario_3_proj(): + SCENARIO_DIR = os.path.join(TEST_DIR, "scenario_3") + + test_path = os.path.join(SCENARIO_DIR, 'raider_example_3_proj.yaml') + process = subprocess.run(['raider.py', test_path],stdout=subprocess.PIPE, universal_newlines=True) + assert process.returncode == 0 + + new_data = xr.load_dataset('HRRR_tropo_20181113T230000_std.nc') + golden_data = xr.load_dataset(os.path.join(SCENARIO_DIR, 'HRRR_tropo_20181113T230000_std.nc')) + + assert np.allclose(golden_data['wet'], new_data['wet']) + assert np.allclose(golden_data['hydro'], new_data['hydro']) + + # Clean up files + subprocess.run(['rm', '-f', './HRRR*']) + subprocess.run(['rm', '-rf', './weather_files']) \ No newline at end of file From ce24cfef563955280f495bc33c4a738be6aafbfe Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Sat, 10 Dec 2022 21:09:17 -0600 Subject: [PATCH 39/82] fix bugs in out grid logic --- tools/RAiDER/delay.py | 60 ++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index e23cbca4e..27c81b260 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -97,6 +97,14 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 # TODO - move this to configuration crs = CRS(out_proj) + # Load CRS from weather model file + wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] + if wm_proj is None: + print("WARNING: I can't find a CRS in the weather model file, so I will assume you are using WGS84") + wm_proj = CRS.from_epsg(4326) + else: + wm_proj = CRS.from_wkt(wm_proj.to_wkt()) + # Determine the output grid extent here wesn = ll_bounds[2:] + ll_bounds[:2] out_snwe = transform_bbox( @@ -104,46 +112,28 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 ) # Clip output grid to multiples of spacing - # If output is desired in degrees - use_weather_model_cube = False - - if (cube_spacing_m is None) and (crs == CRS(4326)): - - use_weather_model_cube = True - + if cube_spacing_m is None: + with xarray.load_dataset(weather_model_file) as ds: + xpts = ds.x.values + ypts = ds.y.values + cube_spacing_m = np.nanmean([np.nanmean(np.diff(xpts)), np.nanmean(np.diff(ypts))]) + if wm_proj.axis_info[0].unit_name == "degree": + cube_spacing_m = cube_spacing_m * 1.0e5 # Scale by 100km + + if crs.axis_info[0].unit_name == "degree": + out_spacing = cube_spacing_m / 1.0e5 # Scale by 100km else: - #TODO handle this better - if cube_spacing_m is None: - cube_spacing_m = 5000 - - if crs.axis_info[0].unit_name == "degree": - out_spacing = cube_spacing_m / 1.0e5 # Scale by 100km - out_snwe = clip_bbox(out_snwe, out_spacing) - else: - out_spacing = cube_spacing_m - out_snwe = clip_bbox(out_snwe, out_spacing) - - logger.debug(f"Output SNWE: {out_snwe}") - logger.debug(f"Output cube spacing: {out_spacing}") + out_spacing = cube_spacing_m + out_snwe = clip_bbox(out_snwe, out_spacing) - - # Load CRS from weather model file - wm_proj = rio_profile(f"netcdf:{weather_model_file}:t")["crs"] - if wm_proj is None: - print("WARNING: I can't find a CRS in the weather model file, so I will assume you are using WGS84") - wm_proj = CRS.from_epsg(4326) - else: - wm_proj = CRS.from_wkt(wm_proj.to_wkt()) + logger.debug(f"Output SNWE: {out_snwe}") + logger.debug(f"Output cube spacing: {out_spacing}") # Build the output grid zpts = np.array(heights) - if not use_weather_model_cube: - xpts = np.arange(out_snwe[2], out_snwe[3] + out_spacing, out_spacing) - ypts = np.arange(out_snwe[1], out_snwe[0] - out_spacing, -out_spacing) - else: - with xarray.load_dataset(weather_model_file) as ds: - xpts = ds.x.values - ypts = ds.y.values + xpts = np.arange(out_snwe[2], out_snwe[3] + out_spacing, out_spacing) + ypts = np.arange(out_snwe[1], out_snwe[0] - out_spacing, -out_spacing) + # If no orbit is provided # Build zenith delay cube From 40d1acb06831e5069855f7ec55c58b2cf927f920 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 11:46:27 -0800 Subject: [PATCH 40/82] Restructure to use ++process and enable cfg generation from GUNW --- pyproject.toml | 17 +- tools/RAiDER/aria/calcARIA.py | 29 ++ tools/RAiDER/aria/prepFromARIA.py | 196 +++++++++++++ tools/RAiDER/cli/__init__.py | 41 +++ tools/RAiDER/cli/__main__.py | 438 ++++++++++++++++++++++++++++++ tools/RAiDER/cli/raider.py | 6 +- tools/RAiDER/prepFromAria.py | 107 -------- 7 files changed, 718 insertions(+), 116 deletions(-) create mode 100644 tools/RAiDER/aria/calcARIA.py create mode 100644 tools/RAiDER/aria/prepFromARIA.py create mode 100644 tools/RAiDER/cli/__main__.py delete mode 100644 tools/RAiDER/prepFromAria.py diff --git a/pyproject.toml b/pyproject.toml index 170f9830e..74ea80033 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,8 @@ authors = [ {name="David Bekaert", email="david.bekaert@jpl.nasa.gov"}, {name="Jeremy Maurer", email="maurer.jeremy@gmail.com"}, {name="Piyush Agram", email="piyush@descarteslabs.com"}, - {name="Simran Sangha", email="simran.s.sangha@jpl.nasa.gov"} + {name="Simran Sangha", email="simran.s.sangha@jpl.nasa.gov"}, + {name="Brett Buzzanga", email="buzzanga@jpl.nasa.gov"}, ] description = "Raytracing Atmospheric Delay Estimation for RADAR" readme = "README.md" @@ -40,11 +41,15 @@ repository = "https://github.com/dbekaert/RAiDER" "Bug Tracker" = "https://github.com/dbekaert/RAiDER/issues" [project.scripts] -"generateGACOSVRT.py" = "RAiDER.models.generateGACOSVRT:main" -"raiderDownloadGNSS.py" = "RAiDER.gnss.downloadGNSSDelays:main" -"raider.py" = "RAiDER.cli.raider:main" -"prepFromAria.py" = "RAiDER.cli.prepFromAria:main" -"raiderStats.py" = "RAiDER.cli.statsPlot:main" +"raider.py" = "RAiDER.cli.__main__:main" +"calcDelays.py" = "RAiDER.cli.__main__:calcDelays" +"calcDelaysARIA.py" = "RAiDER.cli.__main__:calcDelaysARIA" +"raiderDownloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" +"downloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" + +"prepFromARIA.py" = "RAiDER.cli.__main__:prepFromARIA" +"raiderStats.py" = "RAiDER.cli.statsPlot:main" +"generateGACOSVRT.py" = "RAiDER.models.generateGACOSVRT:main" [tool.setuptools] zip-safe = false diff --git a/tools/RAiDER/aria/calcARIA.py b/tools/RAiDER/aria/calcARIA.py new file mode 100644 index 000000000..a101f13b2 --- /dev/null +++ b/tools/RAiDER/aria/calcARIA.py @@ -0,0 +1,29 @@ +""" +Calculate the interferometric phase from the 4 delays files of a GUNW +Write it to disk +""" +import xarray as xr +import numpy as np + +## z is a placeholder +def get_delay(delayFile): + ds = xr.open_dataset(delayFile) + wet = ds['wet'].isel(z=0).data + hydro = ds['hydro'].isel(z=0).data + tot = wet + hydro + return tot + + +def main(delayFiles, update_nc): + """ Pull the wet and hydrostatic delays from the netcdf + + Calculate the interferometric phase delay + Write to disk, and optionally update the netcdf + """ + delays = np.stack([get_delay(delayFile) for delayFile in delayFiles]) + breakpoint() + + + +if __name__ == '__main__': + main() diff --git a/tools/RAiDER/aria/prepFromARIA.py b/tools/RAiDER/aria/prepFromARIA.py new file mode 100644 index 000000000..e27bc8864 --- /dev/null +++ b/tools/RAiDER/aria/prepFromARIA.py @@ -0,0 +1,196 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Author: Jeremy Maurer, Brett Buzzanga +# Copyright 2022, by the California Institute of Technology. ALL RIGHTS +# RESERVED. United States Government Sponsorship acknowledged. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +import os +from datetime import datetime +import numpy as np +import xarray as xr +import rasterio +import yaml +import RAiDER +from RAiDER.utilFcns import rio_open, writeArrayToRaster +from RAiDER.logger import logger + +""" Should be refactored into a class that takes filename as input """ +## ---------------------------------------------------- Prepare Input from GUNW +def makeLatLonGrid(f:str): +# def makeLatLonGrid(f:str, reg, out_dir): + ds0 = xr.open_dataset(f, group='science/grids/data') + + Lat, Lon = np.meshgrid(ds0.latitude.data, ds0.longitude.data) + + da_lat = xr.DataArray(Lat.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split()) + da_lon = xr.DataArray(Lon.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split()) + # dst_lat = op.join(out_dir, f'lat_{reg}.geo') + # dst_lon = op.join(out_dir, f'lon_{reg}.geo') + + dst_lat = f'lat.geo' + dst_lon = f'lon.geo' + da_lat.to_netcdf(dst_lat) + da_lon.to_netcdf(dst_lon) + logger.debug('Wrote: %s', dst_lat) + logger.debug('Wrote: %s', dst_lon) + return dst_lat, dst_lon + + +def getHeights(f:str): + ds = xr.open_dataset(f, group='science/grids/imagingGeometry') + hgts = ds.heightsMeta.data.tolist() + return hgts + + +def makeLOSFile(f:str, filename:str): + """ Create line-of-sight file from ARIA azimuth and incidence layers """ + + group = 'science/grids/imagingGeometry' + azFile = os.path.join(f'NETCDF:"{f}":{group}/azimuthAngle') + incFile = os.path.join(f'NETCDF:"{f}":{group}/incidenceAngle') + + az, az_prof = rio_open(azFile, returnProj=True) + az = np.stack(az) + az[az == 0] = np.nan + array_shp = az.shape[1:] + + heading = 90 - az + heading[np.isnan(heading)] = 0. + + inc = rio_open(incFile) + inc = np.stack(inc) + + hgt = np.arange(inc.shape[0]) + y = np.arange(inc.shape[1]) + x = np.arange(inc.shape[2]) + + da_inc = xr.DataArray(inc, name='incidenceAngle', + coords={'hgt': hgt, 'x': x, 'y': y}, + dims='hgt y x'.split()) + + da_head = xr.DataArray(heading, name='heading', + coords={'hgt': hgt, 'x': x, 'y': y}, + dims='hgt y x'.split()) + + ds = xr.merge([da_head, da_inc]).assign_attrs( + crs=str(az_prof['crs']), geo_transform=az_prof['transform']) + + + dst = f'{filename}.nc' + ds.to_netcdf(dst) + logger.debug('Wrote: %s', dst) + return dst + + +## make not get correct S1A vs S1B +def getOrbitFile(f:str, orbit_dir='./orbits'): + from eof.download import download_eofs + os.makedirs(orbit_dir, exist_ok=True) + # import s1reader + group ='science/radarMetaData/inputSLC' + sats = [] + for key in 'reference secondary'.split(): + ds = xr.open_dataset(f, group=f'{group}/{key}') + slc = ds['L1InputGranules'].item() + sats.append(slc.split('_')[0]) + # orbit_path = s1reader.get_orbit_file_from_dir(slc, orbit_dir, auto_download=True) + + dates = parse_dates_GUNW(f) + time = parse_time_GUNW(f) + dts = [datetime.strptime(f'{dt}T{time}', '%Y%m%dT%H:%M:%S') for dt in dates] + paths = download_eofs(dts, sats, save_dir=orbit_dir) + return paths + + +## only this one opens the product; need to get lats/lons actually +def get_bbox_GUNW(f:str, buff:float=1e-5): + """ Get the bounding box (SNWE) from an ARIA GUNW product """ + import shapely.wkt + ds = xr.open_dataset(f) + poly_str = ds['productBoundingBox'].data[0].decode('utf-8') + poly = shapely.wkt.loads(poly_str) + W, S, E, N = poly.bounds + + ### buffer slightly? + W, S, E, N = W-buff, S-buff, E+buff, N+buff + return [S, N, W, E] + + +def parse_dates_GUNW(f:str): + """ Get the ref/sec date from the filename """ + sec, ref = f.split('-')[6].split('_') + + return int(sec), int(ref) + + +def parse_time_GUNW(f:str): + """ Get the center time of the secondary date from the filename """ + tt = f.split('-')[7] + return f'{tt[:2]}:{tt[2:4]}:{tt[4:]}' + + +def parse_look_dir(f:str): + look_dir = f.split('-')[3].lower() + return 'right' if look_dir == 'r' else 'left' + + +def update_yaml(dct_cfg, dst='GUNW.yaml'): + """ Write a temporary yaml file with the new 'value' for 'key', preserving parms in example_yaml""" + + template_file = os.path.join( + os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml') + + with open(template_file, 'r') as f: + try: + params = yaml.safe_load(f) + except yaml.YAMLError as exc: + print(exc) + raise ValueError(f'Something is wrong with the yaml file {example_yaml}') + + params = {**params, **dct_cfg} + + with open(dst, 'w') as fh: + yaml.safe_dump(params, fh, default_flow_style=False) + + logger.info (f'Wrote new cfg file: %s', dst) + return dst + + +def main(args): + ''' + A command-line utility to convert ARIA standard product outputs from ARIA-tools to + RAiDER-compatible format + ''' + + for f in args.files: + # version = xr.open_dataset(f).attrs['version'] # not used yet + # SNWE = get_bbox_GUNW(f) + + # wavelen = xr.open_dataset(f, group='science/radarMetaData')['wavelength'].item() + dates = parse_dates_GUNW(f) + time = parse_time_GUNW(f) + heights = getHeights(f) + lookdir = parse_look_dir(f) + + # makeLOSFile(f, args.los_file) + f_lats, f_lons = makeLatLonGrid(f) + # orbits = getOrbitFile(f) + + cfg = { + 'look_dir': lookdir, + 'weather_model': args.model, + 'aoi_group' : {'lat_file': f_lats, 'lon_file': f_lons}, + 'aoi_group': {'bounding_box': '37.129123314154995 37.9307480710763 -118.44814585278701 -115.494195892019'}, + 'date_group': {'date_list': str(dates)}, + 'time_group': {'time': time}, + 'los_group' : {'ray_trace': False}, + # 'los_convention': args.los_convention, + # 'los_cube': args.los_file}, + # 'orbit_file': orbits}, + 'height_group': {'height_levels': str(heights)}, + 'runtime_group': {'raster_format': 'nc'} + } + path_cfg = f'GUNW_{dates[0]}-{dates[1]}.yaml' + update_yaml(cfg, path_cfg) + return path_cfg diff --git a/tools/RAiDER/cli/__init__.py b/tools/RAiDER/cli/__init__.py index e69de29bb..a2a9a533b 100644 --- a/tools/RAiDER/cli/__init__.py +++ b/tools/RAiDER/cli/__init__.py @@ -0,0 +1,41 @@ +import os +from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M +class AttributeDict(dict): + __getattr__ = dict.__getitem__ + __setattr__ = dict.__setitem__ + __delattr__ = dict.__delitem__ + +DEFAULT_DICT = dict( + look_dir='right', + date_start=None, + date_end=None, + date_step=None, + date_list=None, + time=None, + end_time=None, + weather_model=None, + lat_file=None, + lon_file=None, + station_file=None, + bounding_box=None, + geocoded_file=None, + dem=None, + use_dem_latlon=False, + height_levels=None, + height_file_rdr=None, + ray_trace=False, + zref=_ZREF, + cube_spacing_in_m=_CUBE_SPACING_IN_M, # TODO - Where are these parsed? + los_file=None, + los_convention='isce', + los_cube=None, + orbit_file=None, + verbose=True, + raster_format='GTiff', + output_directory=os.getcwd(), + weather_model_directory=os.path.join( + os.getcwd(), + 'weather_files' + ), + output_projection='EPSG:4236', + ) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py new file mode 100644 index 000000000..b2c8b7968 --- /dev/null +++ b/tools/RAiDER/cli/__main__.py @@ -0,0 +1,438 @@ +import os +import argparse +from importlib.metadata import entry_points +import shutil +import glob +import yaml +from textwrap import dedent +from RAiDER.cli import AttributeDict, DEFAULT_DICT +from RAiDER.cli.parser import add_cpus, add_out, add_verbose +from RAiDER.cli.validators import DateListAction, date_type +from RAiDER.logger import logger + +## make it print the help correctly (using -h, --help, for correct process or for no args) + + +## --------------------------------------------------------------------delay.py +def read_template_file(fname): + """ + Read the template file into a dictionary structure. + Parameters: fname - str, full path to the template file + delimiter - str, string to separate the key and value + skip_chars - list of str, skip certain charaters in values + Returns: template - dict, file content + Examples: template = read_template('raider.yaml') + + Modified from MintPy's 'read_template' + """ + from RAiDER.cli.validators import (enforce_time, enforce_bbox, parse_dates, + get_query_region, get_heights, get_los, enforce_wm) + with open(fname, 'r') as f: + try: + params = yaml.safe_load(f) + except yaml.YAMLError as exc: + print(exc) + raise ValueError('Something is wrong with the yaml file {}'.format(fname)) + + # Drop any values not specified + params = drop_nans(params) + + # Need to ensure that all the groups exist, even if they are not specified by the user + group_keys = ['date_group', 'time_group', 'aoi_group', 'height_group', 'los_group', 'runtime_group'] + for key in group_keys: + if not key in params.keys(): + params[key] = {} + + # Parse the user-provided arguments + template = DEFAULT_DICT + for key, value in params.items(): + if key == 'runtime_group': + for k, v in value.items(): + if v is not None: + template[k] = v + if key == 'weather_model': + template[key]= enforce_wm(value) + if key == 'time_group': + template.update(enforce_time(AttributeDict(value))) + if key == 'date_group': + template['date_list'] = parse_dates(AttributeDict(value)) + if key == 'aoi_group': + ## in case a DEM is passed and should be used + dct_temp = {**AttributeDict(value), + **AttributeDict(params['height_group'])} + template['aoi'] = get_query_region(AttributeDict(dct_temp)) + + if key == 'los_group': + template['los'] = get_los(AttributeDict(value)) + if key == 'look_dir': + if value.lower() not in ['right', 'left']: + raise ValueError(f"Unknown look direction {value}") + template['look_dir'] = value.lower() + + # Have to guarantee that certain variables exist prior to looking at heights + for key, value in params.items(): + if key == 'height_group': + template.update( + get_heights( + AttributeDict(value), + template['output_directory'], + template['station_file'], + template['bounding_box'], + ) + ) + return AttributeDict(template) + + +def drop_nans(d): + for key in list(d.keys()): + if d[key] is None: + del d[key] + elif isinstance(d[key], dict): + for k in list(d[key].keys()): + if d[key][k] is None: + del d[key][k] + return d + + +def calcDelays(iargs=None): + """ Parse command line arguments using argparse. """ + import RAiDER + from RAiDER.delay import tropo_delay + from RAiDER.checkArgs import checkArgs + from RAiDER.processWM import prepareWeatherModel + examples = 'Examples of use:' \ + '\n\t raider.py customTemplatefile.cfg' \ + '\n\t raider.py -g' + + p = argparse.ArgumentParser( + description = + 'Command line interface for RAiDER processing with a configure file.' + 'Default options can be found by running: raider.py --generate_config', + epilog=examples, formatter_class=argparse.RawDescriptionHelpFormatter) + + + p.add_argument( + 'customTemplateFile', nargs='?', + help='custom template with option settings.\n' + + "ignored if the default smallbaselineApp.cfg is input." + ) + + p.add_argument( + '-g', '--generate_template', action='store_true', + help='generate default template (if it does not exist) and exit.' + ) + + p.add_argument( + '--download_only', action='store_true', + help='only download a weather model.' + ) + + ## if not None, will replace first argument (customTemplateFile) + args = p.parse_args(args=iargs) + + # default input file + template_file = os.path.join(os.path.dirname(RAiDER.__file__), + 'cli', 'raider.yaml') + + if args.generate_template: + dst = os.path.join(os.getcwd(), 'raider.yaml') + shutil.copyfile(template_file, dst) + logger.info('Wrote %s', dst) + + + # check: existence of input template files + if (not args.customTemplateFile + and not os.path.isfile(os.path.basename(template_file)) + and not args.generate_template): + msg = "No template file found! It requires that either:" + msg += "\n a custom template file, OR the default template " + msg += "\n file 'raider.yaml' exists in current directory." + + p.print_usage() + print(examples) + raise SystemExit(f'ERROR: {msg}') + + if args.customTemplateFile: + # check the existence + if not os.path.isfile(args.customTemplateFile): + raise FileNotFoundError(args.customTemplateFile) + + args.customTemplateFile = os.path.abspath(args.customTemplateFile) + + # Read the template file + params = read_template_file(args.customTemplateFile) + + # Argument checking + params = checkArgs(params) + + params['download_only'] = args.download_only # backward compatability? + + if not params.verbose: + logger.setLevel(logging.INFO) + + delay_dct = {} + for t, w, f in zip( + params['date_list'], + params['wetFilenames'], + params['hydroFilenames'] + ): + + los = params['los'] + aoi = params['aoi'] + model = params['weather_model'] + + if los.ray_trace(): + ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing + else: + ll_bounds = aoi.bounds() + + ########################################################### + # weather model calculation + logger.debug('Starting to run the weather model calculation') + logger.debug('Time: {}'.format(t.strftime('%Y%m%d'))) + logger.debug('Beginning weather model pre-processing') + try: + weather_model_file = prepareWeatherModel( + model, t, + ll_bounds=ll_bounds, # SNWE + wmLoc=params['weather_model_directory'], + zref=params['zref'], + makePlots=params['verbose'], + ) + except RuntimeError: + logger.exception("Date %s failed", t) + continue + + # Now process the delays + try: + wet_delay, hydro_delay = tropo_delay( + t, weather_model_file, aoi, los, + params['height_levels'], + params['output_projection'], + params['look_dir'], + params['cube_spacing_in_m'] + ) + except RuntimeError: + logger.exception("Date %s failed", t) + continue + + ########################################################### + # Write the delays to file + # Different options depending on the inputs + + if los.is_Projected(): + out_filename = w.replace("_ztd", "_std") + f = f.replace("_ztd", "_std") + elif los.ray_trace(): + out_filename = w.replace("_std", "_ray") + f = f.replace("_std", "_ray") + else: + out_filename = w + + if hydro_delay is None: + # means that a dataset was returned + ds = wet_delay + ext = os.path.splitext(out_filename)[1] + if ext not in ['.nc', '.h5']: + out_filename = f'{os.path.splitext(out_filename)[0]}.nc' + + out_filename = out_filename.replace("wet", "tropo") + + if out_filename.endswith(".nc"): + ds.to_netcdf(out_filename, mode="w") + elif out_filename.endswith(".h5"): + ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) + + logger.info('Wrote delays to: %s', out_filename) + + else: + if aoi.type() == 'station_file': + w = f'{os.path.splitext(out_filename)[0]}.csv' + + if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: + writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) + + # delay_dct[t] = wet_delay, hydro_delay + delay_dct[t] = out_filename, f + + return delay_dct + + +## ------------------------------------------------------ downloadGNSSDelays.py +def downloadGNSS(): + """Parse command line arguments using argparse.""" + from RAiDER.gnss.downloadGNSSDelays import main as dlGNSS + p = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" \ + Check for and download tropospheric zenith delays for a set of GNSS stations from UNR + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and all available times of day, and confined to specified + geographic bounding box : + downloadGNSSdelay.py --out products -y 20100101 20141231 -b '39 40 -79 -78' + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally : + downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' + + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time in 12 day steps (in YYMMDD YYMMDD days) and specified time of day, and distributed globally : + downloadGNSSdelay.py --out products -y 20100101 20141231 12 --returntime '00:00:00' + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally but restricted + to list of stations specified in input textfile : + downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' -f station_list.txt + + NOTE, following example call to physically download zenith delay information not recommended as it is not + necessary for most applications. + Example call to physically download and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and confined to specified + geographic bounding box : + downloadGNSSdelay.py --download --out products -y 20100101 20141231 --returntime '00:00:00' -b '39 40 -79 -78' + """) + + # Stations to check/download + area = p.add_argument_group( + 'Stations to check/download. Can be a lat/lon bounding box or file, or will run the whole world if not specified') + area.add_argument( + '--station_file', '-f', default=None, dest='station_file', + help=('Text file containing a list of 4-char station IDs separated by newlines')) + area.add_argument( + '-b', '--bounding_box', dest='bounding_box', type=str, default=None, + help="Provide either valid shapefile or Lat/Lon Bounding SNWE. -- Example : '19 20 -99.5 -98.5'") + area.add_argument( + '--gpsrepo', '-gr', default='UNR', dest='gps_repo', + help=('Specify GPS repository you wish to query. Currently supported archives: UNR.')) + + misc = p.add_argument_group("Run parameters") + add_out(misc) + + misc.add_argument( + '--date', dest='dateList', + help=dedent("""\ + Date to calculate delay. + Can be a single date, a list of two dates (earlier, later) with 1-day interval, or a list of two dates and interval in days (earlier, later, interval). + Example accepted formats: + YYYYMMDD or + YYYYMMDD YYYYMMDD + YYYYMMDD YYYYMMDD N + """), + nargs="+", + action=DateListAction, + type=date_type, + required=True + ) + + misc.add_argument( + '--returntime', dest='returnTime', + help="Return delays closest to this specified time. If not specified, the GPS delays for all times will be returned. Input in 'HH:MM:SS', e.g. '16:00:00'", + default=None) + + misc.add_argument( + '--download', + help='Physically download data. Note this option is not necessary to proceed with statistical analyses, as data can be handled virtually in the program.', + action='store_true', dest='download', default=False) + + add_cpus(misc) + add_verbose(misc) + + args = p.parse_args() + + dlGNSS(args) + return + + +## ------------------------------------------------------------ prepFromARIA.py +def calcDelaysARIA(iargs=None): + from RAiDER.aria.prepFromARIA import main as ARIA_prep + from RAiDER.aria.calcARIA import main as ARIA_calc + + p = argparse.ArgumentParser( + description='Prepare files from ARIA-tools output to use with RAiDER') + + p.add_argument( + 'files', type=str, + help='ARIA GUNW netcdf files (accepts single file and wildcard matching)\n' + ) + + p.add_argument( + '-m', '--model', default='GMAO', type=str, + help='Weather model (default=HRRR)') + + + p.add_argument( + '-w', '--write', default=True, + help=('Optionally write the delays into the GUNW products')) + + ## always doing ray + # p.add_argument( + # '-s', '--slant', choices=('projection', 'ray'), + # type=str, default='projection', + # help='Delay calculation projecting the zenith to slant or along rays ') + + # p.add_argument( + # '-o', '--orbit_directory', default='./orbits', type=str, + # help='Path where orbits will be downloaded') + + # p.add_argument( + # '--los_convention', '-lc', choices=('isce', 'hyp3'), default='isce', + # type=str, help='LOS convention') + + # p.add_argument( + # '--los_file', default='los.geo', type=str, + # help='Output line-of-sight filename') + + # Line of sight + # p.add_argument( + # '--incFile', type=str, + # help='GDAL-readable raster image file of inclination (optional)') + # + # p.add_argument( + # '--azFile', type=str, + # help='GDAL-readable raster image file of azimuth (optional)') + + # p.add_argument( + # '--format', '-t', default='.tif', type=str, dest='fmt', + # help='Output file format (default=tif)') + + args = p.parse_args(args=iargs) + args.argv = iargs if iargs else os.sys.argv[1:] + args.files = glob.glob(args.files) + + ## prep the files needed for delay calcs + path_cfg = ARIA_prep(args) + ## do the delay calcs + # call(['raider.py', path_cfg]) + + tropoDelayFile = calcDelays([path_cfg]) + + + ## calculate the interferometric phase and write it out + # ARIA_calc(tropoDelayFile, args.write) + + return + + +def main(): + parser = argparse.ArgumentParser(prefix_chars='+', + formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser.add_argument( + '++process', choices=['calcDelays', 'downloadGNSS', 'calcDelaysARIA'], + default='calcDelays', + help='Select the HyP3 entrypoint to use' + ) + args, unknowns = parser.parse_known_args() + os.sys.argv = [args.process, *unknowns] + + process_entry_point = entry_points(group='console_scripts', + name=f'{args.process}.py')[0] + + process_entry_point.load()() + # ret_code = os.sys.exit(process_entry_point.load()()) # only if fn returns nothing + + +if __name__ == '__main__': + main() diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index d3f0c90fc..d639568eb 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -213,11 +213,11 @@ def read_template_file(fname): template['bounding_box'], ) ) - + template.update(params['aoi_group']) template = AttributeDict(template) template['aoi'] = get_query_region(template) - + return template @@ -243,7 +243,7 @@ def main(iargs=None): # Argument checking params = checkArgs(params) - + if not params.verbose: logger.setLevel(logging.INFO) diff --git a/tools/RAiDER/prepFromAria.py b/tools/RAiDER/prepFromAria.py deleted file mode 100644 index a4a3f1712..000000000 --- a/tools/RAiDER/prepFromAria.py +++ /dev/null @@ -1,107 +0,0 @@ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Author: Jeremy Maurer -# Copyright 2020, by the California Institute of Technology. ALL RIGHTS -# RESERVED. United States Government Sponsorship acknowledged. -# -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -import numpy as np -import rasterio -from RAiDER.utilFcns import rio_open, writeArrayToRaster - - -def parse_args(): - """Parse command line arguments using argparse.""" - import argparse - p = argparse.ArgumentParser( - description='Prepare files from ARIA-tools output to use with RAiDER') - - # Line of sight - p.add_argument( - '--incFile', '-i', type=str, - help='GDAL-readable raster image file of inclination', - metavar='INC', required=True) - p.add_argument( - '--azFile', '-a', type=str, - help='GDAL-readable raster image file of azimuth', - metavar='AZ', required=True) - - p.add_argument( - '--los_filename', '-f', default='los.rdr', type=str, dest='los_file', - help=('Output Line-of-sight filename')) - p.add_argument( - '--lat_filename', '-l', default='lat.rdr', type=str, dest='lat_file', - help=('Output latitude filename')) - p.add_argument( - '--lon_filename', '-L', default='lon.rdr', type=str, dest='lon_file', - help=('Output longitude filename')) - - p.add_argument( - '--format', '-t', default='ENVI', type=str, dest='fmt', - help=('Output file format')) - - return p.parse_args(), p - - -def makeLatLonGrid(inFile, lonFileName, latFileName, fmt='ENVI'): - ''' - Convert the geocoded grids to lat/lon files for input to RAiDER - ''' - ds = rasterio.open(inFile) - xSize = ds.width - ySize = ds.height - gt = ds.transform.to_gdal() - proj = ds.crs - - # Create the xy grid - xStart = gt[0] - yStart = gt[3] - xStep = gt[1] - yStep = gt[-1] - - xEnd = xStart + xStep * xSize - 0.5 * xStep - yEnd = yStart + yStep * ySize - 0.5 * yStep - - x = np.arange(xStart, xEnd, xStep) - y = np.arange(yStart, yEnd, yStep) - X, Y = np.meshgrid(x, y) - writeArrayToRaster(X, lonFileName, 0., fmt, proj, gt) - writeArrayToRaster(Y, latFileName, 0., fmt, proj, gt) - - -def makeLOSFile(incFile, azFile, fmt='ENVI', filename='los.rdr'): - ''' - Create a line-of-sight file from ARIA-derived azimuth and inclination files - ''' - az, az_prof = rio_open(azFile, returnProj=True) - az[az == 0] = np.nan - inc = rio_open(incFile) - - heading = 90 - az - heading[np.isnan(heading)] = 0. - - array_shp = np.shape(az)[:2] - - # Write the data to a file - with rasterio.open(filename, mode="w", count=2, - driver=fmt, width=array_shp[1], - height=array_shp[0], crs=az_prof.crs, - transform=az_prof.transform, - dtype=az.dtype, nodata=0.) as dst: - dst.write(inc, 1) - dst.write(heading, 2) - - return 0 - - -def prepFromAria(): - ''' - A command-line utility to convert ARIA standard product outputs from ARIA-tools to - RAiDER-compatible format - ''' - args, p = parse_args() - makeLOSFile(args.incFile, args.azFile, args.fmt, args.los_file) - makeLatLonGrid(args.incFile, args.lon_file, args.lat_file, args.fmt) - -def main(): - prepFromAria() From 6cfa567f62dfd575a3c2441974aadbe3387fd94d Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 12:06:34 -0800 Subject: [PATCH 41/82] few pep8 --- tools/RAiDER/llreader.py | 10 +++++----- tools/RAiDER/losreader.py | 15 +++++++++++---- tools/RAiDER/models/hrrr.py | 10 +++++++--- 3 files changed, 23 insertions(+), 12 deletions(-) diff --git a/tools/RAiDER/llreader.py b/tools/RAiDER/llreader.py index da7cfc8e7..4b00816be 100644 --- a/tools/RAiDER/llreader.py +++ b/tools/RAiDER/llreader.py @@ -60,7 +60,7 @@ def add_buffer(self, buffer): class StationFile(AOI): '''Use a .csv file containing at least Lat, Lon, and optionally Hgt_m columns''' def __init__(self, station_file): - AOI.__init__(self) + super().__init__() self._filename = station_file self._bounding_box = bounds_from_csv(station_file) self._type = 'station_file' @@ -87,13 +87,13 @@ def readZ(self): class RasterRDR(AOI): def __init__(self, lat_file, lon_file=None, hgt_file=None, dem_file=None, convention='isce'): - AOI.__init__(self) + super().__init__() self._type = 'radar_rasters' self._latfile = lat_file self._lonfile = lon_file if (self._latfile is None) and (self._lonfile is None): raise ValueError('You need to specify a 2-band file or two single-band files') - + try: self._proj, self._bounding_box, _ = bounds_from_latlon_rasters(lat_file, lon_file) except rasterio.errors.RasterioIOError: @@ -140,7 +140,7 @@ def __init__(self, bbox): class GeocodedFile(AOI): '''Parse a Geocoded file for coordinates''' def __init__(self, filename, is_dem=False): - AOI.__init__(self) + super().__init__() self._filename = filename self.p = rio_profile(filename) self._bounding_box = rio_extents(self.p) @@ -173,7 +173,7 @@ def readZ(self): class Geocube(AOI): '''Parse a georeferenced data cube''' def __init__(self): - AOI.__init__(self) + super().__init__() self._type = 'geocube' raise NotImplementedError diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index 52a714468..f92913b25 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -24,7 +24,6 @@ class LOS(ABC): '''LOS Class definition for handling look vectors''' - def __init__(self): self._lats, self._lons, self._heights = None, None, None self._look_vecs = None @@ -32,6 +31,7 @@ def __init__(self): self._is_zenith = False self._is_projected = False + def setPoints(self, lats, lons=None, heights=None): '''Set the pixel locations''' if (lats is None) and (self._lats is None): @@ -52,15 +52,19 @@ def setPoints(self, lats, lons=None, heights=None): self._lons = lons self._heights = heights + def setTime(self, dt): self._time = dt + def is_Zenith(self): return self._is_zenith + def is_Projected(self): return self._is_projected + def ray_trace(self): return self._ray_trace @@ -68,9 +72,10 @@ def ray_trace(self): class Zenith(LOS): """Special value indicating a look vector of "zenith".""" def __init__(self): - LOS.__init__(self) + super().__init__() self._is_zenith = True + def setLookVectors(self): '''Set point locations and calculate Zenith look vectors''' if self._lats is None: @@ -78,6 +83,7 @@ def setLookVectors(self): if self._look_vecs is None: self._look_vecs = getZenithLookVecs(self._lats, self._lons, self._heights) + def __call__(self, delays): '''Placeholder method for consistency with the other classes''' return delays @@ -88,7 +94,6 @@ class Conventional(LOS): Special value indicating that the zenith delay will be projected using the standard cos(inc) scaling. """ - def __init__(self, filename=None, los_convention='isce', time=None, pad=600): super().__init__() self._file = filename @@ -99,6 +104,7 @@ def __init__(self, filename=None, los_convention='isce', time=None, pad=600): if self._convention.lower() != 'isce': raise NotImplementedError() + def __call__(self, delays): '''Read the LOS file and convert it to look vectors''' if self._lats is None: @@ -166,7 +172,7 @@ class Raytracing(LOS): def __init__(self, filename=None, los_convention='isce', time=None, pad=600): '''read in and parse a statevector file''' - LOS.__init__(self) + super().__init__() self._ray_trace = True self._file = filename self._time = time @@ -175,6 +181,7 @@ def __init__(self, filename=None, los_convention='isce', time=None, pad=600): if self._convention.lower() != 'isce': raise NotImplementedError() + def getLookVectors(self, time, pad=3 * 60): ''' Calculate look vectors for raytracing diff --git a/tools/RAiDER/models/hrrr.py b/tools/RAiDER/models/hrrr.py index 94e09f75c..1ef692f0a 100644 --- a/tools/RAiDER/models/hrrr.py +++ b/tools/RAiDER/models/hrrr.py @@ -22,7 +22,7 @@ class HRRR(WeatherModel): def __init__(self): # initialize a weather model - WeatherModel.__init__(self) + super().__init__() self._humidityType = 'q' self._model_level_type = 'pl' # Default, pressure levels are 'pl' @@ -66,9 +66,12 @@ def __init__(self): x0 = 0 y0 = 0 earth_radius = 6371229 - p1 = CRS('+proj=lcc +lat_1={lat1} +lat_2={lat2} +lat_0={lat0} +lon_0={lon0} +x_0={x0} +y_0={y0} +a={a} +b={a} +units=m +no_defs'.format(lat1=lat1, lat2=lat2, lat0=lat0, lon0=lon0, x0=x0, y0=y0, a=earth_radius)) + p1 = CRS(f'+proj=lcc +lat_1={lat1} +lat_2={lat2} +lat_0={lat0} '\ + f'+lon_0={lon0} +x_0={x0} +y_0={y0} +a={earth_radius} '\ + '+b={earth_radius} +units=m +no_defs') self._proj = p1 + def _fetch(self, out): ''' Fetch weather model data from HRRR @@ -139,7 +142,7 @@ def _makeDataCubes(self, filename, out=None): # Get profile information from gdal prof = rio_profile(str(filename)) - + # Now get bounds S, N, W, E = self._ll_bounds @@ -254,6 +257,7 @@ def _makeDataCubes(self, filename, out=None): ds_new.to_netcdf(out, engine='netcdf4') + def _download_hrrr_file(self, DATE, out, model='hrrr', product='prs', fxx=0, verbose=False): ''' Download a HRRR model From b3159da125024b7cbd5ff88eb3e238859487fc33 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 12:46:57 -0800 Subject: [PATCH 42/82] add import --- tools/RAiDER/cli/__main__.py | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index b2c8b7968..4c822a303 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -100,6 +100,7 @@ def calcDelays(iargs=None): from RAiDER.delay import tropo_delay from RAiDER.checkArgs import checkArgs from RAiDER.processWM import prepareWeatherModel + from RAiDER.utilFcns import writeDelays examples = 'Examples of use:' \ '\n\t raider.py customTemplatefile.cfg' \ '\n\t raider.py -g' @@ -110,7 +111,6 @@ def calcDelays(iargs=None): 'Default options can be found by running: raider.py --generate_config', epilog=examples, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument( 'customTemplateFile', nargs='?', help='custom template with option settings.\n' + @@ -356,16 +356,27 @@ def calcDelaysARIA(iargs=None): p.add_argument( 'files', type=str, help='ARIA GUNW netcdf files (accepts single file and wildcard matching)\n' - ) + ) p.add_argument( '-m', '--model', default='GMAO', type=str, - help='Weather model (default=HRRR)') + help='Weather model (default=HRRR)' + ) + p.add_argument( + '-d', '--dem', default='', type=str, + help='Custom DEM. Default: Copernicus GLO30 (downloaded on fly)' + ) + + p.add_argument( + '-o', '--output_directory', default=os.getcwd(), type=str, + help='Directory to store results. Default: ./' + ) p.add_argument( '-w', '--write', default=True, - help=('Optionally write the delays into the GUNW products')) + help='Optionally write the delays into the GUNW products' + ) ## always doing ray # p.add_argument( From c8cab5bdc5ce36c4c83a3adcfee4263b810bd580 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 12:52:43 -0800 Subject: [PATCH 43/82] correct the hooks/call for downloadGNSS --- tools/RAiDER/gnss/downloadGNSSDelays.py | 98 +------------------------ 1 file changed, 3 insertions(+), 95 deletions(-) diff --git a/tools/RAiDER/gnss/downloadGNSSDelays.py b/tools/RAiDER/gnss/downloadGNSSDelays.py index 971dc778b..04f1ed283 100755 --- a/tools/RAiDER/gnss/downloadGNSSDelays.py +++ b/tools/RAiDER/gnss/downloadGNSSDelays.py @@ -5,15 +5,11 @@ # RESERVED. United States Government Sponsorship acknowledged. # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -import argparse import itertools import multiprocessing import os import pandas as pd -from textwrap import dedent -from RAiDER.cli.parser import add_cpus, add_out, add_verbose -from RAiDER.cli.validators import DateListAction, date_type from RAiDER.logger import logger, logging from RAiDER.getStationDelays import get_station_data from RAiDER.utilFcns import requests_retry_session @@ -22,92 +18,6 @@ _UNR_URL = "http://geodesy.unr.edu/" -def create_parser(): - """Parse command line arguments using argparse.""" - p = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=""" -Check for and download tropospheric zenith delays for a set of GNSS stations from UNR - -Example call to virtually access and append zenith delay information to a CSV table in specified output -directory, across specified range of time (in YYMMDD YYMMDD) and all available times of day, and confined to specified -geographic bounding box : -downloadGNSSdelay.py --out products -y 20100101 20141231 -b '39 40 -79 -78' - -Example call to virtually access and append zenith delay information to a CSV table in specified output -directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally : -downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' - - -Example call to virtually access and append zenith delay information to a CSV table in specified output -directory, across specified range of time in 12 day steps (in YYMMDD YYMMDD days) and specified time of day, and distributed globally : -downloadGNSSdelay.py --out products -y 20100101 20141231 12 --returntime '00:00:00' - -Example call to virtually access and append zenith delay information to a CSV table in specified output -directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally but restricted -to list of stations specified in input textfile : -downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' -f station_list.txt - -NOTE, following example call to physically download zenith delay information not recommended as it is not -necessary for most applications. -Example call to physically download and append zenith delay information to a CSV table in specified output -directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and confined to specified -geographic bounding box : -downloadGNSSdelay.py --download --out products -y 20100101 20141231 --returntime '00:00:00' -b '39 40 -79 -78' -""") - - # Stations to check/download - area = p.add_argument_group( - 'Stations to check/download. Can be a lat/lon bounding box or file, or will run the whole world if not specified') - area.add_argument( - '--station_file', '-f', default=None, dest='station_file', - help=('Text file containing a list of 4-char station IDs separated by newlines')) - area.add_argument( - '-b', '--bounding_box', dest='bounding_box', type=str, default=None, - help="Provide either valid shapefile or Lat/Lon Bounding SNWE. -- Example : '19 20 -99.5 -98.5'") - area.add_argument( - '--gpsrepo', '-gr', default='UNR', dest='gps_repo', - help=('Specify GPS repository you wish to query. Currently supported archives: UNR.')) - - misc = p.add_argument_group("Run parameters") - add_out(misc) - - misc.add_argument( - '--date', dest='dateList', - help=dedent("""\ - Date to calculate delay. - Can be a single date, a list of two dates (earlier, later) with 1-day interval, or a list of two dates and interval in days (earlier, later, interval). - Example accepted formats: - YYYYMMDD or - YYYYMMDD YYYYMMDD - YYYYMMDD YYYYMMDD N - """), - nargs="+", - action=DateListAction, - type=date_type, - required=True - ) - - misc.add_argument( - '--returntime', dest='returnTime', - help="Return delays closest to this specified time. If not specified, the GPS delays for all times will be returned. Input in 'HH:MM:SS', e.g. '16:00:00'", - default=None) - - misc.add_argument( - '--download', - help='Physically download data. Note this option is not necessary to proceed with statistical analyses, as data can be handled virtually in the program.', - action='store_true', dest='download', default=False) - - add_cpus(misc) - add_verbose(misc) - - return p - - -def cmd_line_parse(iargs=None): - parser = create_parser() - return parser.parse_args(args=iargs) - def get_station_list(bbox=None, writeLoc=None, userstatList=None, name_appendix=''): ''' @@ -286,16 +196,14 @@ def get_ID(line): return stat_id, float(lat), float(lon), float(height) -def main(params=None): +def main(inps=None): """ Main workflow for querying supported GPS repositories for zenith delay information. """ - if params is not None: - inps = params + try: dateList = inps.date_list returnTime = inps.time - else: - inps = cmd_line_parse() + except: dateList = inps.dateList returnTime = inps.returnTime From 6838cc4add8d47b241481902055ce132964d4aca Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 13:03:33 -0800 Subject: [PATCH 44/82] working prepFromARIA (minus orbit download) --- tools/RAiDER/aria/prepFromARIA.py | 50 ++++++++++++++++++++----------- tools/RAiDER/cli/__main__.py | 2 +- 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/tools/RAiDER/aria/prepFromARIA.py b/tools/RAiDER/aria/prepFromARIA.py index e27bc8864..d48655ec7 100644 --- a/tools/RAiDER/aria/prepFromARIA.py +++ b/tools/RAiDER/aria/prepFromARIA.py @@ -17,7 +17,7 @@ """ Should be refactored into a class that takes filename as input """ ## ---------------------------------------------------- Prepare Input from GUNW -def makeLatLonGrid(f:str): +def makeLatLonGrid(f:str, out_dir:str): # def makeLatLonGrid(f:str, reg, out_dir): ds0 = xr.open_dataset(f, group='science/grids/data') @@ -28,8 +28,8 @@ def makeLatLonGrid(f:str): # dst_lat = op.join(out_dir, f'lat_{reg}.geo') # dst_lon = op.join(out_dir, f'lon_{reg}.geo') - dst_lat = f'lat.geo' - dst_lon = f'lon.geo' + dst_lat = os.path.join(out_dir, 'lat.geo') + dst_lon = os.path.join(out_dir, 'lon.geo') da_lat.to_netcdf(dst_lat) da_lon.to_netcdf(dst_lon) logger.debug('Wrote: %s', dst_lat) @@ -84,15 +84,22 @@ def makeLOSFile(f:str, filename:str): ## make not get correct S1A vs S1B -def getOrbitFile(f:str, orbit_dir='./orbits'): +def getOrbitFile(f:str, out_dir): from eof.download import download_eofs + orbit_dir = os.path.join(out_dir, 'orbits') os.makedirs(orbit_dir, exist_ok=True) # import s1reader group ='science/radarMetaData/inputSLC' sats = [] for key in 'reference secondary'.split(): - ds = xr.open_dataset(f, group=f'{group}/{key}') - slc = ds['L1InputGranules'].item() + ds = xr.open_dataset(f, group=f'{group}/{key}') + slcs = ds['L1InputGranules'] + ## sometimes there are more than one, but the second is blank + ## sometimes there are more than one, but the second is same except version + ## error out if its not blank; + # assert np.count_nonzero(slcs.data) == 1, 'Cannot handle more than 1 SLC/GUNW' + # slc = list(filter(None, slcs))[0].item() + slc = slcs.data[0] # temporary hack; not going to use this anyway sats.append(slc.split('_')[0]) # orbit_path = s1reader.get_orbit_file_from_dir(slc, orbit_dir, auto_download=True) @@ -167,30 +174,39 @@ def main(args): # version = xr.open_dataset(f).attrs['version'] # not used yet # SNWE = get_bbox_GUNW(f) + work_dir = args.output_directory # wavelen = xr.open_dataset(f, group='science/radarMetaData')['wavelength'].item() - dates = parse_dates_GUNW(f) + dates0 = parse_dates_GUNW(f) time = parse_time_GUNW(f) heights = getHeights(f) lookdir = parse_look_dir(f) # makeLOSFile(f, args.los_file) - f_lats, f_lons = makeLatLonGrid(f) - # orbits = getOrbitFile(f) + f_lats, f_lons = makeLatLonGrid(f, work_dir) + orbits = getOrbitFile(f, work_dir) + # bbox_la = '37.129123314154995 37.9307480710763 -118.44814585278701 -115.494195892019' + + # temp + dates = dates0[1] + orbits = orbits[0] cfg = { 'look_dir': lookdir, 'weather_model': args.model, 'aoi_group' : {'lat_file': f_lats, 'lon_file': f_lons}, - 'aoi_group': {'bounding_box': '37.129123314154995 37.9307480710763 -118.44814585278701 -115.494195892019'}, 'date_group': {'date_list': str(dates)}, + 'height_group': {'dem': args.dem}, 'time_group': {'time': time}, - 'los_group' : {'ray_trace': False}, - # 'los_convention': args.los_convention, - # 'los_cube': args.los_file}, - # 'orbit_file': orbits}, - 'height_group': {'height_levels': str(heights)}, - 'runtime_group': {'raster_format': 'nc'} + 'los_group' : {'ray_trace': True, + 'orbit_file': orbits, + 'los_convention': 'isce', + }, + 'runtime_group': {'raster_format': 'nc', + 'output_directory': work_dir, + # 'los_convention': args.los_convention, + } } - path_cfg = f'GUNW_{dates[0]}-{dates[1]}.yaml' + + path_cfg = f'GUNW_{dates0[0]}-{dates0[1]}.yaml' update_yaml(cfg, path_cfg) return path_cfg diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 4c822a303..70e507a6e 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -359,7 +359,7 @@ def calcDelaysARIA(iargs=None): ) p.add_argument( - '-m', '--model', default='GMAO', type=str, + '-m', '--model', default='ERA5', type=str, help='Weather model (default=HRRR)' ) From 57d61151fbd253a0c672696d2a2fdf060d3c22db Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 13:44:52 -0800 Subject: [PATCH 45/82] fix typo in proj str --- tools/RAiDER/models/hrrr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/RAiDER/models/hrrr.py b/tools/RAiDER/models/hrrr.py index 1ef692f0a..88599bf93 100644 --- a/tools/RAiDER/models/hrrr.py +++ b/tools/RAiDER/models/hrrr.py @@ -68,7 +68,7 @@ def __init__(self): earth_radius = 6371229 p1 = CRS(f'+proj=lcc +lat_1={lat1} +lat_2={lat2} +lat_0={lat0} '\ f'+lon_0={lon0} +x_0={x0} +y_0={y0} +a={earth_radius} '\ - '+b={earth_radius} +units=m +no_defs') + f'+b={earth_radius} +units=m +no_defs') self._proj = p1 @@ -257,7 +257,7 @@ def _makeDataCubes(self, filename, out=None): ds_new.to_netcdf(out, engine='netcdf4') - + def _download_hrrr_file(self, DATE, out, model='hrrr', product='prs', fxx=0, verbose=False): ''' Download a HRRR model From daa8c0b6813a9e76ba60dccae8b2f2a169797e45 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 13:54:11 -0800 Subject: [PATCH 46/82] Update changelog --- CHANGELOG.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 541e480d6..fef639914 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) -and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [0.3.1] +RAiDER package was refactored to use a __main__ file to allow calls to different functionality. +The default is `calcDelays` which maintains the original functionality of calling `raider.py`. +`raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ...` +`raider.py ++calcDelaysARIA GUNW` is enabled but not fully functional ## [0.3.0] RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` From 69a8cc0bfd2ccd9738a618a995823be3458c6262 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 14:12:32 -0800 Subject: [PATCH 47/82] write rasters to GTiff if fmt is nc Writing to netcdf not supported without additional steps by rasterio --- tools/RAiDER/utilFcns.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/RAiDER/utilFcns.py b/tools/RAiDER/utilFcns.py index 6c4742aa2..8989a476b 100755 --- a/tools/RAiDER/utilFcns.py +++ b/tools/RAiDER/utilFcns.py @@ -262,10 +262,12 @@ def writeArrayToRaster(array, filename, noDataValue=0., fmt='ENVI', proj=None, g if gt is not None: trans = rasterio.Affine.from_gdal(*gt) + ## cant write netcdfs with rasterio in a simple way + driver = fmt if not fmt == 'nc' else 'GTiff' with rasterio.open(filename, mode="w", count=1, width=array_shp[1], height=array_shp[0], dtype=dtype, crs=proj, nodata=noDataValue, - driver=fmt, transform=trans) as dst: + driver=driver, transform=trans) as dst: dst.write(array, 1) logger.info('Wrote: %s', filename) return From 94abd4bf3b0fede910c1e505360c30f09dc80c9a Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 16:45:14 -0800 Subject: [PATCH 48/82] exit if -g option; catch if no template but default --- tools/RAiDER/cli/__main__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 70e507a6e..d8b277338 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -138,6 +138,7 @@ def calcDelays(iargs=None): dst = os.path.join(os.getcwd(), 'raider.yaml') shutil.copyfile(template_file, dst) logger.info('Wrote %s', dst) + os.sys.exit() # check: existence of input template files @@ -158,6 +159,8 @@ def calcDelays(iargs=None): raise FileNotFoundError(args.customTemplateFile) args.customTemplateFile = os.path.abspath(args.customTemplateFile) + else: + args.customTemplateFile = template_file # Read the template file params = read_template_file(args.customTemplateFile) From 4b7e51f49876c17e30397ab0a2d38c6e8a9d45de Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 17:15:24 -0800 Subject: [PATCH 49/82] fix bugs in naming and gnss sta --- tools/RAiDER/cli/__main__.py | 2 +- tools/RAiDER/delay.py | 10 +++++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index d8b277338..0ef2d47a2 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -250,7 +250,7 @@ def calcDelays(iargs=None): else: if aoi.type() == 'station_file': - w = f'{os.path.splitext(out_filename)[0]}.csv' + out_filename = f'{os.path.splitext(out_filename)[0]}.csv' if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 27c81b260..a6e768eb2 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -74,7 +74,11 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 pnt_proj = CRS.from_epsg(4326) lats, lons = aoi.readLL() hgts = aoi.readZ() - pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj).transpose(1,2,0) + pnts = transformPoints(lats, lons, hgts, pnt_proj, out_proj) + if pnts.ndim == 3: + pnts = pnts.transpose(1,2,0) + elif pnts.ndim == 2: + pnts = pnts.T ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) @@ -104,7 +108,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 wm_proj = CRS.from_epsg(4326) else: wm_proj = CRS.from_wkt(wm_proj.to_wkt()) - + # Determine the output grid extent here wesn = ll_bounds[2:] + ll_bounds[:2] out_snwe = transform_bbox( @@ -119,7 +123,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 cube_spacing_m = np.nanmean([np.nanmean(np.diff(xpts)), np.nanmean(np.diff(ypts))]) if wm_proj.axis_info[0].unit_name == "degree": cube_spacing_m = cube_spacing_m * 1.0e5 # Scale by 100km - + if crs.axis_info[0].unit_name == "degree": out_spacing = cube_spacing_m / 1.0e5 # Scale by 100km else: From 987d13e83132e35dd3e3e4d2cf08d849dff7fb2c Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 17:21:46 -0800 Subject: [PATCH 50/82] update change log; write message for delays writing --- CHANGELOG.md | 1 + tools/RAiDER/cli/__main__.py | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fef639914..ee1d61c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ RAiDER package was refactored to use a __main__ file to allow calls to different The default is `calcDelays` which maintains the original functionality of calling `raider.py`. `raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ...` `raider.py ++calcDelaysARIA GUNW` is enabled but not fully functional +Fix bugs to ensure working RAiDER_tutorial notebook ## [0.3.0] RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 0ef2d47a2..3ca224b4f 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -246,8 +246,6 @@ def calcDelays(iargs=None): elif out_filename.endswith(".h5"): ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) - logger.info('Wrote delays to: %s', out_filename) - else: if aoi.type() == 'station_file': out_filename = f'{os.path.splitext(out_filename)[0]}.csv' @@ -255,6 +253,10 @@ def calcDelays(iargs=None): if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) + logger.info('Wrote hydro delays to: %s', f) + + logger.info('Wrote wet delays to: %s', out_filename) + # delay_dct[t] = wet_delay, hydro_delay delay_dct[t] = out_filename, f From 879fd48c0141f7556f22705bbdc1ec85bfe672fa Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 17:33:03 -0800 Subject: [PATCH 51/82] get rid of old raider cli --- tools/RAiDER/cli/raider.py | 330 ------------------------------------- 1 file changed, 330 deletions(-) delete mode 100644 tools/RAiDER/cli/raider.py diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py deleted file mode 100644 index d639568eb..000000000 --- a/tools/RAiDER/cli/raider.py +++ /dev/null @@ -1,330 +0,0 @@ -import argparse -import os -import shutil -import sys -import yaml -import re, glob - -import RAiDER -from RAiDER.checkArgs import checkArgs -from RAiDER.cli.validators import ( - enforce_time, parse_dates, get_query_region, get_heights, get_los, enforce_wm -) -from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M -from RAiDER.delay import tropo_delay -from RAiDER.logger import logger, logging -from RAiDER.processWM import prepareWeatherModel -from RAiDER.utilFcns import writeDelays - - -HELP_MESSAGE = """ -Command line options for RAiDER processing. Default options can be found by running -raider.py --generate_config - -Download a weather model and calculate tropospheric delays -""" - -SHORT_MESSAGE = """ -Program to calculate troposphere total delays using a weather model -""" - -EXAMPLES = """ -Usage examples: -raider.py -g -raider.py customTemplatefile.cfg -""" - -class AttributeDict(dict): - __getattr__ = dict.__getitem__ - __setattr__ = dict.__setitem__ - __delattr__ = dict.__delitem__ - -DEFAULT_DICT = dict( - look_dir='right', - date_start=None, - date_end=None, - date_step=None, - date_list=None, - time=None, - end_time=None, - weather_model=None, - lat_file=None, - lon_file=None, - station_file=None, - bounding_box=None, - geocoded_file=None, - dem=None, - use_dem_latlon=False, - height_levels=None, - height_file_rdr=None, - ray_trace=False, - zref=_ZREF, - cube_spacing_in_m=_CUBE_SPACING_IN_M, # TODO - Where are these parsed? - los_file=None, - los_convention='isce', - los_cube=None, - orbit_file=None, - verbose=True, - raster_format='GTiff', - output_directory=os.getcwd(), - weather_model_directory=os.path.join( - os.getcwd(), - 'weather_files' - ), - output_projection='EPSG:4236', - ) - - -def create_parser(): - """Parse command line arguments using argparse.""" - p = argparse.ArgumentParser( - formatter_class = argparse.RawDescriptionHelpFormatter, - description = HELP_MESSAGE, - epilog = EXAMPLES, - ) - - p.add_argument( - 'customTemplateFile', nargs='?', - help='custom template with option settings.\n' + - "ignored if the default smallbaselineApp.cfg is input." - ) - - p.add_argument( - '-g', '--generate_template', - dest='generate_template', - action='store_true', - help='generate default template (if it does not exist) and exit.' - ) - - - p.add_argument( - '--download-only', - action='store_true', - help='only download a weather model.' - ) - - return p - - -def parseCMD(iargs=None): - """ - Parse command-line arguments and pass to delay.py - """ - - p = create_parser() - args = p.parse_args(args=iargs) - - args.argv = iargs if iargs else sys.argv[1:] - - # default input file - template_file = os.path.join( - os.path.dirname( - RAiDER.__file__ - ), - 'cli', 'raider.yaml' - ) - if '-g' in args.argv: - dst = os.path.join(os.getcwd(), 'raider.yaml') - shutil.copyfile( - template_file, - dst, - ) - - logger.info('Wrote %s', dst) - sys.exit(0) - - # check: existence of input template files - if (not args.customTemplateFile - and not os.path.isfile(os.path.basename(template_file)) - and not args.generate_template): - p.print_usage() - print(EXAMPLES) - - msg = "No template file found! It requires that either:" - msg += "\n a custom template file, OR the default template " - msg += "\n file 'raider.yaml' exists in current directory." - raise SystemExit(f'ERROR: {msg}') - - if args.customTemplateFile: - # check the existence - if not os.path.isfile(args.customTemplateFile): - raise FileNotFoundError(args.customTemplateFile) - - args.customTemplateFile = os.path.abspath(args.customTemplateFile) - - return args - - -def read_template_file(fname): - """ - Read the template file into a dictionary structure. - Parameters: fname - str, full path to the template file - delimiter - str, string to separate the key and value - skip_chars - list of str, skip certain charaters in values - Returns: template - dict, file content - Examples: template = read_template('raider.yaml') - - Modified from MintPy's 'read_template' - """ - with open(fname, 'r') as f: - try: - params = yaml.safe_load(f) - except yaml.YAMLError as exc: - print(exc) - raise ValueError('Something is wrong with the yaml file {}'.format(fname)) - - # Drop any values not specified - params = drop_nans(params) - - # Need to ensure that all the groups exist, even if they are not specified by the user - group_keys = ['date_group', 'time_group', 'aoi_group', 'height_group', 'los_group', 'runtime_group'] - for key in group_keys: - if not key in params.keys(): - params[key] = {} - - # Parse the user-provided arguments - template = DEFAULT_DICT - for key, value in params.items(): - if key == 'runtime_group': - for k, v in value.items(): - if v is not None: - template[k] = v - if key == 'weather_model': - template[key]= enforce_wm(value) - if key == 'time_group': - template.update(enforce_time(AttributeDict(value))) - if key == 'date_group': - template['date_list'] = parse_dates(AttributeDict(value)) - if key == 'los_group': - template['los'] = get_los(AttributeDict(value)) - if key == 'look_dir': - if value.lower() not in ['right', 'left']: - raise ValueError(f"Unknown look direction {value}") - template['look_dir'] = value.lower() - - # Have to guarantee that certain variables exist prior to looking at heights - for key, value in params.items(): - if key == 'height_group': - template.update( - get_heights( - AttributeDict(value), - template['output_directory'], - template['station_file'], - template['bounding_box'], - ) - ) - - template.update(params['aoi_group']) - template = AttributeDict(template) - template['aoi'] = get_query_region(template) - - return template - - -def drop_nans(d): - for key in list(d.keys()): - if d[key] is None: - del d[key] - elif isinstance(d[key], dict): - for k in list(d[key].keys()): - if d[key][k] is None: - del d[key][k] - return d - - - -########################################################################## -def main(iargs=None): - # parse - inps = parseCMD(iargs) - - # Read the template file - params = read_template_file(inps.customTemplateFile) - - # Argument checking - params = checkArgs(params) - - if not params.verbose: - logger.setLevel(logging.INFO) - - - for t, w, f in zip( - params['date_list'], - params['wetFilenames'], - params['hydroFilenames'] - ): - - los = params['los'] - aoi = params['aoi'] - model = params['weather_model'] - - if los.ray_trace(): - ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing - else: - ll_bounds = aoi.bounds() - - ########################################################### - # weather model calculation - logger.debug('Starting to run the weather model calculation') - logger.debug('Time: {}'.format(t.strftime('%Y%m%d'))) - logger.debug('Beginning weather model pre-processing') - try: - weather_model_file = prepareWeatherModel( - model, t, - ll_bounds=ll_bounds, # SNWE - wmLoc=params['weather_model_directory'], - zref=params['zref'], - makePlots=params['verbose'], - ) - except RuntimeError: - logger.exception("Date %s failed", t) - continue - - # Now process the delays - try: - wet_delay, hydro_delay = tropo_delay( - t, weather_model_file, aoi, los, - params['height_levels'], - params['output_projection'], - params['look_dir'], - params['cube_spacing_in_m'] - ) - except RuntimeError: - logger.exception("Date %s failed", t) - continue - - ########################################################### - # Write the delays to file - # Different options depending on the inputs - - if los.is_Projected(): - out_filename = w.replace("_ztd", "_std") - f = f.replace("_ztd", "_std") - elif los.ray_trace(): - out_filename = w.replace("_std", "_ray") - f = f.replace("_std", "_ray") - else: - out_filename = w - - if hydro_delay is None: - # means that a dataset was returned - ds = wet_delay - ext = os.path.splitext(out_filename)[1] - if ext not in ['.nc', '.h5']: - out_filename = f'{os.path.splitext(out_filename)[0]}.nc' - - out_filename = out_filename.replace("wet", "tropo") - - if out_filename.endswith(".nc"): - ds.to_netcdf(out_filename, mode="w") - elif out_filename.endswith(".h5"): - ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) - - logger.info('Wrote delays to: %s', out_filename) - - else: - if aoi.type() == 'station_file': - w = f'{os.path.splitext(out_filename)[0]}.csv' - - if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: - writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) From 1bdb02c1a90fda66293e40306644a82de96dbfe3 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 18:02:28 -0800 Subject: [PATCH 52/82] update tests to reflect cli change --- test/cli/test_raiderDelay.py | 2 +- test/test_llreader.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/cli/test_raiderDelay.py b/test/cli/test_raiderDelay.py index 446ee0b91..803e0fa0b 100644 --- a/test/cli/test_raiderDelay.py +++ b/test/cli/test_raiderDelay.py @@ -1,7 +1,7 @@ from argparse import ArgumentParser, ArgumentTypeError from datetime import datetime, time -from RAiDER.cli.raider import drop_nans +from RAiDER.cli.__main__ import drop_nans def test_drop_nans(): test_d = { diff --git a/test/test_llreader.py b/test/test_llreader.py index 31d7062bb..51179b686 100644 --- a/test/test_llreader.py +++ b/test/test_llreader.py @@ -6,7 +6,7 @@ from test import GEOM_DIR, TEST_DIR -from RAiDER.cli.raider import create_parser +from RAiDER.cli.__main__ import calcDelays from RAiDER.utilFcns import rio_open from RAiDER.llreader import ( @@ -20,7 +20,7 @@ @pytest.fixture def parser(): - return create_parser() + return calcDelays() @pytest.fixture From 1876f1db23c3c8225617afb6ef2bb6fbfde13667 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Mon, 12 Dec 2022 18:06:08 -0800 Subject: [PATCH 53/82] get rid of breakpoint --- tools/RAiDER/cli/__main__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 3ca224b4f..118b831b7 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -359,8 +359,8 @@ def calcDelaysARIA(iargs=None): description='Prepare files from ARIA-tools output to use with RAiDER') p.add_argument( - 'files', type=str, - help='ARIA GUNW netcdf files (accepts single file and wildcard matching)\n' + 'file', type=str, + help='1 ARIA GUNW netcdf file' ) p.add_argument( @@ -416,7 +416,8 @@ def calcDelaysARIA(iargs=None): args = p.parse_args(args=iargs) args.argv = iargs if iargs else os.sys.argv[1:] - args.files = glob.glob(args.files) + # args.files = glob.glob(args.files) + args.file = args.file ## prep the files needed for delay calcs path_cfg = ARIA_prep(args) From 5d127c5977428cfa216c0cbc164055c4603889d4 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 10:51:16 -0800 Subject: [PATCH 54/82] Change aria->gunw filenames, placeholder fns --- tools/RAiDER/aria/calcARIA.py | 29 ---- tools/RAiDER/aria/calcGUNW.py | 8 ++ tools/RAiDER/aria/prepFromARIA.py | 212 ------------------------------ tools/RAiDER/aria/prepFromGUNW.py | 11 ++ 4 files changed, 19 insertions(+), 241 deletions(-) delete mode 100644 tools/RAiDER/aria/calcARIA.py create mode 100644 tools/RAiDER/aria/calcGUNW.py delete mode 100644 tools/RAiDER/aria/prepFromARIA.py create mode 100644 tools/RAiDER/aria/prepFromGUNW.py diff --git a/tools/RAiDER/aria/calcARIA.py b/tools/RAiDER/aria/calcARIA.py deleted file mode 100644 index a101f13b2..000000000 --- a/tools/RAiDER/aria/calcARIA.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -Calculate the interferometric phase from the 4 delays files of a GUNW -Write it to disk -""" -import xarray as xr -import numpy as np - -## z is a placeholder -def get_delay(delayFile): - ds = xr.open_dataset(delayFile) - wet = ds['wet'].isel(z=0).data - hydro = ds['hydro'].isel(z=0).data - tot = wet + hydro - return tot - - -def main(delayFiles, update_nc): - """ Pull the wet and hydrostatic delays from the netcdf - - Calculate the interferometric phase delay - Write to disk, and optionally update the netcdf - """ - delays = np.stack([get_delay(delayFile) for delayFile in delayFiles]) - breakpoint() - - - -if __name__ == '__main__': - main() diff --git a/tools/RAiDER/aria/calcGUNW.py b/tools/RAiDER/aria/calcGUNW.py new file mode 100644 index 000000000..79ea59ca1 --- /dev/null +++ b/tools/RAiDER/aria/calcGUNW.py @@ -0,0 +1,8 @@ +def main(args): + """ Calculate interferometric phase delay at dates in ARIA standard product (GUNW) """ + pass + + + +if __name__ == '__main__': + main() diff --git a/tools/RAiDER/aria/prepFromARIA.py b/tools/RAiDER/aria/prepFromARIA.py deleted file mode 100644 index d48655ec7..000000000 --- a/tools/RAiDER/aria/prepFromARIA.py +++ /dev/null @@ -1,212 +0,0 @@ -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# -# Author: Jeremy Maurer, Brett Buzzanga -# Copyright 2022, by the California Institute of Technology. ALL RIGHTS -# RESERVED. United States Government Sponsorship acknowledged. -# -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -import os -from datetime import datetime -import numpy as np -import xarray as xr -import rasterio -import yaml -import RAiDER -from RAiDER.utilFcns import rio_open, writeArrayToRaster -from RAiDER.logger import logger - -""" Should be refactored into a class that takes filename as input """ -## ---------------------------------------------------- Prepare Input from GUNW -def makeLatLonGrid(f:str, out_dir:str): -# def makeLatLonGrid(f:str, reg, out_dir): - ds0 = xr.open_dataset(f, group='science/grids/data') - - Lat, Lon = np.meshgrid(ds0.latitude.data, ds0.longitude.data) - - da_lat = xr.DataArray(Lat.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split()) - da_lon = xr.DataArray(Lon.T, coords=[Lon[0, :], Lat[:, 0]], dims='lon lat'.split()) - # dst_lat = op.join(out_dir, f'lat_{reg}.geo') - # dst_lon = op.join(out_dir, f'lon_{reg}.geo') - - dst_lat = os.path.join(out_dir, 'lat.geo') - dst_lon = os.path.join(out_dir, 'lon.geo') - da_lat.to_netcdf(dst_lat) - da_lon.to_netcdf(dst_lon) - logger.debug('Wrote: %s', dst_lat) - logger.debug('Wrote: %s', dst_lon) - return dst_lat, dst_lon - - -def getHeights(f:str): - ds = xr.open_dataset(f, group='science/grids/imagingGeometry') - hgts = ds.heightsMeta.data.tolist() - return hgts - - -def makeLOSFile(f:str, filename:str): - """ Create line-of-sight file from ARIA azimuth and incidence layers """ - - group = 'science/grids/imagingGeometry' - azFile = os.path.join(f'NETCDF:"{f}":{group}/azimuthAngle') - incFile = os.path.join(f'NETCDF:"{f}":{group}/incidenceAngle') - - az, az_prof = rio_open(azFile, returnProj=True) - az = np.stack(az) - az[az == 0] = np.nan - array_shp = az.shape[1:] - - heading = 90 - az - heading[np.isnan(heading)] = 0. - - inc = rio_open(incFile) - inc = np.stack(inc) - - hgt = np.arange(inc.shape[0]) - y = np.arange(inc.shape[1]) - x = np.arange(inc.shape[2]) - - da_inc = xr.DataArray(inc, name='incidenceAngle', - coords={'hgt': hgt, 'x': x, 'y': y}, - dims='hgt y x'.split()) - - da_head = xr.DataArray(heading, name='heading', - coords={'hgt': hgt, 'x': x, 'y': y}, - dims='hgt y x'.split()) - - ds = xr.merge([da_head, da_inc]).assign_attrs( - crs=str(az_prof['crs']), geo_transform=az_prof['transform']) - - - dst = f'{filename}.nc' - ds.to_netcdf(dst) - logger.debug('Wrote: %s', dst) - return dst - - -## make not get correct S1A vs S1B -def getOrbitFile(f:str, out_dir): - from eof.download import download_eofs - orbit_dir = os.path.join(out_dir, 'orbits') - os.makedirs(orbit_dir, exist_ok=True) - # import s1reader - group ='science/radarMetaData/inputSLC' - sats = [] - for key in 'reference secondary'.split(): - ds = xr.open_dataset(f, group=f'{group}/{key}') - slcs = ds['L1InputGranules'] - ## sometimes there are more than one, but the second is blank - ## sometimes there are more than one, but the second is same except version - ## error out if its not blank; - # assert np.count_nonzero(slcs.data) == 1, 'Cannot handle more than 1 SLC/GUNW' - # slc = list(filter(None, slcs))[0].item() - slc = slcs.data[0] # temporary hack; not going to use this anyway - sats.append(slc.split('_')[0]) - # orbit_path = s1reader.get_orbit_file_from_dir(slc, orbit_dir, auto_download=True) - - dates = parse_dates_GUNW(f) - time = parse_time_GUNW(f) - dts = [datetime.strptime(f'{dt}T{time}', '%Y%m%dT%H:%M:%S') for dt in dates] - paths = download_eofs(dts, sats, save_dir=orbit_dir) - return paths - - -## only this one opens the product; need to get lats/lons actually -def get_bbox_GUNW(f:str, buff:float=1e-5): - """ Get the bounding box (SNWE) from an ARIA GUNW product """ - import shapely.wkt - ds = xr.open_dataset(f) - poly_str = ds['productBoundingBox'].data[0].decode('utf-8') - poly = shapely.wkt.loads(poly_str) - W, S, E, N = poly.bounds - - ### buffer slightly? - W, S, E, N = W-buff, S-buff, E+buff, N+buff - return [S, N, W, E] - - -def parse_dates_GUNW(f:str): - """ Get the ref/sec date from the filename """ - sec, ref = f.split('-')[6].split('_') - - return int(sec), int(ref) - - -def parse_time_GUNW(f:str): - """ Get the center time of the secondary date from the filename """ - tt = f.split('-')[7] - return f'{tt[:2]}:{tt[2:4]}:{tt[4:]}' - - -def parse_look_dir(f:str): - look_dir = f.split('-')[3].lower() - return 'right' if look_dir == 'r' else 'left' - - -def update_yaml(dct_cfg, dst='GUNW.yaml'): - """ Write a temporary yaml file with the new 'value' for 'key', preserving parms in example_yaml""" - - template_file = os.path.join( - os.path.dirname(RAiDER.__file__), 'cli', 'raider.yaml') - - with open(template_file, 'r') as f: - try: - params = yaml.safe_load(f) - except yaml.YAMLError as exc: - print(exc) - raise ValueError(f'Something is wrong with the yaml file {example_yaml}') - - params = {**params, **dct_cfg} - - with open(dst, 'w') as fh: - yaml.safe_dump(params, fh, default_flow_style=False) - - logger.info (f'Wrote new cfg file: %s', dst) - return dst - - -def main(args): - ''' - A command-line utility to convert ARIA standard product outputs from ARIA-tools to - RAiDER-compatible format - ''' - - for f in args.files: - # version = xr.open_dataset(f).attrs['version'] # not used yet - # SNWE = get_bbox_GUNW(f) - - work_dir = args.output_directory - # wavelen = xr.open_dataset(f, group='science/radarMetaData')['wavelength'].item() - dates0 = parse_dates_GUNW(f) - time = parse_time_GUNW(f) - heights = getHeights(f) - lookdir = parse_look_dir(f) - - # makeLOSFile(f, args.los_file) - f_lats, f_lons = makeLatLonGrid(f, work_dir) - orbits = getOrbitFile(f, work_dir) - # bbox_la = '37.129123314154995 37.9307480710763 -118.44814585278701 -115.494195892019' - - # temp - dates = dates0[1] - orbits = orbits[0] - - cfg = { - 'look_dir': lookdir, - 'weather_model': args.model, - 'aoi_group' : {'lat_file': f_lats, 'lon_file': f_lons}, - 'date_group': {'date_list': str(dates)}, - 'height_group': {'dem': args.dem}, - 'time_group': {'time': time}, - 'los_group' : {'ray_trace': True, - 'orbit_file': orbits, - 'los_convention': 'isce', - }, - 'runtime_group': {'raster_format': 'nc', - 'output_directory': work_dir, - # 'los_convention': args.los_convention, - } - } - - path_cfg = f'GUNW_{dates0[0]}-{dates0[1]}.yaml' - update_yaml(cfg, path_cfg) - return path_cfg diff --git a/tools/RAiDER/aria/prepFromGUNW.py b/tools/RAiDER/aria/prepFromGUNW.py new file mode 100644 index 000000000..6076d5b02 --- /dev/null +++ b/tools/RAiDER/aria/prepFromGUNW.py @@ -0,0 +1,11 @@ +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Author: Jeremy Maurer, Brett Buzzanga +# Copyright 2022, by the California Institute of Technology. ALL RIGHTS +# RESERVED. United States Government Sponsorship acknowledged. +# +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +def main(args): + """ Get needed RAiDER parameters from an ARIA standard product (GUNW) """ + pass From f50dbe0434fd69a5c51a349d53ea54e25712d201 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 10:52:20 -0800 Subject: [PATCH 55/82] update aria->gunw functions --- CHANGELOG.md | 3 +- pyproject.toml | 14 +++---- tools/RAiDER/cli/__main__.py | 73 ++++++++++-------------------------- 3 files changed, 27 insertions(+), 63 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee1d61c0f..10045dbd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,12 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) +and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [0.3.1] RAiDER package was refactored to use a __main__ file to allow calls to different functionality. The default is `calcDelays` which maintains the original functionality of calling `raider.py`. `raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ...` -`raider.py ++calcDelaysARIA GUNW` is enabled but not fully functional +`raider.py ++calcDelaysGUNW GUNWFILE` is enabled as a placeholder only. Fix bugs to ensure working RAiDER_tutorial notebook ## [0.3.0] diff --git a/pyproject.toml b/pyproject.toml index 74ea80033..19b81705d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,7 @@ authors = [ {name="David Bekaert", email="david.bekaert@jpl.nasa.gov"}, {name="Jeremy Maurer", email="maurer.jeremy@gmail.com"}, {name="Piyush Agram", email="piyush@descarteslabs.com"}, - {name="Simran Sangha", email="simran.s.sangha@jpl.nasa.gov"}, + {name="Simran Sangha", email="simran.s.sangha@jpl.nasa.gov"}, {name="Brett Buzzanga", email="buzzanga@jpl.nasa.gov"}, ] description = "Raytracing Atmospheric Delay Estimation for RADAR" @@ -43,13 +43,11 @@ repository = "https://github.com/dbekaert/RAiDER" [project.scripts] "raider.py" = "RAiDER.cli.__main__:main" "calcDelays.py" = "RAiDER.cli.__main__:calcDelays" -"calcDelaysARIA.py" = "RAiDER.cli.__main__:calcDelaysARIA" -"raiderDownloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" -"downloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" - -"prepFromARIA.py" = "RAiDER.cli.__main__:prepFromARIA" -"raiderStats.py" = "RAiDER.cli.statsPlot:main" -"generateGACOSVRT.py" = "RAiDER.models.generateGACOSVRT:main" +"calcDelaysGUNW.py" = "RAiDER.cli.__main__:calcDelaysGUNW" +"raiderDownloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" +"downloadGNSS.py" = "RAiDER.cli.__main__:downloadGNSS" +"raiderStats.py" = "RAiDER.cli.statsPlot:main" +"generateGACOSVRT.py" = "RAiDER.models.generateGACOSVRT:main" [tool.setuptools] zip-safe = false diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 118b831b7..928dac0c1 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -350,13 +350,13 @@ def downloadGNSS(): return -## ------------------------------------------------------------ prepFromARIA.py -def calcDelaysARIA(iargs=None): - from RAiDER.aria.prepFromARIA import main as ARIA_prep - from RAiDER.aria.calcARIA import main as ARIA_calc +## ------------------------------------------------------------ prepFromGUNW.py +def calcDelaysGUNW(iargs=None): + from RAiDER.aria.prepFromGUNW import main as GUNW_prep + from RAiDER.aria.calcGUNW import main as GUNW_calc p = argparse.ArgumentParser( - description='Prepare files from ARIA-tools output to use with RAiDER') + description='Calculate a cube of interferometic delays for GUNW files') p.add_argument( 'file', type=str, @@ -364,71 +364,36 @@ def calcDelaysARIA(iargs=None): ) p.add_argument( - '-m', '--model', default='ERA5', type=str, - help='Weather model (default=HRRR)' + '-m', '--model', default='HRRR', type=str, + help='Weather model (Default=HRRR).' ) - p.add_argument( - '-d', '--dem', default='', type=str, - help='Custom DEM. Default: Copernicus GLO30 (downloaded on fly)' - ) p.add_argument( '-o', '--output_directory', default=os.getcwd(), type=str, - help='Directory to store results. Default: ./' + help='Directory to store results (Default=./).' ) p.add_argument( '-w', '--write', default=True, - help='Optionally write the delays into the GUNW products' + help='Optionally write the delays into the given GUNW product (Default=True).' ) - ## always doing ray - # p.add_argument( - # '-s', '--slant', choices=('projection', 'ray'), - # type=str, default='projection', - # help='Delay calculation projecting the zenith to slant or along rays ') - - # p.add_argument( - # '-o', '--orbit_directory', default='./orbits', type=str, - # help='Path where orbits will be downloaded') - - # p.add_argument( - # '--los_convention', '-lc', choices=('isce', 'hyp3'), default='isce', - # type=str, help='LOS convention') - - # p.add_argument( - # '--los_file', default='los.geo', type=str, - # help='Output line-of-sight filename') - - # Line of sight - # p.add_argument( - # '--incFile', type=str, - # help='GDAL-readable raster image file of inclination (optional)') - # - # p.add_argument( - # '--azFile', type=str, - # help='GDAL-readable raster image file of azimuth (optional)') - - # p.add_argument( - # '--format', '-t', default='.tif', type=str, dest='fmt', - # help='Output file format (default=tif)') args = p.parse_args(args=iargs) args.argv = iargs if iargs else os.sys.argv[1:] - # args.files = glob.glob(args.files) - args.file = args.file - - ## prep the files needed for delay calcs - path_cfg = ARIA_prep(args) - ## do the delay calcs - # call(['raider.py', path_cfg]) + # args.files = glob.glob(args.files) # eventually support multiple files - tropoDelayFile = calcDelays([path_cfg]) + ## below are placeholders and not yet implemented + ## prep the config needed for delay calcs + # path_cfg, wavelength = GUNW_prep(args) + ## write the delays to disk using config and return dictionary of: + # date: wet/hydro filename + # dct_delays = calcDelays([path_cfg]) ## calculate the interferometric phase and write it out - # ARIA_calc(tropoDelayFile, args.write) + # GUNW_calc(tropoDelayFile, args.file, wavelength, args.output_directory, args.write) return @@ -437,9 +402,9 @@ def main(): parser = argparse.ArgumentParser(prefix_chars='+', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( - '++process', choices=['calcDelays', 'downloadGNSS', 'calcDelaysARIA'], + '++process', choices=['calcDelays', 'downloadGNSS', 'calcDelaysGUNW'], default='calcDelays', - help='Select the HyP3 entrypoint to use' + help='Select the entrypoint to use' ) args, unknowns = parser.parse_known_args() os.sys.argv = [args.process, *unknowns] From 5c45bfa7591b40baacb13f1949e8db3e4d2aca4f Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 13:22:41 -0800 Subject: [PATCH 56/82] pin isce to 0.9 --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 7c02e4354..4c5e6048e 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - h5py - herbie-data - h5netcdf - - isce3==0.8.0=*_2 + - isce3==0.9.0 - lxml - matplotlib - netcdf4 From 48db47ad2496f62cbc96d24d664dac1409c61388 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 13:39:55 -0800 Subject: [PATCH 57/82] pin ISCE version to >=0.9 --- CHANGELOG.md | 1 + environment.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10045dbd7..5a94a3680 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The default is `calcDelays` which maintains the original functionality of callin `raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ...` `raider.py ++calcDelaysGUNW GUNWFILE` is enabled as a placeholder only. Fix bugs to ensure working RAiDER_tutorial notebook +Upgraded ISCE3 to `>=v0.9.0` to fix a conda build issue as described in [#425](https://github.com/dbekaert/RAiDER/issues/425) ## [0.3.0] RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` diff --git a/environment.yml b/environment.yml index 4c5e6048e..b426a5973 100644 --- a/environment.yml +++ b/environment.yml @@ -22,7 +22,7 @@ dependencies: - h5py - herbie-data - h5netcdf - - isce3==0.9.0 + - isce3>=0.9.0 - lxml - matplotlib - netcdf4 From c5d0801a144e0728e088c5fba4d01d427c0183e0 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 14:48:19 -0800 Subject: [PATCH 58/82] reenable download_only --- tools/RAiDER/cli/__init__.py | 1 + tools/RAiDER/cli/__main__.py | 6 +++--- tools/RAiDER/processWM.py | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/RAiDER/cli/__init__.py b/tools/RAiDER/cli/__init__.py index a2a9a533b..ee49f6ac8 100644 --- a/tools/RAiDER/cli/__init__.py +++ b/tools/RAiDER/cli/__init__.py @@ -32,6 +32,7 @@ class AttributeDict(dict): orbit_file=None, verbose=True, raster_format='GTiff', + download_only=False, output_directory=os.getcwd(), weather_model_directory=os.path.join( os.getcwd(), diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 928dac0c1..407c84111 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -166,9 +166,8 @@ def calcDelays(iargs=None): params = read_template_file(args.customTemplateFile) # Argument checking - params = checkArgs(params) - - params['download_only'] = args.download_only # backward compatability? + params = checkArgs(params) + dl_only = True if params['download_only'] or args.download_only else False if not params.verbose: logger.setLevel(logging.INFO) @@ -200,6 +199,7 @@ def calcDelays(iargs=None): ll_bounds=ll_bounds, # SNWE wmLoc=params['weather_model_directory'], zref=params['zref'], + download_only=dl_only, makePlots=params['verbose'], ) except RuntimeError: diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index ac944fa06..d0073f3e1 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -67,6 +67,7 @@ def prepareWeatherModel( logger.warning( 'download_only flag selected. No further processing will happen.' ) + os.sys.exit() return None # Otherwise, load the weather model data From 6a3a2da0531b6c6ae711897076f1b2ae16e3ec37 Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 14:48:53 -0800 Subject: [PATCH 59/82] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a94a3680..f90b42adc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ The default is `calcDelays` which maintains the original functionality of callin `raider.py ++calcDelaysGUNW GUNWFILE` is enabled as a placeholder only. Fix bugs to ensure working RAiDER_tutorial notebook Upgraded ISCE3 to `>=v0.9.0` to fix a conda build issue as described in [#425](https://github.com/dbekaert/RAiDER/issues/425) +Allow user to specify --download_only or download_only=True in the configure file ## [0.3.0] RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` From b18b1ca6010d7c6f9c9869aeb5c650cf22c8769a Mon Sep 17 00:00:00 2001 From: bbuzz31 Date: Tue, 13 Dec 2022 15:00:00 -0800 Subject: [PATCH 60/82] better handling of download_only --- tools/RAiDER/cli/__main__.py | 4 ++++ tools/RAiDER/processWM.py | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 407c84111..809951ed1 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -206,6 +206,10 @@ def calcDelays(iargs=None): logger.exception("Date %s failed", t) continue + # dont process the delays for download only + if dl_only: + continue + # Now process the delays try: wet_delay, hydro_delay = tropo_delay( diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index d0073f3e1..ac944fa06 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -67,7 +67,6 @@ def prepareWeatherModel( logger.warning( 'download_only flag selected. No further processing will happen.' ) - os.sys.exit() return None # Otherwise, load the weather model data From 26584d0f59785fb2d8b136eaab623de47f0bab3c Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 08:33:55 -0600 Subject: [PATCH 61/82] some rearrangement --- test/cli/test_raiderDelay.py | 2 +- test/test_llreader.py | 6 +- tools/RAiDER/cli/__main__.py | 415 +--------------------------- tools/RAiDER/cli/raider.py | 403 +++++++++++++++++++++++++++ tools/RAiDER/delay.py | 31 +-- tools/RAiDER/delayFcns.py | 4 +- tools/RAiDER/losreader.py | 112 ++++---- tools/RAiDER/models/weatherModel.py | 10 +- tools/RAiDER/processWM.py | 13 +- tools/RAiDER/utilFcns.py | 10 +- 10 files changed, 507 insertions(+), 499 deletions(-) create mode 100644 tools/RAiDER/cli/raider.py diff --git a/test/cli/test_raiderDelay.py b/test/cli/test_raiderDelay.py index 803e0fa0b..446ee0b91 100644 --- a/test/cli/test_raiderDelay.py +++ b/test/cli/test_raiderDelay.py @@ -1,7 +1,7 @@ from argparse import ArgumentParser, ArgumentTypeError from datetime import datetime, time -from RAiDER.cli.__main__ import drop_nans +from RAiDER.cli.raider import drop_nans def test_drop_nans(): test_d = { diff --git a/test/test_llreader.py b/test/test_llreader.py index 51179b686..71da892dd 100644 --- a/test/test_llreader.py +++ b/test/test_llreader.py @@ -6,7 +6,7 @@ from test import GEOM_DIR, TEST_DIR -from RAiDER.cli.__main__ import calcDelays +from RAiDER.cli.raider import calcDelays from RAiDER.utilFcns import rio_open from RAiDER.llreader import ( @@ -35,10 +35,10 @@ def llfiles(): def test_latlon_reader_2(): with pytest.raises(ValueError): - query = RasterRDR(lat_file=None, lon_file=None) + RasterRDR(lat_file=None, lon_file=None) with pytest.raises(ValueError): - query = RasterRDR(lat_file='doesnotexist.rdr', lon_file='doesnotexist.rdr') + RasterRDR(lat_file='doesnotexist.rdr', lon_file='doesnotexist.rdr') def test_latlon_reader(): diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index 809951ed1..a0aa7e056 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -1,410 +1,12 @@ -import os import argparse +import os from importlib.metadata import entry_points -import shutil -import glob -import yaml -from textwrap import dedent -from RAiDER.cli import AttributeDict, DEFAULT_DICT -from RAiDER.cli.parser import add_cpus, add_out, add_verbose -from RAiDER.cli.validators import DateListAction, date_type -from RAiDER.logger import logger - -## make it print the help correctly (using -h, --help, for correct process or for no args) - - -## --------------------------------------------------------------------delay.py -def read_template_file(fname): - """ - Read the template file into a dictionary structure. - Parameters: fname - str, full path to the template file - delimiter - str, string to separate the key and value - skip_chars - list of str, skip certain charaters in values - Returns: template - dict, file content - Examples: template = read_template('raider.yaml') - - Modified from MintPy's 'read_template' - """ - from RAiDER.cli.validators import (enforce_time, enforce_bbox, parse_dates, - get_query_region, get_heights, get_los, enforce_wm) - with open(fname, 'r') as f: - try: - params = yaml.safe_load(f) - except yaml.YAMLError as exc: - print(exc) - raise ValueError('Something is wrong with the yaml file {}'.format(fname)) - - # Drop any values not specified - params = drop_nans(params) - - # Need to ensure that all the groups exist, even if they are not specified by the user - group_keys = ['date_group', 'time_group', 'aoi_group', 'height_group', 'los_group', 'runtime_group'] - for key in group_keys: - if not key in params.keys(): - params[key] = {} - - # Parse the user-provided arguments - template = DEFAULT_DICT - for key, value in params.items(): - if key == 'runtime_group': - for k, v in value.items(): - if v is not None: - template[k] = v - if key == 'weather_model': - template[key]= enforce_wm(value) - if key == 'time_group': - template.update(enforce_time(AttributeDict(value))) - if key == 'date_group': - template['date_list'] = parse_dates(AttributeDict(value)) - if key == 'aoi_group': - ## in case a DEM is passed and should be used - dct_temp = {**AttributeDict(value), - **AttributeDict(params['height_group'])} - template['aoi'] = get_query_region(AttributeDict(dct_temp)) - - if key == 'los_group': - template['los'] = get_los(AttributeDict(value)) - if key == 'look_dir': - if value.lower() not in ['right', 'left']: - raise ValueError(f"Unknown look direction {value}") - template['look_dir'] = value.lower() - - # Have to guarantee that certain variables exist prior to looking at heights - for key, value in params.items(): - if key == 'height_group': - template.update( - get_heights( - AttributeDict(value), - template['output_directory'], - template['station_file'], - template['bounding_box'], - ) - ) - return AttributeDict(template) - - -def drop_nans(d): - for key in list(d.keys()): - if d[key] is None: - del d[key] - elif isinstance(d[key], dict): - for k in list(d[key].keys()): - if d[key][k] is None: - del d[key][k] - return d - - -def calcDelays(iargs=None): - """ Parse command line arguments using argparse. """ - import RAiDER - from RAiDER.delay import tropo_delay - from RAiDER.checkArgs import checkArgs - from RAiDER.processWM import prepareWeatherModel - from RAiDER.utilFcns import writeDelays - examples = 'Examples of use:' \ - '\n\t raider.py customTemplatefile.cfg' \ - '\n\t raider.py -g' - - p = argparse.ArgumentParser( - description = - 'Command line interface for RAiDER processing with a configure file.' - 'Default options can be found by running: raider.py --generate_config', - epilog=examples, formatter_class=argparse.RawDescriptionHelpFormatter) - - p.add_argument( - 'customTemplateFile', nargs='?', - help='custom template with option settings.\n' + - "ignored if the default smallbaselineApp.cfg is input." - ) - - p.add_argument( - '-g', '--generate_template', action='store_true', - help='generate default template (if it does not exist) and exit.' - ) - - p.add_argument( - '--download_only', action='store_true', - help='only download a weather model.' - ) - - ## if not None, will replace first argument (customTemplateFile) - args = p.parse_args(args=iargs) - - # default input file - template_file = os.path.join(os.path.dirname(RAiDER.__file__), - 'cli', 'raider.yaml') - - if args.generate_template: - dst = os.path.join(os.getcwd(), 'raider.yaml') - shutil.copyfile(template_file, dst) - logger.info('Wrote %s', dst) - os.sys.exit() - - - # check: existence of input template files - if (not args.customTemplateFile - and not os.path.isfile(os.path.basename(template_file)) - and not args.generate_template): - msg = "No template file found! It requires that either:" - msg += "\n a custom template file, OR the default template " - msg += "\n file 'raider.yaml' exists in current directory." - - p.print_usage() - print(examples) - raise SystemExit(f'ERROR: {msg}') - - if args.customTemplateFile: - # check the existence - if not os.path.isfile(args.customTemplateFile): - raise FileNotFoundError(args.customTemplateFile) - - args.customTemplateFile = os.path.abspath(args.customTemplateFile) - else: - args.customTemplateFile = template_file - - # Read the template file - params = read_template_file(args.customTemplateFile) - - # Argument checking - params = checkArgs(params) - dl_only = True if params['download_only'] or args.download_only else False - - if not params.verbose: - logger.setLevel(logging.INFO) - - delay_dct = {} - for t, w, f in zip( - params['date_list'], - params['wetFilenames'], - params['hydroFilenames'] - ): - - los = params['los'] - aoi = params['aoi'] - model = params['weather_model'] - - if los.ray_trace(): - ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing - else: - ll_bounds = aoi.bounds() - - ########################################################### - # weather model calculation - logger.debug('Starting to run the weather model calculation') - logger.debug('Time: {}'.format(t.strftime('%Y%m%d'))) - logger.debug('Beginning weather model pre-processing') - try: - weather_model_file = prepareWeatherModel( - model, t, - ll_bounds=ll_bounds, # SNWE - wmLoc=params['weather_model_directory'], - zref=params['zref'], - download_only=dl_only, - makePlots=params['verbose'], - ) - except RuntimeError: - logger.exception("Date %s failed", t) - continue - - # dont process the delays for download only - if dl_only: - continue - - # Now process the delays - try: - wet_delay, hydro_delay = tropo_delay( - t, weather_model_file, aoi, los, - params['height_levels'], - params['output_projection'], - params['look_dir'], - params['cube_spacing_in_m'] - ) - except RuntimeError: - logger.exception("Date %s failed", t) - continue - - ########################################################### - # Write the delays to file - # Different options depending on the inputs - - if los.is_Projected(): - out_filename = w.replace("_ztd", "_std") - f = f.replace("_ztd", "_std") - elif los.ray_trace(): - out_filename = w.replace("_std", "_ray") - f = f.replace("_std", "_ray") - else: - out_filename = w - - if hydro_delay is None: - # means that a dataset was returned - ds = wet_delay - ext = os.path.splitext(out_filename)[1] - if ext not in ['.nc', '.h5']: - out_filename = f'{os.path.splitext(out_filename)[0]}.nc' - - out_filename = out_filename.replace("wet", "tropo") - - if out_filename.endswith(".nc"): - ds.to_netcdf(out_filename, mode="w") - elif out_filename.endswith(".h5"): - ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) - - else: - if aoi.type() == 'station_file': - out_filename = f'{os.path.splitext(out_filename)[0]}.csv' - - if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: - writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) - - logger.info('Wrote hydro delays to: %s', f) - - logger.info('Wrote wet delays to: %s', out_filename) - - # delay_dct[t] = wet_delay, hydro_delay - delay_dct[t] = out_filename, f - - return delay_dct - - -## ------------------------------------------------------ downloadGNSSDelays.py -def downloadGNSS(): - """Parse command line arguments using argparse.""" - from RAiDER.gnss.downloadGNSSDelays import main as dlGNSS - p = argparse.ArgumentParser( - formatter_class=argparse.RawDescriptionHelpFormatter, - description=""" \ - Check for and download tropospheric zenith delays for a set of GNSS stations from UNR - - Example call to virtually access and append zenith delay information to a CSV table in specified output - directory, across specified range of time (in YYMMDD YYMMDD) and all available times of day, and confined to specified - geographic bounding box : - downloadGNSSdelay.py --out products -y 20100101 20141231 -b '39 40 -79 -78' - - Example call to virtually access and append zenith delay information to a CSV table in specified output - directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally : - downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' - - - Example call to virtually access and append zenith delay information to a CSV table in specified output - directory, across specified range of time in 12 day steps (in YYMMDD YYMMDD days) and specified time of day, and distributed globally : - downloadGNSSdelay.py --out products -y 20100101 20141231 12 --returntime '00:00:00' - - Example call to virtually access and append zenith delay information to a CSV table in specified output - directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally but restricted - to list of stations specified in input textfile : - downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' -f station_list.txt - - NOTE, following example call to physically download zenith delay information not recommended as it is not - necessary for most applications. - Example call to physically download and append zenith delay information to a CSV table in specified output - directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and confined to specified - geographic bounding box : - downloadGNSSdelay.py --download --out products -y 20100101 20141231 --returntime '00:00:00' -b '39 40 -79 -78' - """) - - # Stations to check/download - area = p.add_argument_group( - 'Stations to check/download. Can be a lat/lon bounding box or file, or will run the whole world if not specified') - area.add_argument( - '--station_file', '-f', default=None, dest='station_file', - help=('Text file containing a list of 4-char station IDs separated by newlines')) - area.add_argument( - '-b', '--bounding_box', dest='bounding_box', type=str, default=None, - help="Provide either valid shapefile or Lat/Lon Bounding SNWE. -- Example : '19 20 -99.5 -98.5'") - area.add_argument( - '--gpsrepo', '-gr', default='UNR', dest='gps_repo', - help=('Specify GPS repository you wish to query. Currently supported archives: UNR.')) - - misc = p.add_argument_group("Run parameters") - add_out(misc) - - misc.add_argument( - '--date', dest='dateList', - help=dedent("""\ - Date to calculate delay. - Can be a single date, a list of two dates (earlier, later) with 1-day interval, or a list of two dates and interval in days (earlier, later, interval). - Example accepted formats: - YYYYMMDD or - YYYYMMDD YYYYMMDD - YYYYMMDD YYYYMMDD N - """), - nargs="+", - action=DateListAction, - type=date_type, - required=True - ) - - misc.add_argument( - '--returntime', dest='returnTime', - help="Return delays closest to this specified time. If not specified, the GPS delays for all times will be returned. Input in 'HH:MM:SS', e.g. '16:00:00'", - default=None) - - misc.add_argument( - '--download', - help='Physically download data. Note this option is not necessary to proceed with statistical analyses, as data can be handled virtually in the program.', - action='store_true', dest='download', default=False) - - add_cpus(misc) - add_verbose(misc) - - args = p.parse_args() - - dlGNSS(args) - return - - -## ------------------------------------------------------------ prepFromGUNW.py -def calcDelaysGUNW(iargs=None): - from RAiDER.aria.prepFromGUNW import main as GUNW_prep - from RAiDER.aria.calcGUNW import main as GUNW_calc - - p = argparse.ArgumentParser( - description='Calculate a cube of interferometic delays for GUNW files') - - p.add_argument( - 'file', type=str, - help='1 ARIA GUNW netcdf file' - ) - - p.add_argument( - '-m', '--model', default='HRRR', type=str, - help='Weather model (Default=HRRR).' - ) - - - p.add_argument( - '-o', '--output_directory', default=os.getcwd(), type=str, - help='Directory to store results (Default=./).' - ) - - p.add_argument( - '-w', '--write', default=True, - help='Optionally write the delays into the given GUNW product (Default=True).' - ) - - - args = p.parse_args(args=iargs) - args.argv = iargs if iargs else os.sys.argv[1:] - # args.files = glob.glob(args.files) # eventually support multiple files - - ## below are placeholders and not yet implemented - ## prep the config needed for delay calcs - # path_cfg, wavelength = GUNW_prep(args) - - ## write the delays to disk using config and return dictionary of: - # date: wet/hydro filename - # dct_delays = calcDelays([path_cfg]) - - ## calculate the interferometric phase and write it out - # GUNW_calc(tropoDelayFile, args.file, wavelength, args.output_directory, args.write) - - return - def main(): - parser = argparse.ArgumentParser(prefix_chars='+', - formatter_class=argparse.ArgumentDefaultsHelpFormatter) + parser = argparse.ArgumentParser( + prefix_chars='+', + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) parser.add_argument( '++process', choices=['calcDelays', 'downloadGNSS', 'calcDelaysGUNW'], default='calcDelays', @@ -413,12 +15,9 @@ def main(): args, unknowns = parser.parse_known_args() os.sys.argv = [args.process, *unknowns] - process_entry_point = entry_points(group='console_scripts', - name=f'{args.process}.py')[0] - + process_entry_point = entry_points(group='console_scripts', name=f'{args.process}.py')[0] process_entry_point.load()() - # ret_code = os.sys.exit(process_entry_point.load()()) # only if fn returns nothing if __name__ == '__main__': - main() + main() \ No newline at end of file diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py new file mode 100644 index 000000000..fc141d6b5 --- /dev/null +++ b/tools/RAiDER/cli/raider.py @@ -0,0 +1,403 @@ +import os +import argparse +import shutil +import yaml +from textwrap import dedent +from RAiDER.cli import AttributeDict, DEFAULT_DICT +from RAiDER.cli.parser import add_cpus, add_out, add_verbose +from RAiDER.cli.validators import DateListAction, date_type +from RAiDER.logger import logger + +## make it print the help correctly (using -h, --help, for correct process or for no args) + + +## --------------------------------------------------------------------delay.py +def read_template_file(fname): + """ + Read the template file into a dictionary structure. + Args:: fname - str, full path to the template file + delimiter - str, string to separate the key and value + skip_chars - list of str, skip certain charaters in values + Returns:: template - dict, file content + Examples: template = read_template('raider.yaml') + + Modified from MintPy's 'read_template' + """ + from RAiDER.cli.validators import (enforce_time, enforce_bbox, parse_dates, + get_query_region, get_heights, get_los, enforce_wm) + with open(fname, 'r') as f: + try: + params = yaml.safe_load(f) + except yaml.YAMLError as exc: + print(exc) + raise ValueError('Something is wrong with the yaml file {}'.format(fname)) + + # Drop any values not specified + params = drop_nans(params) + + # Need to ensure that all the groups exist, even if they are not specified by the user + group_keys = ['date_group', 'time_group', 'aoi_group', 'height_group', 'los_group', 'runtime_group'] + for key in group_keys: + if not key in params.keys(): + params[key] = {} + + # Parse the user-provided arguments + template = DEFAULT_DICT + for key, value in params.items(): + if key == 'runtime_group': + for k, v in value.items(): + if v is not None: + template[k] = v + if key == 'weather_model': + template[key]= enforce_wm(value) + if key == 'time_group': + template.update(enforce_time(AttributeDict(value))) + if key == 'date_group': + template['date_list'] = parse_dates(AttributeDict(value)) + if key == 'aoi_group': + ## in case a DEM is passed and should be used + dct_temp = {**AttributeDict(value), + **AttributeDict(params['height_group'])} + template['aoi'] = get_query_region(AttributeDict(dct_temp)) + + if key == 'los_group': + template['los'] = get_los(AttributeDict(value)) + if key == 'look_dir': + if value.lower() not in ['right', 'left']: + raise ValueError(f"Unknown look direction {value}") + template['look_dir'] = value.lower() + + # Have to guarantee that certain variables exist prior to looking at heights + for key, value in params.items(): + if key == 'height_group': + template.update( + get_heights( + AttributeDict(value), + template['output_directory'], + template['station_file'], + template['bounding_box'], + ) + ) + return AttributeDict(template) + + +def drop_nans(d): + for key in list(d.keys()): + if d[key] is None: + del d[key] + elif isinstance(d[key], dict): + for k in list(d[key].keys()): + if d[key][k] is None: + del d[key][k] + return d + + +def calcDelays(iargs=None): + """ Parse command line arguments using argparse. """ + import RAiDER + from RAiDER.delay import tropo_delay + from RAiDER.checkArgs import checkArgs + from RAiDER.processWM import prepareWeatherModel + from RAiDER.utilFcns import writeDelays + examples = 'Examples of use:' \ + '\n\t raider.py customTemplatefile.cfg' \ + '\n\t raider.py -g' + + p = argparse.ArgumentParser( + description = + 'Command line interface for RAiDER processing with a configure file.' + 'Default options can be found by running: raider.py --generate_config', + epilog=examples, formatter_class=argparse.RawDescriptionHelpFormatter) + + p.add_argument( + 'customTemplateFile', nargs='?', + help='custom template with option settings.\n' + + "ignored if the default smallbaselineApp.cfg is input." + ) + + p.add_argument( + '-g', '--generate_template', action='store_true', + help='generate default template (if it does not exist) and exit.' + ) + + p.add_argument( + '--download_only', action='store_true', + help='only download a weather model.' + ) + + ## if not None, will replace first argument (customTemplateFile) + args = p.parse_args(args=iargs) + + # default input file + template_file = os.path.join(os.path.dirname(RAiDER.__file__), + 'cli', 'raider.yaml') + + if args.generate_template: + dst = os.path.join(os.getcwd(), 'raider.yaml') + shutil.copyfile(template_file, dst) + logger.info('Wrote %s', dst) + os.sys.exit() + + + # check: existence of input template files + if (not args.customTemplateFile + and not os.path.isfile(os.path.basename(template_file)) + and not args.generate_template): + msg = "No template file found! It requires that either:" + msg += "\n a custom template file, OR the default template " + msg += "\n file 'raider.yaml' exists in current directory." + + p.print_usage() + print(examples) + raise SystemExit(f'ERROR: {msg}') + + if args.customTemplateFile: + # check the existence + if not os.path.isfile(args.customTemplateFile): + raise FileNotFoundError(args.customTemplateFile) + + args.customTemplateFile = os.path.abspath(args.customTemplateFile) + else: + args.customTemplateFile = template_file + + # Read the template file + params = read_template_file(args.customTemplateFile) + + # Argument checking + params = checkArgs(params) + dl_only = True if params['download_only'] or args.download_only else False + + if not params.verbose: + logger.setLevel(logging.INFO) + + delay_dct = {} + for t, w, f in zip( + params['date_list'], + params['wetFilenames'], + params['hydroFilenames'] + ): + + los = params['los'] + aoi = params['aoi'] + model = params['weather_model'] + + if los.ray_trace(): + ll_bounds = aoi.add_buffer(buffer=1) # add a buffer for raytracing + else: + ll_bounds = aoi.bounds() + + ########################################################### + # weather model calculation + logger.debug('Starting to run the weather model calculation') + logger.debug('Time: {}'.format(t.strftime('%Y%m%d'))) + logger.debug('Beginning weather model pre-processing') + try: + weather_model_file = prepareWeatherModel( + model, t, + ll_bounds=ll_bounds, # SNWE + wmLoc=params['weather_model_directory'], + zref=params['zref'], + download_only=dl_only, + makePlots=params['verbose'], + ) + except RuntimeError: + logger.exception("Date %s failed", t) + continue + + # dont process the delays for download only + if dl_only: + continue + + # Now process the delays + try: + wet_delay, hydro_delay = tropo_delay( + t, weather_model_file, aoi, los, + params['height_levels'], + params['output_projection'], + params['look_dir'], + params['cube_spacing_in_m'] + ) + except RuntimeError: + logger.exception("Date %s failed", t) + continue + + ########################################################### + # Write the delays to file + # Different options depending on the inputs + + if los.is_Projected(): + out_filename = w.replace("_ztd", "_std") + f = f.replace("_ztd", "_std") + elif los.ray_trace(): + out_filename = w.replace("_std", "_ray") + f = f.replace("_std", "_ray") + else: + out_filename = w + + if hydro_delay is None: + # means that a dataset was returned + ds = wet_delay + ext = os.path.splitext(out_filename)[1] + if ext not in ['.nc', '.h5']: + out_filename = f'{os.path.splitext(out_filename)[0]}.nc' + + out_filename = out_filename.replace("wet", "tropo") + + if out_filename.endswith(".nc"): + ds.to_netcdf(out_filename, mode="w") + elif out_filename.endswith(".h5"): + ds.to_netcdf(out_filename, engine="h5netcdf", invalid_netcdf=True) + + else: + if aoi.type() == 'station_file': + out_filename = f'{os.path.splitext(out_filename)[0]}.csv' + + if aoi.type() in ['station_file', 'radar_rasters', 'geocoded_file']: + writeDelays(aoi, wet_delay, hydro_delay, out_filename, f, outformat=params['raster_format']) + + logger.info('Wrote hydro delays to: %s', f) + + logger.info('Wrote wet delays to: %s', out_filename) + + # delay_dct[t] = wet_delay, hydro_delay + delay_dct[t] = out_filename, f + + return delay_dct + + +## ------------------------------------------------------ downloadGNSSDelays.py +def downloadGNSS(): + """Parse command line arguments using argparse.""" + from RAiDER.gnss.downloadGNSSDelays import main as dlGNSS + p = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" \ + Check for and download tropospheric zenith delays for a set of GNSS stations from UNR + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and all available times of day, and confined to specified + geographic bounding box : + downloadGNSSdelay.py --out products -y 20100101 20141231 -b '39 40 -79 -78' + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally : + downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' + + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time in 12 day steps (in YYMMDD YYMMDD days) and specified time of day, and distributed globally : + downloadGNSSdelay.py --out products -y 20100101 20141231 12 --returntime '00:00:00' + + Example call to virtually access and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and distributed globally but restricted + to list of stations specified in input textfile : + downloadGNSSdelay.py --out products -y 20100101 20141231 --returntime '00:00:00' -f station_list.txt + + NOTE, following example call to physically download zenith delay information not recommended as it is not + necessary for most applications. + Example call to physically download and append zenith delay information to a CSV table in specified output + directory, across specified range of time (in YYMMDD YYMMDD) and specified time of day, and confined to specified + geographic bounding box : + downloadGNSSdelay.py --download --out products -y 20100101 20141231 --returntime '00:00:00' -b '39 40 -79 -78' + """) + + # Stations to check/download + area = p.add_argument_group( + 'Stations to check/download. Can be a lat/lon bounding box or file, or will run the whole world if not specified') + area.add_argument( + '--station_file', '-f', default=None, dest='station_file', + help=('Text file containing a list of 4-char station IDs separated by newlines')) + area.add_argument( + '-b', '--bounding_box', dest='bounding_box', type=str, default=None, + help="Provide either valid shapefile or Lat/Lon Bounding SNWE. -- Example : '19 20 -99.5 -98.5'") + area.add_argument( + '--gpsrepo', '-gr', default='UNR', dest='gps_repo', + help=('Specify GPS repository you wish to query. Currently supported archives: UNR.')) + + misc = p.add_argument_group("Run parameters") + add_out(misc) + + misc.add_argument( + '--date', dest='dateList', + help=dedent("""\ + Date to calculate delay. + Can be a single date, a list of two dates (earlier, later) with 1-day interval, or a list of two dates and interval in days (earlier, later, interval). + Example accepted formats: + YYYYMMDD or + YYYYMMDD YYYYMMDD + YYYYMMDD YYYYMMDD N + """), + nargs="+", + action=DateListAction, + type=date_type, + required=True + ) + + misc.add_argument( + '--returntime', dest='returnTime', + help="Return delays closest to this specified time. If not specified, the GPS delays for all times will be returned. Input in 'HH:MM:SS', e.g. '16:00:00'", + default=None) + + misc.add_argument( + '--download', + help='Physically download data. Note this option is not necessary to proceed with statistical analyses, as data can be handled virtually in the program.', + action='store_true', dest='download', default=False) + + add_cpus(misc) + add_verbose(misc) + + args = p.parse_args() + + dlGNSS(args) + return + + +## ------------------------------------------------------------ prepFromGUNW.py +def calcDelaysGUNW(iargs=None): + from RAiDER.aria.prepFromGUNW import main as GUNW_prep + from RAiDER.aria.calcGUNW import main as GUNW_calc + + p = argparse.ArgumentParser( + description='Calculate a cube of interferometic delays for GUNW files') + + p.add_argument( + 'file', type=str, + help='1 ARIA GUNW netcdf file' + ) + + p.add_argument( + '-m', '--model', default='HRRR', type=str, + help='Weather model (Default=HRRR).' + ) + + + p.add_argument( + '-o', '--output_directory', default=os.getcwd(), type=str, + help='Directory to store results (Default=./).' + ) + + p.add_argument( + '-w', '--write', default=True, + help='Optionally write the delays into the given GUNW product (Default=True).' + ) + + + args = p.parse_args(args=iargs) + args.argv = iargs if iargs else os.sys.argv[1:] + # args.files = glob.glob(args.files) # eventually support multiple files + + ## below are placeholders and not yet implemented + ## prep the config needed for delay calcs + # path_cfg, wavelength = GUNW_prep(args) + + ## write the delays to disk using config and return dictionary of: + # date: wet/hydro filename + # dct_delays = calcDelays([path_cfg]) + + ## calculate the interferometric phase and write it out + # GUNW_calc(tropoDelayFile, args.file, wavelength, args.output_directory, args.write) + + return + + + diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index a6e768eb2..dc9b3bcae 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -36,21 +36,18 @@ def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4 """ Calculate integrated delays on query points. - Parameterss - ---------- - dt: Datetime - Datetime object for determining when to calculate delays - weather_model_File: string - Name of the NETCDF file containing a pre-processed weather model - aoi: AOI object - AOI object - los: LOS object - LOS object - height_levels: list - (optional) list of height levels on which to calculate delays. Only needed for cube generation. - out_proj: int,str - (optional) EPSG code for output projection - look_dir: str - (optional) Satellite look direction. Only needed for slant delay calculation - cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes - - Returns - ------- - xarray Dataset or wet and hydrostatic delays at the query points. The dataset will contain fields - 'wet' and 'hydro' which are the total (integrated) wet and hydrostatic delays + Args: + dt: Datetime - Datetime object for determining when to calculate delays + weather_model_File: string - Name of the NETCDF file containing a pre-processed weather model + aoi: AOI object - AOI object + los: LOS object - LOS object + height_levels: list - (optional) list of height levels on which to calculate delays. Only needed for cube generation. + out_proj: int,str - (optional) EPSG code for output projection + look_dir: str - (optional) Satellite look direction. Only needed for slant delay calculation + cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes + + Returns: + xarray Dataset *or* ndarrays: wet and hydrostatic delays at the grid nodes / query points. """ # get heights if height_levels is None: @@ -266,7 +263,7 @@ def transformPoints(lats, lons, hgts, old_proj, new_proj): Transform lat/lon/hgt data to an array of points in a new projection - Parameters + Args: ---------- lats - WGS-84 latitude (EPSG: 4326) lons - ditto for longitude @@ -274,7 +271,7 @@ def transformPoints(lats, lons, hgts, old_proj, new_proj): old_proj - the original projection of the points new_proj - the new projection in which to return the points - Returns + Returns: ------- the array of query points in the weather model coordinate system (YX) ''' diff --git a/tools/RAiDER/delayFcns.py b/tools/RAiDER/delayFcns.py index ab5f8b7b6..60e5f1e88 100755 --- a/tools/RAiDER/delayFcns.py +++ b/tools/RAiDER/delayFcns.py @@ -20,10 +20,10 @@ def calculate_start_points(x, y, z, ds): ''' - Parameters + Args: ---------- wm_file: str - A file containing a regularized weather model. - Returns + Returns: ------- SP: ndarray - a * x 3 array containing the XYZ locations of the pixels in ECEF coordinates. Note the ordering of the array is [Y X Z] diff --git a/tools/RAiDER/losreader.py b/tools/RAiDER/losreader.py index f92913b25..c8235eae1 100644 --- a/tools/RAiDER/losreader.py +++ b/tools/RAiDER/losreader.py @@ -23,7 +23,9 @@ class LOS(ABC): - '''LOS Class definition for handling look vectors''' + ''' + LOS Class definition for handling look vectors + ''' def __init__(self): self._lats, self._lons, self._heights = None, None, None self._look_vecs = None @@ -70,7 +72,9 @@ def ray_trace(self): class Zenith(LOS): - """Special value indicating a look vector of "zenith".""" + """ + Class definition for a "Zenith" object. + """ def __init__(self): super().__init__() self._is_zenith = True @@ -106,7 +110,9 @@ def __init__(self, filename=None, los_convention='isce', time=None, pad=600): def __call__(self, delays): - '''Read the LOS file and convert it to look vectors''' + ''' + Read the LOS file and convert it to look vectors + ''' if self._lats is None: raise ValueError('Target points not set') if self._file is None: @@ -145,7 +151,7 @@ class Raytracing(LOS): because they are in an ECEF reference frame instead of a local ENU. This is done because the construction of rays is done in ECEF rather than the local ENU. - Parameters + Args: ---------- time: python datetime - user-requested query time. Must be compatible with the orbit file passed. @@ -154,14 +160,14 @@ class Raytracing(LOS): the user-specified time; default 3 hours Only required for a statevector file. - Returns + Returns: ------- - look_vecs: ndarray - an x 3 array of unit look vectors, defined in - an Earth-centered, earth-fixed reference frame (ECEF). - Convention is vectors point from the target pixel to the - sensor. - lengths: ndarray - array of of the distnce from the surface to - the top of the troposphere (denoted by zref) + ndarray - an x 3 array of unit look vectors, defined in + an Earth-centered, earth-fixed reference frame (ECEF). + Convention is vectors point from the target pixel to the + sensor. + ndarray - array of of the distnce from the surface to + the top of the troposphere (denoted by zref) Example: -------- @@ -272,13 +278,11 @@ def getZenithLookVecs(lats, lons, heights): ''' Returns look vectors when Zenith is used. - Parameters - ---------- - lats/lons/heights: ndarray - Numpy arrays containing WGS-84 target locations + Args: + lats/lons/heights (ndarray): - Numpy arrays containing WGS-84 target locations - Returns - ------- - zenLookVecs: ndarray - (in_shape) x 3 unit look vectors in an ECEF reference frame + Returns: + zenLookVecs (ndarray): - (in_shape) x 3 unit look vectors in an ECEF reference frame ''' x = np.cos(np.radians(lats)) * np.cos(np.radians(lons)) y = np.cos(np.radians(lats)) * np.sin(np.radians(lons)) @@ -291,20 +295,17 @@ def get_sv(los_file, ref_time, pad=3 * 60): """ Read an LOS file and return orbital state vectors - Parameters - ---------- - los_file: str - user-passed file containing either look - vectors or statevectors for the sensor - ref_time: python datetime - User-requested datetime; if not encompassed - by the orbit times will raise a ValueError - pad: int - number of seconds to keep around the - requested time - - Returns - ------- - svs: 7 x 1 list of Nt x 1 ndarrays - the times, x/y/z positions and - velocities of the sensor for the given - window around the reference time + Args: + los_file (str): - user-passed file containing either look + vectors or statevectors for the sensor + ref_time (datetime):- User-requested datetime; if not encompassed + by the orbit times will raise a ValueError + pad (int): - number of seconds to keep around the + requested time + + Returns: + svs (list of ndarrays): - the times, x/y/z positions and velocities + of the sensor for the given window around the reference time """ try: svs = read_txt_file(los_file) @@ -331,15 +332,13 @@ def inc_hd_to_enu(incidence, heading): Convert incidence and heading to line-of-sight vectors from the ground to the top of the troposphere. - Parameters - ---------- - incidence: ndarray - incidence angle in deg from vertical - heading: ndarray - heading angle in deg clockwise from north - lats/lons/heights: ndarray - WGS84 ellipsoidal target (ground pixel) locations + Args: + incidence: ndarray - incidence angle in deg from vertical + heading: ndarray - heading angle in deg clockwise from north + lats/lons/heights: ndarray - WGS84 ellipsoidal target (ground pixel) locations - Returns - ------- - LOS: ndarray - (input_shape) x 3 array of unit look vectors in local ENU + Returns: + LOS: ndarray - (input_shape) x 3 array of unit look vectors in local ENU Algorithm referenced from http://earthdef.caltech.edu/boards/4/topics/327 ''' @@ -389,16 +388,15 @@ def read_txt_file(filename): should be denoted as integer time in seconds since the reference epoch (user-requested time). - Parameters - ---------- - filename: str - user-supplied space-delimited text file with no header - containing orbital statevectors as 7 columns: - - time in seconds since the user-supplied epoch - - x / y / z locations in ECEF cartesian coordinates - - vx / vy / vz velocities in m/s in ECEF coordinates - Returns - svs: list - a length-7 list of numpy vectors containing the above - variables + Args: + filename (str): - user-supplied space-delimited text file with no header + containing orbital statevectors as 7 columns: + - time in seconds since the user-supplied epoch + - x / y / z locations in ECEF cartesian coordinates + - vx / vy / vz velocities in m/s in ECEF coordinates + Returns: + svs (list): - a length-7 list of numpy vectors containing the above + variables ''' t = list() x = list() @@ -436,11 +434,11 @@ def read_ESA_Orbit_file(filename): ''' Read orbit data from an orbit file supplied by ESA - Parameters + Args: ---------- filename: str - string of the orbit filename - Returns + Returns: ------- t: Nt x 1 ndarray - a numpy vector with Nt elements containing time in python datetime @@ -484,12 +482,12 @@ def state_to_los(svs, llh_targets, out="lookangle"): Converts information from a state vector for a satellite orbit, given in terms of position and velocity, to line-of-sight information at each (lon,lat, height) coordinate requested by the user. - Parameters + Args: ---------- svs - t, x, y, z, vx, vy, vz - time, position, and velocity in ECEF of the sensor llh_targets - lats, lons, heights - Ellipsoidal (WGS84) positions of target ground pixels - Returns + Returns: ------- LOS - * x 3 matrix of LOS unit vectors in ECEF (*not* ENU) Example: @@ -546,12 +544,12 @@ def cut_times(times, ref_time, pad=3600 * 3): Slice the orbit file around the reference aquisition time. This is done by default using a three-hour window, which for Sentinel-1 empirically works out to be roughly the largest window allowed by the orbit time. - Parameters + Args: ---------- times: Nt x 1 ndarray - Vector of orbit times as datetime ref_time: datetime - Reference time pad: int - integer time in seconds to use as padding - Returns + Returns: ------- idx: Nt x 1 logical ndarray - a mask of times within the padded request time. """ @@ -565,13 +563,13 @@ def get_radar_pos(llh, orb, out="lookangle"): ''' Calculate the coordinate of the sensor in ECEF at the time corresponding to ***. - Parameters + Args: ---------- orb: isce3.core.Orbit - Nt x 7 matrix of statevectors: [t x y z vx vy vz] llh: ndarray - position of the target in LLH out: str - either lookangle or ecef for vector - Returns + Returns: ------- if out == "lookangle" los: ndarray - Satellite incidence angle diff --git a/tools/RAiDER/models/weatherModel.py b/tools/RAiDER/models/weatherModel.py index 76b9811c7..ef9e31e6f 100755 --- a/tools/RAiDER/models/weatherModel.py +++ b/tools/RAiDER/models/weatherModel.py @@ -127,7 +127,7 @@ def fetch(self, out, ll_bounds, time): Checks the input datetime against the valid date range for the model and then calls the model _fetch routine - Parameters + Args: ---------- out - ll_bounds - 4 x 1 array, SNWE @@ -381,7 +381,7 @@ def bbox(self) -> list: """ Obtains the bounding box of the weather model in lat/lon CRS. - Returns + Returns: ------- list xmin, ymin, xmax, ymax @@ -417,7 +417,7 @@ def checkContainment(self: weatherModel, Checks containment of weather model bbox of outLats and outLons provided. - Parameters + Args: ---------- weather_model : weatherModel outLats : np.ndarray @@ -429,7 +429,7 @@ def checkContainment(self: weatherModel, this ensures that translates have some overlap. The default is 1e-5 or ~11.1 meters. - Returns + Returns: ------- bool True if weather model contains bounding box of OutLats and outLons @@ -538,7 +538,7 @@ def _calculategeoh(self, z, lnsp): def getProjection(self): ''' - Returns the native weather projection, which should be a pyproj object + Returns: the native weather projection, which should be a pyproj object ''' return self._proj diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index ac944fa06..039f4eb70 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -17,13 +17,24 @@ def prepareWeatherModel( time=None, wmLoc=None, ll_bounds=None, - zref=None, download_only=False, makePlots=False, force_download=False, ): ''' Parse inputs to download and prepare a weather model grid for interpolation + + Args: + weather_model (WeatherModel): instantiated weather model object + time (datetime): Python datetime to request. Will be rounded to nearest available time + wmLoc (str): file path to which to write weather model file(s) + ll_bounds (list of float): bounding box to download in [S, N, W, E] format + download_only (bool): False if preprocessing weather model data + makePlots (bool): whether to write debug plots + force_download (bool): True if you want to download even when the weather model exists + + Returns: + filename of the netcdf file to which the weather model has been written ''' # Ensure the file output location exists if wmLoc is None: diff --git a/tools/RAiDER/utilFcns.py b/tools/RAiDER/utilFcns.py index 8989a476b..bf353884d 100755 --- a/tools/RAiDER/utilFcns.py +++ b/tools/RAiDER/utilFcns.py @@ -67,7 +67,7 @@ def enu2ecef( h0: ndarray, ): """ - Parameters + Args: ---------- e1 : float target east ENU coordinate (meters) @@ -345,7 +345,7 @@ def _get_g_ll(lats): def _get_Re(lats): ''' - Returns the ellipsoid as a fcn of latitude + Returns: the ellipsoid as a fcn of latitude ''' # TODO: verify constants, add to base class constants? return np.sqrt(1 / (((cosd(lats)**2) / Rmax**2) + ((sind(lats)**2) / Rmin**2))) @@ -675,7 +675,7 @@ def UTM_to_WGS84(z, l, x, y): def transform_bbox(wesn, dest_crs=4326, src_crs=4326, margin=100.): """ Transform bbox to lat/lon or another CRS for use with rest of workflow - Returns SNWE + Returns: SNWE """ # TODO - Handle dateline crossing if isinstance(src_crs, int): @@ -995,7 +995,7 @@ def calcgeoh(lnsp, t, q, z, a, b, R_d, num_levels): Calculate pressure, geopotential, and geopotential height from the surface pressure and model levels provided by a weather model. The model levels are numbered from the highest eleveation to the lowest. - Parameters + Args: ---------- lnsp: ndarray - [y, x] array of log surface pressure t: ndarray - [z, y, x] cube of temperatures @@ -1004,7 +1004,7 @@ def calcgeoh(lnsp, t, q, z, a, b, R_d, num_levels): a: ndarray - [z] vector of a values b: ndarray - [z] vector of b values num_levels: int - integer number of model levels - Returns + Returns: ------- geopotential - The geopotential in units of height times acceleration pressurelvs - The pressure at each of the model levels for each of From 72e976403e524fe3c200e73c32235a35cd6a5511 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 08:58:03 -0600 Subject: [PATCH 62/82] fix a few bugs --- tools/RAiDER/cli/__main__.py | 2 ++ tools/RAiDER/cli/raider.py | 1 - tools/RAiDER/processWM.py | 23 ++++++++++------------- 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/tools/RAiDER/cli/__main__.py b/tools/RAiDER/cli/__main__.py index a0aa7e056..b1cc91a95 100644 --- a/tools/RAiDER/cli/__main__.py +++ b/tools/RAiDER/cli/__main__.py @@ -2,6 +2,8 @@ import os from importlib.metadata import entry_points +from RAiDER.cli.raider import calcDelays, downloadGNSS, calcDelaysGUNW + def main(): parser = argparse.ArgumentParser( prefix_chars='+', diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index fc141d6b5..26cf24622 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -196,7 +196,6 @@ def calcDelays(iargs=None): model, t, ll_bounds=ll_bounds, # SNWE wmLoc=params['weather_model_directory'], - zref=params['zref'], download_only=dl_only, makePlots=params['verbose'], ) diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 039f4eb70..0558ac621 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -84,7 +84,6 @@ def prepareWeatherModel( f = weather_model.load( wmLoc, ll_bounds = ll_bounds, - zref=zref, ) if f is not None: logger.warning( @@ -149,17 +148,16 @@ def checkBounds(weather_model, outLats, outLons): def weather_model_debug( - los, - lats, - lons, - ll_bounds, - weather_model, - wmLoc, - zref, - time, - out, - download_only -): + los, + lats, + lons, + ll_bounds, + weather_model, + wmLoc, + time, + out, + download_only + ): """ raiderWeatherModelDebug main function. """ @@ -191,7 +189,6 @@ def weather_model_debug( lats=lats, lons=lons, ll_bounds=ll_bounds, - zref=zref, download_only=download_only, makePlots=True ) From 1d404f012c759619071c24771ead7d9fd319a5fe Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Wed, 14 Dec 2022 09:31:26 -0600 Subject: [PATCH 63/82] start refactoring for __main__.py --- test/_checkArgs.py | 29 +++++---------- tools/RAiDER/cli/__init__.py | 70 ++++++++++++++++++------------------ 2 files changed, 44 insertions(+), 55 deletions(-) diff --git a/test/_checkArgs.py b/test/_checkArgs.py index 8be5b17e0..66159d636 100644 --- a/test/_checkArgs.py +++ b/test/_checkArgs.py @@ -9,9 +9,11 @@ from test import TEST_DIR, pushd # import RAiDER.runProgram +from RAiDER.cli import AttributeDict, DEFAULT_DICT from RAiDER.cli.raider import ( - parseCMD, read_template_file, create_parser, read_template_file, + parseCMD, read_template_file, create_parser, read_template_file, DEFAULT_DICT, ) + from RAiDER.checkArgs import checkArgs, makeDelayFileNames from RAiDER.constants import _ZREF from RAiDER.losreader import Zenith, Conventional, Raytracing @@ -30,27 +32,12 @@ def isWriteable(dirpath): except IOError: return False - -@pytest.fixture -def parsed_args(tmp_path): - parser = create_parser() - args = parser.parse_args([ - '--date', '20200103', - '--time', '23:00:00', - # '--latlon', 'latfile.dat', 'lonfile.dat', - '--bbox', '-1', '1', '-1', '1', - '--model', 'ERA5', - '--outformat', 'hdf5' - ]) - return args, parser - - -def test_checkArgs_outfmt_1(parsed_args): +def test_checkArgs_outfmt_1(): '''Test that passing height levels with hdf5 outformat works''' - args, p = parsed_args - args.outformat = 'hdf5' - args.heightlvs = [10, 100, 1000] - checkArgs(args, p) + args = + args['outformat'] = 'hdf5' + args['heightlvs'] = [10, 100, 1000] + checkArgs(args) assert True diff --git a/tools/RAiDER/cli/__init__.py b/tools/RAiDER/cli/__init__.py index ee49f6ac8..33e1b51da 100644 --- a/tools/RAiDER/cli/__init__.py +++ b/tools/RAiDER/cli/__init__.py @@ -1,42 +1,44 @@ import os from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M + class AttributeDict(dict): __getattr__ = dict.__getitem__ __setattr__ = dict.__setitem__ __delattr__ = dict.__delitem__ -DEFAULT_DICT = dict( - look_dir='right', - date_start=None, - date_end=None, - date_step=None, - date_list=None, - time=None, - end_time=None, - weather_model=None, - lat_file=None, - lon_file=None, - station_file=None, - bounding_box=None, - geocoded_file=None, - dem=None, - use_dem_latlon=False, - height_levels=None, - height_file_rdr=None, - ray_trace=False, - zref=_ZREF, - cube_spacing_in_m=_CUBE_SPACING_IN_M, # TODO - Where are these parsed? - los_file=None, - los_convention='isce', - los_cube=None, - orbit_file=None, - verbose=True, - raster_format='GTiff', - download_only=False, - output_directory=os.getcwd(), - weather_model_directory=os.path.join( - os.getcwd(), - 'weather_files' - ), - output_projection='EPSG:4236', +DEFAULT_DICT = AttributeDict( + dict( + look_dir='right', + date_start=None, + date_end=None, + date_step=None, + date_list=None, + time=None, + end_time=None, + weather_model=None, + lat_file=None, + lon_file=None, + station_file=None, + bounding_box=None, + geocoded_file=None, + dem=None, + use_dem_latlon=False, + height_levels=None, + height_file_rdr=None, + ray_trace=False, + zref=_ZREF, + cube_spacing_in_m=_CUBE_SPACING_IN_M, + los_file=None, + los_convention='isce', + los_cube=None, + orbit_file=None, + verbose=True, + raster_format='GTiff', + output_directory=os.getcwd(), + weather_model_directory=os.path.join( + os.getcwd(), + 'weather_files' + ), + output_projection='EPSG:4236', + ) ) From 42f6a9da6f06fa196bcbf0176a8fe7cfa9c6165f Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Wed, 14 Dec 2022 13:12:37 -0600 Subject: [PATCH 64/82] checkArgs unit tests --- test/_checkArgs.py | 429 ----------------------------------- test/test_checkArgs.py | 180 +++++++++++++++ tools/RAiDER/checkArgs.py | 61 +++-- tools/RAiDER/cli/__init__.py | 1 + tools/RAiDER/cli/raider.py | 220 ++++++++++++++++++ tools/RAiDER/cli/raider.yaml | 2 +- 6 files changed, 450 insertions(+), 443 deletions(-) delete mode 100644 test/_checkArgs.py create mode 100644 test/test_checkArgs.py create mode 100644 tools/RAiDER/cli/raider.py diff --git a/test/_checkArgs.py b/test/_checkArgs.py deleted file mode 100644 index 66159d636..000000000 --- a/test/_checkArgs.py +++ /dev/null @@ -1,429 +0,0 @@ -import datetime -import os -import pytest - -import multiprocessing as mp -import numpy as np -import pandas as pd - -from test import TEST_DIR, pushd - -# import RAiDER.runProgram -from RAiDER.cli import AttributeDict, DEFAULT_DICT -from RAiDER.cli.raider import ( - parseCMD, read_template_file, create_parser, read_template_file, DEFAULT_DICT, -) - -from RAiDER.checkArgs import checkArgs, makeDelayFileNames -from RAiDER.constants import _ZREF -from RAiDER.losreader import Zenith, Conventional, Raytracing - - -SCENARIO_1 = os.path.join(TEST_DIR, "scenario_1") -SCENARIO_2 = os.path.join(TEST_DIR, "scenario_2") - - -def isWriteable(dirpath): - '''Test whether a directory is writeable''' - try: - filehandle = open(os.path.join(dirpath, 'tmp.txt'), 'w') - filehandle.close() - return True - except IOError: - return False - -def test_checkArgs_outfmt_1(): - '''Test that passing height levels with hdf5 outformat works''' - args = - args['outformat'] = 'hdf5' - args['heightlvs'] = [10, 100, 1000] - checkArgs(args) - assert True - - -def test_checkArgs_outfmt_2(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.heightlvs = [10, 100, 1000] - args.outformat = 'envi' - with pytest.raises(ValueError): - checkArgs(args, p) - - -def test_checkArgs_outfmt_3(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.query_area = os.path.join(SCENARIO_2, 'stations.csv') - argDict = checkArgs(args, p) - assert argDict['flag'] == 'station_file' - - -def test_checkArgs_outfmt_4(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.query_area = [os.path.join(SCENARIO_1, 'geom', 'lat.dat'), os.path.join(SCENARIO_1, 'geom', 'lat.dat')] - argDict = checkArgs(args, p) - assert argDict['flag'] == 'files' - - -def test_checkArgs_outfmt_5(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.query_area = os.path.join(SCENARIO_2, 'stations.csv') - argDict = checkArgs(args, p) - assert pd.read_csv(argDict['wetFilenames'][0]).shape == (8, 4) - - -def test_checkArgs_outloc_1(parsed_args): - '''Test that the default output and weather model directories are correct''' - args, p = parsed_args - argDict = checkArgs(args, p) - out = argDict['out'] - wmLoc = argDict['wmLoc'] - assert os.path.abspath(out) == os.getcwd() - assert os.path.abspath(wmLoc) == os.path.join(os.getcwd(), 'weather_files') - - -def test_checkArgs_outloc_2(parsed_args, tmp_path): - '''Tests that the correct output location gets assigned when provided''' - with pushd(tmp_path): - args, p = parsed_args - args.out = tmp_path - argDict = checkArgs(args, p) - out = argDict['out'] - assert out == tmp_path - - -def test_checkArgs_outloc_2b(parsed_args, tmp_path): - ''' Tests that the weather model directory gets passed through by itself''' - with pushd(tmp_path): - args, p = parsed_args - args.out = tmp_path - args.wmLoc = 'weather_dir' - argDict = checkArgs(args, p) - assert argDict['wmLoc'] == 'weather_dir' - - -def test_checkArgs_outloc_3(parsed_args): - '''Tests that the weather model directory gets created when needed''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert os.path.isdir(argDict['wmLoc']) - - -def test_checkArgs_outloc_4(parsed_args): - '''Tests for creating writeable weather model directory''' - args, p = parsed_args - argDict = checkArgs(args, p) - - assert isWriteable(argDict['wmLoc']) - - -def test_ll_bounds_1(parsed_args): - '''Tests that lats out of bounds raises error''' - args, p = parsed_args - args.query_area[0] = -91 - with pytest.raises(ValueError): - checkArgs(args, p) - - -def test_ll_bounds_2(parsed_args): - '''Tests that lats out of bounds raises error''' - args, p = parsed_args - args.query_area[1] = 91 - with pytest.raises(ValueError): - checkArgs(args, p) - - -def test_los_1(parsed_args): - '''Tests that lats out of bounds raises error''' - args, p = parsed_args - args.lineofsight = 'los.rdr' - argDict = checkArgs(args, p) - assert isinstance(argDict['los'], Conventional) - assert argDict['los']._file == 'los.rdr' - - -def test_los_2(parsed_args): - '''Tests that lats out of bounds raises error''' - args, p = parsed_args - args.statevectors = 'sv.txt' - argDict = checkArgs(args, p) - assert isinstance(argDict['los'], Conventional) - assert argDict['los']._file == 'sv.txt' - - -def test_los_3(parsed_args): - '''Tests that lats out of bounds raises error''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert isinstance(argDict['los'], Zenith) - - -def test_models_1a(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'ERA-5' - assert argDict['weather_model']['name'] == 'era5' - - -def test_models_1b(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - args.model = 'HRRR' - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'HRRR' - assert argDict['weather_model']['name'] == 'hrrr' - - -def test_models_1c(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - args.model = 'NCMR' - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'NCMR' - assert argDict['weather_model']['name'] == 'ncmr' - - -def test_models_1d(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - args.model = 'era-5' - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'ERA-5' - assert argDict['weather_model']['name'] == 'era5' - - -def test_models_1e(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - args.model = 'ERA-5' - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'ERA-5' - assert argDict['weather_model']['name'] == 'era5' - - -def test_models_1f(parsed_args): - '''Tests that the weather model gets passed through correctly''' - args, p = parsed_args - args.model = 'Era-5' - argDict = checkArgs(args, p) - assert argDict['weather_model']['type'].Model() == 'ERA-5' - assert argDict['weather_model']['name'] == 'era5' - - -def test_models_2(parsed_args): - '''Tests that unknown weather models get rejected''' - args, p = parsed_args - args.model = 'unknown' - with pytest.raises(NotImplementedError): - checkArgs(args, p) - - -def test_models_3a(parsed_args): - '''Tests that WRF weather models requires files''' - args, p = parsed_args - args.model = 'WRF' - with pytest.raises(RuntimeError): - checkArgs(args, p) - - -def test_models_3b(parsed_args): - '''Tests that HDF5 weather models requires files''' - args, p = parsed_args - args.model = 'HDF5' - with pytest.raises(RuntimeError): - checkArgs(args, p) - - -def test_models_3c(parsed_args): - '''Tests that WRF weather models requires files''' - args, p = parsed_args - args.model = 'WRF' - args.files = ['file1.wrf', 'file2.wrf'] - # argDict = checkArgs(args, p) - # TODO - assert True - - -def test_zref_1(parsed_args): - '''tests that default zref gets generated''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert argDict['zref'] == _ZREF - - -def test_zref_2(parsed_args): - '''tests that default zref gets generated''' - ztest = 20000 - args, p = parsed_args - args.zref = ztest - argDict = checkArgs(args, p) - assert argDict['zref'] == ztest - - -def test_parallel_1(parsed_args): - '''tests that parallel options are handled correctly''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert argDict['parallel'] == 1 - - -def test_parallel_2(parsed_args): - '''tests that parallel options are handled correctly''' - args, p = parsed_args - args.parallel = 'all' - argDict = checkArgs(args, p) - assert argDict['parallel'] == mp.cpu_count() - - -def test_parallel_3(parsed_args): - '''tests that parallel options are handled correctly''' - args, p = parsed_args - args.parallel = 2 - argDict = checkArgs(args, p) - assert argDict['parallel'] == 2 - - -def test_parallel_4(parsed_args): - '''tests that parallel options are handled correctly''' - args, p = parsed_args - args.parallel = 2000 - argDict = checkArgs(args, p) - assert argDict['parallel'] == mp.cpu_count() - - -def test_verbose_1(parsed_args): - '''tests that verbose option is handled correctly''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert not argDict['verbose'] - - -def test_verbose_2(parsed_args): - '''tests that verbose option is handled correctly''' - args, p = parsed_args - args.verbose = True - argDict = checkArgs(args, p) - assert argDict['verbose'] - - -def test_download_only_1(parsed_args): - '''tests that the download-only option is handled correctly''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert not argDict['download_only'] - - -def test_download_only_2(parsed_args): - '''tests that the download-only option is handled correctly''' - args, p = parsed_args - args.download_only = True - argDict = checkArgs(args, p) - assert argDict['download_only'] - - -def test_useWeatherNodes_1(parsed_args): - '''tests that the correct flag gets passed''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert argDict['flag'] == 'bounding_box' # default arguments use a bounding box - - -def test_filenames_1(parsed_args): - '''tests that the correct filenames are generated''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert 'Delay' not in argDict['wetFilenames'][0] - assert 'wet' in argDict['wetFilenames'][0] - assert 'hydro' in argDict['hydroFilenames'][0] - assert '20200103' in argDict['wetFilenames'][0] - assert '20200103' in argDict['hydroFilenames'][0] - assert len(argDict['hydroFilenames']) == 1 - - -def test_filenames_2(parsed_args): - '''tests that the correct filenames are generated''' - args, p = parsed_args - args.query_area = os.path.join(SCENARIO_2, 'stations.csv') - argDict = checkArgs(args, p) - assert 'Delay' in argDict['wetFilenames'][0] - assert '20200103' in argDict['wetFilenames'][0] - assert len(argDict['wetFilenames']) == 1 - - -def test_makeDelayFileNames_1(): - assert makeDelayFileNames(None, None, "h5", "name", "dir") == \ - ("dir/name_wet_ztd.h5", "dir/name_hydro_ztd.h5") - - -def test_makeDelayFileNames_2(): - assert makeDelayFileNames(None, (), "h5", "name", "dir") == \ - ("dir/name_wet_std.h5", "dir/name_hydro_std.h5") - - -def test_makeDelayFileNames_3(): - assert makeDelayFileNames(datetime.datetime(2020, 1, 1, 1, 2, 3), None, "h5", "model_name", "dir") == \ - ( - "dir/model_name_wet_20200101T010203_ztd.h5", - "dir/model_name_hydro_20200101T010203_ztd.h5" - ) - - -def test_makeDelayFileNames_4(): - assert makeDelayFileNames(datetime.datetime(1900, 12, 31, 1, 2, 3), "los", "h5", "model_name", "dir") == \ - ( - "dir/model_name_wet_19001231T010203_std.h5", - "dir/model_name_hydro_19001231T010203_std.h5" - ) - - -def test_model2module(): - model_module_name, model_obj = modelName2Module('ERA5') - assert model_obj().Model() == 'ERA-5' - - -def test_dem_1(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - argDict = checkArgs(args, p) - assert argDict['heights'][0] == 'skip' - assert argDict['heights'][1] is None - - -def test_dem_2(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.heightlvs = [10, 100, 1000] - argDict = checkArgs(args, p) - assert argDict['heights'][0] == 'lvs' - assert np.allclose(argDict['heights'][1], [10, 100, 1000]) - - -def test_dem_3(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.heightlvs = [10, 100, 1000] - args.query_area = os.path.join(SCENARIO_2, 'stations.csv') - argDict = checkArgs(args, p) - assert argDict['heights'][0] == 'lvs' - assert np.allclose(argDict['heights'][1], [10, 100, 1000]) - - -def test_dem_4(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.query_area = os.path.join(SCENARIO_2, 'stations.csv') - argDict = checkArgs(args, p) - assert argDict['heights'][0] == 'pandas' - assert argDict['heights'][1][0] == argDict['wetFilenames'][0] - - -def test_dem_5(parsed_args): - '''Test that passing a raster format with height levels throws an error''' - args, p = parsed_args - args.query_area = [os.path.join(SCENARIO_1, 'geom', 'lat.dat'), os.path.join(SCENARIO_1, 'geom', 'lat.dat')] - argDict = checkArgs(args, p) - assert argDict['heights'][0] == 'download' - assert argDict['heights'][1] == os.path.join(argDict['out'], 'geom', 'warpedDEM.dem') diff --git a/test/test_checkArgs.py b/test/test_checkArgs.py new file mode 100644 index 000000000..e0699e733 --- /dev/null +++ b/test/test_checkArgs.py @@ -0,0 +1,180 @@ +import datetime +import os +import pytest + +import multiprocessing as mp +import numpy as np +import pandas as pd + +from test import TEST_DIR, pushd + +from RAiDER.cli import DEFAULT_DICT +from RAiDER.checkArgs import checkArgs, makeDelayFileNames +from RAiDER.llreader import BoundingBox, StationFile, RasterRDR +from RAiDER.losreader import Zenith, Conventional, Raytracing +from RAiDER.models.gmao import GMAO + + +SCENARIO_1 = os.path.join(TEST_DIR, "scenario_1") +SCENARIO_2 = os.path.join(TEST_DIR, "scenario_2") + +@pytest.fixture +def args(): + d = DEFAULT_DICT + d['date_list'] = [datetime.datetime(2018, 1, 1)] + d['time'] = datetime.time(12,0,0) + d['aoi'] = BoundingBox([38, 39, -92, -91]) + d['los'] = Zenith() + d['weather_model'] = GMAO() + + return d + + +def isWriteable(dirpath): + '''Test whether a directory is writeable''' + try: + filehandle = open(os.path.join(dirpath, 'tmp.txt'), 'w') + filehandle.close() + return True + except IOError: + return False + +def test_checkArgs_outfmt_1(args): + '''Test that passing height levels with hdf5 outformat works''' + args = args + args.file_format = 'h5' + args.heightlvls = [10, 100, 1000] + checkArgs(args) + assert os.path.splitext(args.wetFilenames[0])[-1] == '.h5' + + +def test_checkArgs_outfmt_2(args): + '''Test that passing a raster format with height levels throws an error''' + args = args + args.heightlvs = [10, 100, 1000] + args.file_format = 'GTiff' + args = checkArgs(args) + assert os.path.splitext(args.wetFilenames[0])[-1] == '.nc' + + +def test_checkArgs_outfmt_3(args): + '''Test that passing a raster format with height levels throws an error''' + args = args + with pytest.raises(FileNotFoundError): + args.aoi = StationFile(os.path.join('fake_dir', 'stations.csv')) + + +def test_checkArgs_outfmt_4(args): + '''Test that passing a raster format with height levels throws an error''' + args = args + args.aoi = RasterRDR( + lat_file = os.path.join(SCENARIO_1, 'geom', 'lat.dat'), + lon_file = os.path.join(SCENARIO_1, 'geom', 'lon.dat'), + ) + argDict = checkArgs(args) + assert argDict.aoi.type()=='radar_rasters' + + +def test_checkArgs_outfmt_5(args): + '''Test that passing a raster format with height levels throws an error''' + args = args + args.aoi = StationFile(os.path.join(SCENARIO_2, 'stations.csv')) + argDict = checkArgs(args) + assert pd.read_csv(argDict['wetFilenames'][0]).shape == (8, 4) + + +def test_checkArgs_outloc_1(args): + '''Test that the default output and weather model directories are correct''' + args = args + argDict = checkArgs(args) + out = argDict['output_directory'] + wmLoc = argDict['weather_model_directory'] + assert os.path.abspath(out) == os.getcwd() + assert os.path.abspath(wmLoc) == os.path.join(os.getcwd(), 'weather_files') + + +def test_checkArgs_outloc_2(args, tmp_path): + '''Tests that the correct output location gets assigned when provided''' + with pushd(tmp_path): + args = args + args.output_directory = tmp_path + argDict = checkArgs(args) + out = argDict['output_directory'] + assert out == tmp_path + + +def test_checkArgs_outloc_2b(args, tmp_path): + ''' Tests that the weather model directory gets passed through by itself''' + with pushd(tmp_path): + args = args + args.output_directory = tmp_path + args.weather_model_directory = 'weather_dir' + argDict = checkArgs(args) + assert argDict['weather_model_directory'] == 'weather_dir' + + +def test_checkArgs_outloc_3(args, tmp_path): + '''Tests that the weather model directory gets created when needed''' + with pushd(tmp_path): + args = args + args.output_directory = tmp_path + argDict = checkArgs(args) + assert os.path.isdir(argDict['weather_model_directory']) + + +def test_checkArgs_outloc_4(args): + '''Tests for creating writeable weather model directory''' + args = args + argDict = checkArgs(args) + + assert isWriteable(argDict['weather_model_directory']) + + +def test_filenames_1(args): + '''tests that the correct filenames are generated''' + args = args + argDict = checkArgs(args) + assert 'Delay' not in argDict['wetFilenames'][0] + assert 'wet' in argDict['wetFilenames'][0] + assert 'hydro' in argDict['hydroFilenames'][0] + assert '20180101' in argDict['wetFilenames'][0] + assert '20180101' in argDict['hydroFilenames'][0] + assert len(argDict['hydroFilenames']) == 1 + + +def test_filenames_2(args): + '''tests that the correct filenames are generated''' + args = args + args.aoi = StationFile(os.path.join(SCENARIO_2, 'stations.csv')) + argDict = checkArgs(args) + assert '20180101' in argDict['wetFilenames'][0] + assert len(argDict['wetFilenames']) == 1 + + +def test_makeDelayFileNames_1(): + assert makeDelayFileNames(None, None, "h5", "name", "dir") == \ + ("dir/name_wet_ztd.h5", "dir/name_hydro_ztd.h5") + + +def test_makeDelayFileNames_2(): + assert makeDelayFileNames(None, (), "h5", "name", "dir") == \ + ("dir/name_wet_std.h5", "dir/name_hydro_std.h5") + + +def test_makeDelayFileNames_3(): + assert makeDelayFileNames(datetime.datetime(2020, 1, 1, 1, 2, 3), None, "h5", "model_name", "dir") == \ + ( + "dir/model_name_wet_20200101T010203_ztd.h5", + "dir/model_name_hydro_20200101T010203_ztd.h5" + ) + + +def test_makeDelayFileNames_4(): + assert makeDelayFileNames(datetime.datetime(1900, 12, 31, 1, 2, 3), "los", "h5", "model_name", "dir") == \ + ( + "dir/model_name_wet_19001231T010203_std.h5", + "dir/model_name_hydro_19001231T010203_std.h5" + ) + + + diff --git a/tools/RAiDER/checkArgs.py b/tools/RAiDER/checkArgs.py index 774b548f5..04443f7c8 100644 --- a/tools/RAiDER/checkArgs.py +++ b/tools/RAiDER/checkArgs.py @@ -8,11 +8,14 @@ import os import pandas as pd +import rasterio.drivers as rd from datetime import datetime + from RAiDER.losreader import Zenith from RAiDER.llreader import BoundingBox +from RAiDER.logger import logger def checkArgs(args): @@ -34,36 +37,54 @@ def checkArgs(args): # filenames wetNames, hydroNames = [], [] for d in args.date_list: - if not args.aoi is not BoundingBox: - if args.station_file is not None: + if (args.aoi.type() != 'bounding_box'): + + # Handle the GNSS station file + if (args.aoi.type()=='station_file'): wetFilename = os.path.join( args.output_directory, '{}_Delay_{}.csv' .format( - args.weather_model, - args.time.strftime('%Y%m%dT%H%M%S'), + args.weather_model.Model(), + d.strftime('%Y%m%dT%H%M%S'), ) ) - hydroFilename = wetFilename + hydroFilename = None # only the 'wetFilename' is used for the station_file - # copy the input file to the output location for editing - indf = pd.read_csv(args.query_area).drop_duplicates(subset=["Lat", "Lon"]) + # copy the input station file to the output location for editing + indf = pd.read_csv(args.aoi._filename).drop_duplicates(subset=["Lat", "Lon"]) indf.to_csv(wetFilename, index=False) else: - wetNames.append(None) - hydroNames.append(None) + # This implies rasters + fmt = get_raster_ext(args.file_format) + wetFilename, hydroFilename = makeDelayFileNames( + d, + args.los, + fmt, + args.weather_model._dataset.upper(), + args.output_directory, + ) + + else: + # In this case a cube file format is needed + if args.file_format not in '.nc .h5 h5 hdf5 .hdf5 nc'.split(): + fmt = 'nc' + logger.debug('Invalid extension %s for cube. Defaulting to .nc', args.file_format) + else: + fmt = args.file_format.strip('.').replace('df', '') + wetFilename, hydroFilename = makeDelayFileNames( d, args.los, - args.raster_format, + fmt, args.weather_model._dataset.upper(), args.output_directory, ) - - wetNames.append(wetFilename) - hydroNames.append(hydroFilename) + + wetNames.append(wetFilename) + hydroNames.append(hydroFilename) args.wetFilenames = wetNames args.hydroFilenames = hydroNames @@ -71,6 +92,20 @@ def checkArgs(args): return args +def get_raster_ext(fmt): + drivers = rd.raster_driver_extensions() + extensions = {value.upper():key for key, value in drivers.items()} + + # add in ENVI/ISCE formats with generic extension + extensions['ENVI'] = '.dat' + extensions['ISCE'] = '.dat' + + try: + return extensions[fmt.upper()] + except KeyError: + raise ValueError('{} is not a valid gdal/rasterio file format for rasters'.format(fmt)) + + def makeDelayFileNames(time, los, outformat, weather_model_name, out): ''' return names for the wet and hydrostatic delays. diff --git a/tools/RAiDER/cli/__init__.py b/tools/RAiDER/cli/__init__.py index 33e1b51da..79a9cc19e 100644 --- a/tools/RAiDER/cli/__init__.py +++ b/tools/RAiDER/cli/__init__.py @@ -34,6 +34,7 @@ class AttributeDict(dict): orbit_file=None, verbose=True, raster_format='GTiff', + file_format='GTiff', output_directory=os.getcwd(), weather_model_directory=os.path.join( os.getcwd(), diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py new file mode 100644 index 000000000..9097d8254 --- /dev/null +++ b/tools/RAiDER/cli/raider.py @@ -0,0 +1,220 @@ +import argparse +import os +import shutil +import sys +import yaml +import re, glob + +import RAiDER +from RAiDER.constants import _ZREF, _CUBE_SPACING_IN_M +from RAiDER.logger import logger, logging +from RAiDER.cli import DEFAULT_DICT, AttributeDict +from RAiDER.cli.validators import (enforce_time, enforce_bbox, parse_dates, + get_query_region, get_heights, get_los, enforce_wm) + +from RAiDER.checkArgs import checkArgs +from RAiDER.delay import main as main_delay + + +HELP_MESSAGE = """ +Command line options for RAiDER processing. Default options can be found by running +raider.py --generate_config + +Download a weather model and calculate tropospheric delays +""" + +SHORT_MESSAGE = """ +Program to calculate troposphere total delays using a weather model +""" + +EXAMPLES = """ +Usage examples: +raider.py -g +raider.py customTemplatefile.cfg +""" + +def create_parser(): + """Parse command line arguments using argparse.""" + p = argparse.ArgumentParser( + formatter_class = argparse.RawDescriptionHelpFormatter, + description = HELP_MESSAGE, + epilog = EXAMPLES, + ) + + p.add_argument( + 'customTemplateFile', nargs='?', + help='custom template with option settings.\n' + + "ignored if the default smallbaselineApp.cfg is input." + ) + + p.add_argument( + '-g', '--generate_template', + dest='generate_template', + action='store_true', + help='generate default template (if it does not exist) and exit.' + ) + + + p.add_argument( + '--download-only', + action='store_true', + help='only download a weather model.' + ) + + return p + + +def parseCMD(iargs=None): + """ + Parse command-line arguments and pass to delay.py + """ + + p = create_parser() + args = p.parse_args(args=iargs) + + args.argv = iargs if iargs else sys.argv[1:] + + # default input file + template_file = os.path.join( + os.path.dirname( + RAiDER.__file__ + ), + 'cli', 'raider.yaml' + ) + if '-g' in args.argv: + dst = os.path.join(os.getcwd(), 'raider.yaml') + shutil.copyfile( + template_file, + dst, + ) + + logger.info('Wrote %s', dst) + sys.exit(0) + + # check: existence of input template files + if (not args.customTemplateFile + and not os.path.isfile(os.path.basename(template_file)) + and not args.generate_template): + p.print_usage() + print(EXAMPLES) + + msg = "No template file found! It requires that either:" + msg += "\n a custom template file, OR the default template " + msg += "\n file 'raider.yaml' exists in current directory." + raise SystemExit(f'ERROR: {msg}') + + if args.customTemplateFile: + # check the existence + if not os.path.isfile(args.customTemplateFile): + raise FileNotFoundError(args.customTemplateFile) + + args.customTemplateFile = os.path.abspath(args.customTemplateFile) + + return args + + +def read_template_file(fname): + """ + Read the template file into a dictionary structure. + Parameters: fname - str, full path to the template file + delimiter - str, string to separate the key and value + skip_chars - list of str, skip certain charaters in values + Returns: template - dict, file content + Examples: template = read_template('raider.yaml') + + Modified from MintPy's 'read_template' + """ + with open(fname, 'r') as f: + try: + params = yaml.safe_load(f) + except yaml.YAMLError as exc: + print(exc) + raise ValueError('Something is wrong with the yaml file {}'.format(fname)) + + # Drop any values not specified + params = drop_nans(params) + + # Need to ensure that all the groups exist, even if they are not specified by the user + group_keys = ['date_group', 'time_group', 'aoi_group', 'height_group', 'los_group', 'runtime_group'] + for key in group_keys: + if not key in params.keys(): + params[key] = {} + + # Parse the user-provided arguments + template = DEFAULT_DICT + for key, value in params.items(): + if key == 'runtime_group': + for k, v in value.items(): + if v is not None: + template[k] = v + if key == 'weather_model': + template[key]= enforce_wm(value) + if key == 'time_group': + template.update(enforce_time(AttributeDict(value))) + if key == 'date_group': + template['date_list'] = parse_dates(AttributeDict(value)) + if key == 'aoi_group': + ## in case a DEM is passed and should be used + dct_temp = {**AttributeDict(value), + **AttributeDict(params['height_group'])} + template['aoi'] = get_query_region(AttributeDict(dct_temp)) + + if key == 'los_group': + template['los'] = get_los(AttributeDict(value)) + if key == 'look_dir': + if value.lower() not in ['right', 'left']: + raise ValueError(f"Unknown look direction {value}") + template['look_dir'] = value.lower() + + # Have to guarantee that certain variables exist prior to looking at heights + for key, value in params.items(): + if key == 'height_group': + template.update( + get_heights( + AttributeDict(value), + template['output_directory'], + template['station_file'], + template['bounding_box'], + ) + ) + return AttributeDict(template) + + +def drop_nans(d): + for key in list(d.keys()): + if d[key] is None: + del d[key] + elif isinstance(d[key], dict): + for k in list(d[key].keys()): + if d[key][k] is None: + del d[key][k] + return d + + +########################################################################## +def main(iargs=None): + # parse + inps = parseCMD(iargs) + + # Read the template file + params = read_template_file(inps.customTemplateFile) + + # Argument checking + params = checkArgs(params) + + params['download_only'] = inps.download_only + + if not params.verbose: + logger.setLevel(logging.INFO) + + + for t, w, f in zip( + params['date_list'], + params['wetFilenames'], + params['hydroFilenames'] + ): + try: + (_, _) = main_delay(t, w, f, params) + except RuntimeError: + logger.exception("Date %s failed", t) + continue diff --git a/tools/RAiDER/cli/raider.yaml b/tools/RAiDER/cli/raider.yaml index 1ebe8ee24..8e135917e 100644 --- a/tools/RAiDER/cli/raider.yaml +++ b/tools/RAiDER/cli/raider.yaml @@ -121,7 +121,7 @@ los_group: ## runtime_group: verbose: True - raster_format: GTiff # Can be any rasterio-compatible format + file_format: GTiff # Can be any rasterio-compatible format output_directory: . # uses the runtime directory by default weather_model_directory: # Defaults to /weather_files/ output_projection: # Specify the PROJ-compatible projection of the output delays as an EPSG code From 1aee1fb7b8b6b07ac98b7cb055de05f7bef87e1d Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 09:32:34 -0600 Subject: [PATCH 65/82] few more docstrings --- test/__init__.py | 7 +++++++ tools/RAiDER/delay.py | 32 +++++++++++++++++++++++++------- tools/RAiDER/processWM.py | 22 +++++++++++++--------- 3 files changed, 45 insertions(+), 16 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index 1bc3f6e19..d362ccf8e 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,3 +1,10 @@ +# RAiDER/__init__.py + +"""Raytracing Atmospheric Delay Estimation for RADAR + +Code to calculate tropospheric delays for use with SAR/InSAR. +ZTD, STD - projection and STD - raytracing are all supported. +""" import os from contextlib import contextmanager diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index dc9b3bcae..80311731e 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -5,34 +5,52 @@ # RESERVED. United States Government Sponsorship acknowledged. # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + """RAiDER tropospheric delay calculation + + This module provides the main RAiDER functionality for calculating + tropospheric wet and hydrostatic delays from a weather model. Weather + models are accessed as NETCDF files and should have "wet" "hydro" + "wet_total" and "hydro_total" fields specified. + + Returns: + xarray Dataset: If delays are requested for a cube + *or* + length-2 tuple of ndarray: if delays are requested at query points (e.g. lat/lon files) + """ import os import datetime import h5py -import numpy as np import pyproj import xarray -from netCDF4 import Dataset from pyproj import CRS, Transformer -from pyproj.exceptions import CRSError -from scipy.interpolate import RegularGridInterpolator as Interpolator +from typing import List import isce3.ext.isce3 as isce +import numpy as np -from RAiDER.constants import _STEP from RAiDER.delayFcns import ( getInterpolators ) from RAiDER.logger import logger, logging -from RAiDER.losreader import Zenith, Conventional, Raytracing, get_sv, getTopOfAtmosphere +from RAiDER.losreader import get_sv, getTopOfAtmosphere from RAiDER.utilFcns import ( lla2ecef, transform_bbox, clip_bbox, rio_profile, ) ############################################################################### -def tropo_delay(dt, weather_model_file, aoi, los, height_levels=None, out_proj=4326, look_dir='right', cube_spacing_m=None): +def tropo_delay( + dt, + weather_model_file: str, + aoi, + los, + height_levels: List[float]=None, + out_proj: int | str=4326, + look_dir: str='right', + cube_spacing_m: int=None + ): """ Calculate integrated delays on query points. diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 0558ac621..93b350729 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -6,21 +6,25 @@ # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ import os + +import matplotlib.pyplot as plt import numpy as np + +from typing import List + from RAiDER.logger import logger from RAiDER.utilFcns import getTimeFromFile -import matplotlib.pyplot as plt def prepareWeatherModel( - weather_model, - time=None, - wmLoc=None, - ll_bounds=None, - download_only=False, - makePlots=False, - force_download=False, -): + weather_model, + time=None, + wmLoc: str=None, + ll_bounds: List[float]=None, + download_only: bool=False, + makePlots: bool=False, + force_download: bool=False, + ): ''' Parse inputs to download and prepare a weather model grid for interpolation From b1bee3617503cd9ab6b53a9ff9e60df8388e10c5 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 14:00:24 -0600 Subject: [PATCH 66/82] few typos, updates --- tools/RAiDER/cli/raider.py | 2 -- tools/RAiDER/delay.py | 26 +++++++++++++------------- tools/RAiDER/processWM.py | 22 +++++++++++----------- 3 files changed, 24 insertions(+), 26 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 73bef6dac..dd8030caa 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -12,8 +12,6 @@ from RAiDER.cli.validators import ( enforce_time, enforce_bbox, parse_dates, get_query_region, get_heights, get_los, enforce_wm ) -from RAiDER.checkArgs import checkArgs -from RAiDER.delay import main as main_delay HELP_MESSAGE = """ diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 80311731e..827d769f9 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -5,18 +5,18 @@ # RESERVED. United States Government Sponsorship acknowledged. # # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - """RAiDER tropospheric delay calculation - - This module provides the main RAiDER functionality for calculating - tropospheric wet and hydrostatic delays from a weather model. Weather - models are accessed as NETCDF files and should have "wet" "hydro" - "wet_total" and "hydro_total" fields specified. - - Returns: - xarray Dataset: If delays are requested for a cube - *or* - length-2 tuple of ndarray: if delays are requested at query points (e.g. lat/lon files) - """ +"""RAiDER tropospheric delay calculation + +This module provides the main RAiDER functionality for calculating +tropospheric wet and hydrostatic delays from a weather model. Weather +models are accessed as NETCDF files and should have "wet" "hydro" +"wet_total" and "hydro_total" fields specified. + +Returns: + xarray Dataset: If delays are requested for a cube + *or* + length-2 tuple of ndarray: if delays are requested at query points (e.g. lat/lon files) +""" import os import datetime @@ -65,7 +65,7 @@ def tropo_delay( cube_spacing_m: int - (optional) Horizontal spacing in meters when generating cubes Returns: - xarray Dataset *or* ndarrays: wet and hydrostatic delays at the grid nodes / query points. + xarray Dataset *or* ndarrays: - wet and hydrostatic delays at the grid nodes / query points. """ # get heights if height_levels is None: diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 93b350729..8656d2f1a 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -25,21 +25,21 @@ def prepareWeatherModel( makePlots: bool=False, force_download: bool=False, ): - ''' + """ Parse inputs to download and prepare a weather model grid for interpolation - Args: - weather_model (WeatherModel): instantiated weather model object - time (datetime): Python datetime to request. Will be rounded to nearest available time - wmLoc (str): file path to which to write weather model file(s) - ll_bounds (list of float): bounding box to download in [S, N, W, E] format - download_only (bool): False if preprocessing weather model data - makePlots (bool): whether to write debug plots - force_download (bool): True if you want to download even when the weather model exists + Args: + weather_model: WeatherModel - instantiated weather model object + time: datetime - Python datetime to request. Will be rounded to nearest available time + wmLoc: str - file path to which to write weather model file(s) + ll_bounds: list of float - bounding box to download in [S, N, W, E] format + download_only: bool - False if preprocessing weather model data + makePlots: bool - whether to write debug plots + force_download: bool - True if you want to download even when the weather model exists Returns: - filename of the netcdf file to which the weather model has been written - ''' + str: filename of the netcdf file to which the weather model has been written + """ # Ensure the file output location exists if wmLoc is None: wmLoc = os.path.join(os.getcwd(), 'weather_files') From 63132f4b0ad595ac144b3e7978c38167bd9b6ef5 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 14:01:04 -0600 Subject: [PATCH 67/82] add docs --- docs/index.md | 7 +++++++ docs/reference.md | 10 ++++++++++ docs/reference_pwm.md | 3 +++ docs/reference_td.md | 3 +++ docs/tutorials.md | 3 +++ mkdocs.yml | 17 +++++++++++++++++ 6 files changed, 43 insertions(+) create mode 100644 docs/index.md create mode 100644 docs/reference.md create mode 100644 docs/reference_pwm.md create mode 100644 docs/reference_td.md create mode 100644 docs/tutorials.md create mode 100644 mkdocs.yml diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 000000000..87a8dedb3 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,7 @@ +# Welcome to RAiDER + +This site contains the project documentation for the +[`RAiDER`](https://github.com/dbekaert/RAiDER) project. +Its aims are to provide a uniform interface for: +1) access to different weather models, which can be used to +2) calculate tropospheric delays for SAR/InSAR. \ No newline at end of file diff --git a/docs/reference.md b/docs/reference.md new file mode 100644 index 000000000..7ee3c3b27 --- /dev/null +++ b/docs/reference.md @@ -0,0 +1,10 @@ +Python library API + +### Core Datastructures +TBD + +### Weather Model Access +[`prepareWeatherModel`](reference_pwm.md) + +### Delay calculation +[`tropo_delay`](reference_td.md) diff --git a/docs/reference_pwm.md b/docs/reference_pwm.md new file mode 100644 index 000000000..f1305cb20 --- /dev/null +++ b/docs/reference_pwm.md @@ -0,0 +1,3 @@ +***`prepareWeatherModel`*** + +::: RAiDER.processWM.prepareWeatherModel \ No newline at end of file diff --git a/docs/reference_td.md b/docs/reference_td.md new file mode 100644 index 000000000..70bd7b3cf --- /dev/null +++ b/docs/reference_td.md @@ -0,0 +1,3 @@ +***`tropo_delay`*** + +::: RAiDER.delay.tropo_delay \ No newline at end of file diff --git a/docs/tutorials.md b/docs/tutorials.md new file mode 100644 index 000000000..a6f12a32e --- /dev/null +++ b/docs/tutorials.md @@ -0,0 +1,3 @@ +### Tutorials + +For detailed tutorials, visit the [RAiDER-docs](https://github.com/dbekaert/RAiDER-docs) repo. \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml new file mode 100644 index 000000000..f3001042d --- /dev/null +++ b/mkdocs.yml @@ -0,0 +1,17 @@ +site_name: RAiDER API Documentation +repo_url: https://github.com/dbekaert/RAiDER + + +nav: + - Home page: index.md + - Tutorials: tutorials.md + - API Reference: reference.md + - Weather Model Access: reference_pwm.md + - Delay Calculation: reference_td.md + + +theme: + name: readthedocs + +plugins: + - mkdocstrings \ No newline at end of file From d4ca1b643a581d7f707154b9b75f6cacb70eb88e Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 14:18:13 -0600 Subject: [PATCH 68/82] do not allow 'download_only' when running the calcDelays workflow --- tools/RAiDER/cli/raider.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index dd8030caa..5f4578cf8 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -265,7 +265,6 @@ def calcDelays(iargs=None): # Argument checking params = checkArgs(params) - dl_only = True if params['download_only'] or args.download_only else False if not params.verbose: logger.setLevel(logging.INFO) @@ -296,17 +295,12 @@ def calcDelays(iargs=None): model, t, ll_bounds=ll_bounds, # SNWE wmLoc=params['weather_model_directory'], - download_only=dl_only, makePlots=params['verbose'], ) except RuntimeError: logger.exception("Date %s failed", t) continue - # dont process the delays for download only - if dl_only: - continue - # Now process the delays try: wet_delay, hydro_delay = tropo_delay( From 299fdd2d1eddb439cfe28de9f402435c3cc1c260 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 14:20:27 -0600 Subject: [PATCH 69/82] update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f90b42adc..7bd53149b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.3.x] +Added documentation for the Python library interface. +Added some unit tests. +Fixed some bugs and tweaked the CLI. + ## [0.3.1] RAiDER package was refactored to use a __main__ file to allow calls to different functionality. The default is `calcDelays` which maintains the original functionality of calling `raider.py`. From 2aaf8f805ecf2e9cbd20ff822d3734aab50d8c51 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 14:28:26 -0600 Subject: [PATCH 70/82] put the init message in the wrong place --- test/__init__.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/test/__init__.py b/test/__init__.py index d362ccf8e..1bc3f6e19 100644 --- a/test/__init__.py +++ b/test/__init__.py @@ -1,10 +1,3 @@ -# RAiDER/__init__.py - -"""Raytracing Atmospheric Delay Estimation for RADAR - -Code to calculate tropospheric delays for use with SAR/InSAR. -ZTD, STD - projection and STD - raytracing are all supported. -""" import os from contextlib import contextmanager From 7b798246bb5a00dc57eddeee728f8fd4fcba612a Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Thu, 15 Dec 2022 16:10:29 -0600 Subject: [PATCH 71/82] v0.3.2 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bd53149b..adbb957cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.x] +## [0.3.2] Added documentation for the Python library interface. Added some unit tests. Fixed some bugs and tweaked the CLI. From b50e992e46d5a4b9521d5ee7e5e4aeb7221e86b3 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 19 Dec 2022 15:27:35 -0600 Subject: [PATCH 72/82] hide functions from docs that are not ready to be exposed --- docs/reference_pwm.md | 3 +- docs/reference_td.md | 2 +- mkdocs.yml | 2 +- test/weather_model/test_weather_model.py | 4 +- tools/RAiDER/cli/raider.py | 8 ++-- tools/RAiDER/delay.py | 57 +++++++----------------- tools/RAiDER/processWM.py | 22 ++------- 7 files changed, 29 insertions(+), 69 deletions(-) diff --git a/docs/reference_pwm.md b/docs/reference_pwm.md index f1305cb20..2cc8ddfe9 100644 --- a/docs/reference_pwm.md +++ b/docs/reference_pwm.md @@ -1,3 +1,2 @@ -***`prepareWeatherModel`*** -::: RAiDER.processWM.prepareWeatherModel \ No newline at end of file +::: RAiDER.processWM \ No newline at end of file diff --git a/docs/reference_td.md b/docs/reference_td.md index 70bd7b3cf..8832496ca 100644 --- a/docs/reference_td.md +++ b/docs/reference_td.md @@ -1,3 +1,3 @@ ***`tropo_delay`*** -::: RAiDER.delay.tropo_delay \ No newline at end of file +::: RAiDER.delay \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index f3001042d..f27b21c3c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,7 +11,7 @@ nav: theme: - name: readthedocs + name: material plugins: - mkdocstrings \ No newline at end of file diff --git a/test/weather_model/test_weather_model.py b/test/weather_model/test_weather_model.py index 0cf5cc633..74ddd4f28 100644 --- a/test/weather_model/test_weather_model.py +++ b/test/weather_model/test_weather_model.py @@ -13,7 +13,7 @@ from pyproj import CRS from RAiDER.constants import _ZMIN, _ZREF -from RAiDER.delay import build_cube_ray +from RAiDER.delay import _build_cube_ray from RAiDER.losreader import state_to_los from RAiDER.models.weatherModel import ( WeatherModel, @@ -240,7 +240,7 @@ def test_build_cube_ray(setup_fake_raytracing, model): #TODO: Check that the look vectors are not nans lv, xyz = state_to_los(svs, np.stack([_Y.ravel(), _X.ravel(), _Z.ravel()], axis=-1),out="ecef") - out = build_cube_ray(xs, ys, zs, orb, look_dir, CRS(4326), CRS(4326), [m.interpWet(), m.interpHydro()], elp=elp) + out = _build_cube_ray(xs, ys, zs, orb, look_dir, CRS(4326), CRS(4326), [m.interpWet(), m.interpHydro()], elp=elp) assert out.shape == out_true.shape diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index 5f4578cf8..8613a4f82 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -305,10 +305,10 @@ def calcDelays(iargs=None): try: wet_delay, hydro_delay = tropo_delay( t, weather_model_file, aoi, los, - params['height_levels'], - params['output_projection'], - params['look_dir'], - params['cube_spacing_in_m'] + height_levels = params['height_levels'], + out_proj = params['output_projection'], + look_dir = params['look_dir'], + cube_spacing_m = params['cube_spacing_in_m'], ) except RuntimeError: logger.exception("Date %s failed", t) diff --git a/tools/RAiDER/delay.py b/tools/RAiDER/delay.py index 827d769f9..bd593dcec 100755 --- a/tools/RAiDER/delay.py +++ b/tools/RAiDER/delay.py @@ -11,16 +11,10 @@ tropospheric wet and hydrostatic delays from a weather model. Weather models are accessed as NETCDF files and should have "wet" "hydro" "wet_total" and "hydro_total" fields specified. - -Returns: - xarray Dataset: If delays are requested for a cube - *or* - length-2 tuple of ndarray: if delays are requested at query points (e.g. lat/lon files) """ import os import datetime -import h5py import pyproj import xarray @@ -48,8 +42,8 @@ def tropo_delay( los, height_levels: List[float]=None, out_proj: int | str=4326, + cube_spacing_m: int=None, look_dir: str='right', - cube_spacing_m: int=None ): """ Calculate integrated delays on query points. @@ -73,7 +67,7 @@ def tropo_delay( height_levels = ds.z.values #TODO: expose this as library function - ds = tropo_delay_cube(dt, weather_model_file, aoi.bounds(), height_levels, + ds = _get_delays_on_cube(dt, weather_model_file, aoi.bounds(), height_levels, los, out_proj=out_proj, cube_spacing_m=cube_spacing_m, look_dir=look_dir) if (aoi.type() == 'bounding_box') or (aoi.type() == 'Geocube'): @@ -94,7 +88,7 @@ def tropo_delay( pnts = pnts.transpose(1,2,0) elif pnts.ndim == 2: pnts = pnts.T - ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from tropo_delay_cube calls the total delays 'wet' and 'hydro' + ifWet, ifHydro = getInterpolators(ds, 'ztd') # the cube from get_delays_on_cube calls the total delays 'wet' and 'hydro' wetDelay = ifWet(pnts) hydroDelay = ifHydro(pnts) @@ -108,7 +102,7 @@ def tropo_delay( return wetDelay, hydroDelay -def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4326, cube_spacing_m=None, look_dir='right', nproc=1): +def _get_delays_on_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4326, cube_spacing_m=None, look_dir='right', nproc=1): """ raider cube generation function. """ @@ -163,7 +157,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 ifWet, ifHydro = getInterpolators(weather_model_file, "total") # Build cube - wetDelay, hydroDelay = build_cube( + wetDelay, hydroDelay = _build_cube( xpts, ypts, zpts, wm_proj, crs, [ifWet, ifHydro]) @@ -180,7 +174,7 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 # Build cube if nproc == 1: - wetDelay, hydroDelay = build_cube_ray( + wetDelay, hydroDelay = _build_cube_ray( xpts, ypts, zpts, dt, los._file, look_dir, wm_proj, crs, @@ -261,37 +255,20 @@ def tropo_delay_cube(dt, weather_model_file, ll_bounds, heights, los, out_proj=4 return ds -def checkQueryPntsFile(pnts_file, query_shape): - ''' - Check whether the query points file exists, and if it - does, check that the shapes are all consistent - ''' - write_flag = True - if os.path.exists(pnts_file): - # Check whether the number of points is consistent with the new inputs - with h5py.File(pnts_file, 'r') as f: - if query_shape == tuple(f['lon'].attrs['Shape']): - write_flag = False - - return write_flag - - -def transformPoints(lats, lons, hgts, old_proj, new_proj): +def transformPoints(lats: np.ndarray, lons: np.ndarray, hgts: np.ndarray, old_proj: CRS, new_proj: CRS) -> np.ndarray: ''' Transform lat/lon/hgt data to an array of points in a new projection Args: - ---------- - lats - WGS-84 latitude (EPSG: 4326) - lons - ditto for longitude - hgts - Ellipsoidal height in meters - old_proj - the original projection of the points - new_proj - the new projection in which to return the points + lats: ndarray - WGS-84 latitude (EPSG: 4326) + lons: ndarray - ditto for longitude + hgts: ndarray - Ellipsoidal height in meters + old_proj: CRS - the original projection of the points + new_proj: CRS - the new projection in which to return the points Returns: - ------- - the array of query points in the weather model coordinate system (YX) + ndarray: the array of query points in the weather model coordinate system (YX) ''' t = Transformer.from_crs(old_proj, new_proj) @@ -315,9 +292,9 @@ def transformPoints(lats, lons, hgts, old_proj, new_proj): return np.stack(res, axis=-1).T -def build_cube(xpts, ypts, zpts, model_crs, pts_crs, interpolators): +def _build_cube(xpts, ypts, zpts, model_crs, pts_crs, interpolators): """ - Iterate over interpolators and build a cube + Iterate over interpolators and build a cube using Zenith """ # Create a regular 2D grid xx, yy = np.meshgrid(xpts, ypts) @@ -363,12 +340,12 @@ def build_cube(xpts, ypts, zpts, model_crs, pts_crs, interpolators): return outputArrs -def build_cube_ray( +def _build_cube_ray( xpts, ypts, zpts, ref_time, orbit_file, look_dir, model_crs, pts_crs, interpolators, outputArrs=None ): """ - Iterate over interpolators and build a cube + Iterate over interpolators and build a cube using raytracing """ # Some constants for this module diff --git a/tools/RAiDER/processWM.py b/tools/RAiDER/processWM.py index 8656d2f1a..11c6b9a25 100755 --- a/tools/RAiDER/processWM.py +++ b/tools/RAiDER/processWM.py @@ -24,9 +24,8 @@ def prepareWeatherModel( download_only: bool=False, makePlots: bool=False, force_download: bool=False, - ): - """ - Parse inputs to download and prepare a weather model grid for interpolation + ) -> str: + """Parse inputs to download and prepare a weather model grid for interpolation Args: weather_model: WeatherModel - instantiated weather model object @@ -136,22 +135,7 @@ def prepareWeatherModel( del weather_model -def checkBounds(weather_model, outLats, outLons): - '''Check the bounds of a weather model''' - ds = xr.load_dataset(weather_model.files[0]) # TODO: xr is undefined - coords = ds.coords # coords is dict-like - keys = [k for k in coords.keys()] - xc = coords[keys[0]] - yc = coords[keys[1]] - lat_bounds = [yc.min(), yc.max()] - lon_bounds = [xc.min(), xc.max()] - self_extent = lat_bounds + lon_bounds - in_extent = weather_model._getExtent(outLats, outLons) - - return in_extent, self_extent - - -def weather_model_debug( +def _weather_model_debug( los, lats, lons, From 4f68b465a703e4197175edbca1b14857fd3e2497 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 14:15:39 -0900 Subject: [PATCH 73/82] Autodoc entire RAiDER API --- docs/reference.md | 13 ++++--------- environment.yml | 7 ++++++- mkdocs.yml | 12 ++++++++---- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/reference.md b/docs/reference.md index 7ee3c3b27..eca8767eb 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,10 +1,5 @@ -Python library API +# RAiDER API Reference -### Core Datastructures -TBD - -### Weather Model Access -[`prepareWeatherModel`](reference_pwm.md) - -### Delay calculation -[`tropo_delay`](reference_td.md) +::: RAiDER + options: + show_submodules: true diff --git a/environment.yml b/environment.yml index b426a5973..5fcdc3fa5 100644 --- a/environment.yml +++ b/environment.yml @@ -47,7 +47,12 @@ dependencies: - pytest-timeout - pytest-console-scripts - setuptools_scm >=6.2 - + # For docs website + - mkdocs + - mkdocstrings + - mkdocstrings-python + - mkdocs-material + - mkdocs-material-extensions # For RAiDER-docs - jupyterlab - jupyter_contrib_nbextensions diff --git a/mkdocs.yml b/mkdocs.yml index f27b21c3c..c09ec90b7 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,17 +1,21 @@ site_name: RAiDER API Documentation +site_description: repo_url: https://github.com/dbekaert/RAiDER - +repo_name: RAiDER nav: - Home page: index.md - Tutorials: tutorials.md - - API Reference: reference.md - Weather Model Access: reference_pwm.md - Delay Calculation: reference_td.md - + - API Reference: reference.md + theme: name: material plugins: - - mkdocstrings \ No newline at end of file + - mkdocstrings: + handlers: + python: + inherited_members: true From 69ddee49bce5f4472b54e75b9c7ccbcc3d05a22c Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 15:29:54 -0900 Subject: [PATCH 74/82] auto generate entire API reference --- docs/macros.py | 7 +++++++ docs/reference.md | 2 +- environment.yml | 1 + mkdocs.yml | 2 ++ 4 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 docs/macros.py diff --git a/docs/macros.py b/docs/macros.py new file mode 100644 index 000000000..5fc217c0b --- /dev/null +++ b/docs/macros.py @@ -0,0 +1,7 @@ +def define_env(env): + """Macros Hook""" + + @env.macro + def raider_version(): + import RAiDER + return RAiDER.__version__ diff --git a/docs/reference.md b/docs/reference.md index eca8767eb..da94b41f2 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -1,4 +1,4 @@ -# RAiDER API Reference +# RAiDER *v{{ raider_version() }}* API Reference ::: RAiDER options: diff --git a/environment.yml b/environment.yml index 5fcdc3fa5..b018451be 100644 --- a/environment.yml +++ b/environment.yml @@ -51,6 +51,7 @@ dependencies: - mkdocs - mkdocstrings - mkdocstrings-python + - mkdocs-macros-plugin - mkdocs-material - mkdocs-material-extensions # For RAiDER-docs diff --git a/mkdocs.yml b/mkdocs.yml index c09ec90b7..3b51e5605 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -15,6 +15,8 @@ theme: name: material plugins: + - macros: + module_name: docs/macros - mkdocstrings: handlers: python: From d97e467d660efdd12f91084068140c402d9933f8 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 15:38:38 -0900 Subject: [PATCH 75/82] Reorg --- README.md | 4 ++-- WeatherModels.md => docs/WeatherModels.md | 0 docs/index.md | 8 +------- 3 files changed, 3 insertions(+), 9 deletions(-) rename WeatherModels.md => docs/WeatherModels.md (100%) mode change 100644 => 120000 docs/index.md diff --git a/README.md b/README.md index a401a581c..3f09e1f7e 100755 --- a/README.md +++ b/README.md @@ -75,12 +75,12 @@ python -m pip install -e . ``` ------ ## 2. Setup of third party weather model access -RAiDER has the ability to download weather models from third-parties; some of which require license agreements. See [here](WeatherModels.md) for details. +RAiDER has the ability to download weather models from third-parties; some of which require license agreements. See [here](https://github.com/dbekaert/RAiDER/blob/dev/docs/WeatherModels.md) for details. ------ ## 3. Running RAiDER and Documentation For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs). -We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/master/CONTRIBUTING.md) for instructions). +We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions). ``` raiderDelay.py -h ``` provides a help menu and list of example commands to get started. The RAiDER scripts are highly modulized in Python and allows for building your own processing workflow. diff --git a/WeatherModels.md b/docs/WeatherModels.md similarity index 100% rename from WeatherModels.md rename to docs/WeatherModels.md diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 87a8dedb3..000000000 --- a/docs/index.md +++ /dev/null @@ -1,7 +0,0 @@ -# Welcome to RAiDER - -This site contains the project documentation for the -[`RAiDER`](https://github.com/dbekaert/RAiDER) project. -Its aims are to provide a uniform interface for: -1) access to different weather models, which can be used to -2) calculate tropospheric delays for SAR/InSAR. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md new file mode 120000 index 000000000..32d46ee88 --- /dev/null +++ b/docs/index.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file From 76a77082fd021b220fdbcade2c3c716e6576bfe7 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 15:48:28 -0900 Subject: [PATCH 76/82] update README --- README.md | 55 ++++++++++++------- .../Installing_from_source.md | 0 2 files changed, 36 insertions(+), 19 deletions(-) mode change 100755 => 100644 README.md rename Installing_from_source.md => docs/Installing_from_source.md (100%) diff --git a/README.md b/README.md old mode 100755 new mode 100644 index 3f09e1f7e..1a00261b8 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # RAiDER + Raytracing Atmospheric Delay Estimation for RADAR [![Language](https://img.shields.io/badge/python-3.7%2B-blue.svg)](https://www.python.org/) @@ -13,15 +14,18 @@ Copyright (c) 2019-2022, California Institute of Technology ("Caltech"). All rig THIS IS RESEARCH CODE PROVIDED TO YOU "AS IS" WITH NO WARRANTIES OF CORRECTNESS. USE AT YOUR OWN RISK. ## Contents -- [1. Getting Started](#1-getting-started) - - [Installing With Conda](#installing-with-conda) - - [Using the Docker Image](#using-the-docker-image) - - [Installing from Source](#installing-from-source) -- [2. Setup of third party weather model access](#2-setup-of-third-party-weather-model-access) -- [3. Running RAiDER and Documentation](#3-running-raider-and-documentation) -- [4. Citing](#4-citation) -- [5. Contributors](#5-contributors) + +1. [Getting Started](#1-getting-started) + - [Installing With Conda](#installing-with-conda) + - [Using the Docker Image](#using-the-docker-image) + - [Installing from Source](#installing-from-source) +2. [Setup of third party weather model access](#2-setup-of-third-party-weather-model-access) +3. [Running RAiDER and Documentation](#3-running-raider-and-documentation) +4. [Citing](#4-citation) +5. [Development](#5-development) + - [Contributors](#contributors) ------ + ## 1. Getting Started RAiDER has been tested on the following systems: @@ -31,6 +35,7 @@ RAiDER has been tested on the following systems: RAiDER does **not** currently run on arm64 processors on Mac. We will update this note once the build becomes available. ### Installing With Conda + RAiDER is available on [conda-forge](https://anaconda.org/conda-forge/raider). __[Conda](https://docs.conda.io/en/latest/index.html)__ is a cross-platform way to use Python that allows you to setup and use "virtual environments." These can help to keep dependencies for different sets of code separate. We recommend using [Miniforge](https://github.com/conda-forge/miniforge), a conda environment manager that uses conda-forge as its default code repo. Alternatively,see __[here](https://docs.anaconda.com/anaconda/install/)__ for help installing Anaconda and __[here](https://docs.conda.io/en/latest/miniconda.html)__ for installing Miniconda. Installing RAiDER: @@ -40,6 +45,7 @@ conda activate RAiDER ``` ### Using the Docker image + RAiDER provides a [docker container image](https://docs.docker.com/get-started/) with all the necessary dependencies pre-installed. To get the latest released version: ``` docker pull ghcr.io/dbekaert/raider:latest @@ -64,32 +70,43 @@ cd work ``` For more docker run options, see: . -### Installing from source -You can also install RAiDER directly from source. Doing so is recommended for those who would like to [contribute to the source code](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md), which we heartily encourage! For more details on installing from source see [here](https://github.com/dbekaert/RAiDER/blob/dev/Installing_from_source.md). -``` -git clone https://github.com/dbekaert/RAiDER.git -cd RAiDER -conda create -f environment.yml -conda activate RAiDER -python -m pip install -e . -``` + ------ ## 2. Setup of third party weather model access + RAiDER has the ability to download weather models from third-parties; some of which require license agreements. See [here](https://github.com/dbekaert/RAiDER/blob/dev/docs/WeatherModels.md) for details. ------ ## 3. Running RAiDER and Documentation + For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs). We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions). ``` raiderDelay.py -h ``` provides a help menu and list of example commands to get started. -The RAiDER scripts are highly modulized in Python and allows for building your own processing workflow. +The RAiDER scripts are highly modularized in Python and allows for building your own processing workflow. ------ ## 4. Citation TODO ------ -## 5. Contributors +## 5. Development + +Contributions are welcome and heartily encourage! See our [contributing guide](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md). + +### Development install +For development, we recommend installing directly from source. +``` +git clone https://github.com/dbekaert/RAiDER.git +cd RAiDER +conda create -f environment.yml +conda activate RAiDER +python -m pip install -e . +``` +For more details on installing from source see [here](https://github.com/dbekaert/RAiDER/blob/dev/docs/Installing_from_source.md). + +------ +### Contributors + * David Bekaert * Jeremy Maurer * Raymond Hogenson diff --git a/Installing_from_source.md b/docs/Installing_from_source.md similarity index 100% rename from Installing_from_source.md rename to docs/Installing_from_source.md From b1eac43ec48075f5bde9c6000d451675b948156f Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 15:54:38 -0900 Subject: [PATCH 77/82] use RAiDER-docs README as tutorial page --- docs/macros.py | 12 +++++++++++- docs/tutorials.md | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/docs/macros.py b/docs/macros.py index 5fc217c0b..dc8fd6674 100644 --- a/docs/macros.py +++ b/docs/macros.py @@ -1,7 +1,17 @@ +import requests + +import RAiDER + + def define_env(env): """Macros Hook""" @env.macro def raider_version(): - import RAiDER return RAiDER.__version__ + + @env.macro + def get_content(url): + response = requests.get(url) + response.raise_for_status() + return response.content.decode() diff --git a/docs/tutorials.md b/docs/tutorials.md index a6f12a32e..a56121378 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -1,3 +1,3 @@ ### Tutorials -For detailed tutorials, visit the [RAiDER-docs](https://github.com/dbekaert/RAiDER-docs) repo. \ No newline at end of file +{{ get_content('https://raw.githubusercontent.com/dbekaert/RAiDER-docs/main/README.md') }} From 563c9f98e0e5861552fd0dde1886ac43d043b49e Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 16:06:16 -0900 Subject: [PATCH 78/82] Add tutorials --- mkdocs.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mkdocs.yml b/mkdocs.yml index 3b51e5605..ba5cc4b4b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -5,7 +5,12 @@ repo_name: RAiDER nav: - Home page: index.md - - Tutorials: tutorials.md + - Tutorials: + - Overview: tutorials.md + - RAiDER tutorial: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/RAiDER_tutorial/RAiDER_tutorial.ipynb" target="_blank + - raiderStats tutorial: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/raiderStats/raiderStats_tutorial.ipynb" target="_blank + - Downloading GNSS tropospheric delays: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/raiderDownloadGNSS/raiderDownloadGNSS_tutorial.ipynb" target="_blank + - GNSS delay manipulation with Pandas: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/Pandas_tutorial/Pandas_tutorial.ipynb" target="_blank - Weather Model Access: reference_pwm.md - Delay Calculation: reference_td.md - API Reference: reference.md @@ -15,6 +20,7 @@ theme: name: material plugins: + - search - macros: module_name: docs/macros - mkdocstrings: From 338a9d4f15c25fad1b435f9e3d57707da485f879 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 16:24:27 -0900 Subject: [PATCH 79/82] Better describe weather models --- docs/WeatherModels.md | 75 ++++++++++++++++++++++--------------------- docs/reference_pwm.md | 2 -- mkdocs.yml | 2 +- 3 files changed, 39 insertions(+), 40 deletions(-) delete mode 100644 docs/reference_pwm.md diff --git a/docs/WeatherModels.md b/docs/WeatherModels.md index eae7cbd5b..659155da0 100644 --- a/docs/WeatherModels.md +++ b/docs/WeatherModels.md @@ -1,30 +1,28 @@ # Accessing weather model data -RAiDER has built-in support for a number of different weather models. RAiDER provides all of the interfacing to data servers required to access data for the different weather models, although some weather models require a license agreement and accounts to be set-up. Instructions for accessing data, including license-limited data, are provided below. It is the user's responsibility to accept license agreements for whatever model is desired. +RAiDER has built-in support for a number of different weather models. RAiDER provides all the interfacing to data servers required to access data for the different weather models, although some weather models require a license agreement and accounts to be set-up. Instructions for accessing data, including license-limited data, are provided below. It is the user's responsibility to accept license agreements for whatever model is desired. In addition, RAiDER provides functionality for adding additional weather models. See the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs) page on how to do this. We would love to expand the suite of supported models, and welcome any contributions. Please see the contributing guidelines or reach out through an issue ticket for help. ------ -## Contents +## 1. Usage -1. [NOAA weather models (HRRR)](#noaa-weather-models-(hrrr)) -2. [ECMWF weather models (ERA5, ERA5T, ERAI, HRES)](#ecmwf-weather-models-(era5,-era5t,-erai,-hres)) -3. [NASA weather models (GMAO, MERRA2)](#nasa-weather-models-(gmao,-merra2)) +::: RAiDER.processWM.prepareWeatherModel + options: + show_root_heading: true + heading_level: 3 -#Potential download failure:# +### Potential download failure ERA-5/ERA-I products require access to the ESA Copernicus servers. GMAO and MERRA-2 products require access to the NASA Earthdata servers. If you are unable to download products, ensure that you have registered and have downloaded the public API key, and accepted/added the license/application for type of product you wish to download as detailed below. ------ -## 1. NOAA weather models (HRRR) +## 2. NOAA weather models (HRRR) High-resolution rapid refresh (HRRR) weather model data products are generated by __[NOAA](https://rapidrefresh.noaa.gov/hrrr/)__ for the coninental US (CONUS) but not archived beyond three days. However a public __[archive](home.chpc.utah.edu/~u0553130/Brian_Blaylock/hrrr_FAQ.html)__ is available at the University of Utah. This archive does not require a license agreement. This model has the highest spatial resolution available in RAiDER, with a horizontal grid spacing of about 3 km, and is provided in a Lambert conformal conic projection. -[return to table of content](#contents) - - ------ -## 2. ECMWF weather models (ERA5, ERA5T, ERAI, HRES) +## 3. ECMWF weather models (ERA5, ERA5T, ERAI, HRES) The Copernicus Climate Data Store (CDS) provides access to the European Centre for Medium-Range Weather Forecasts (__[ECMWF](https://www.ecmwf.int/)__) provides a number of different weather models, including ERA5 and ERA5T reanalysis models. The ECMWF provides access to both reanalysis and real-time prediction models. You can read more information about their reanalysis models __[here](https://www.ecmwf.int/en/research/climate-reanalysis)__ and real-time model __[here](https://www.ecmwf.int/en/forecasts/datasets/catalogue-ecmwf-real-time-products)__. ECMWF models are global, with horizontal resolution of about 30 km for ERA-I, ERA-5, and ERA-5T, and 6 km for Hi-RES. All of these models come in a global projection (EPSG 4326, WGS-84). @@ -34,14 +32,19 @@ The ECMWF provides access to both reanalysis and real-time prediction models. Yo 2. Confirm your email, etc. 3. Install the public API key and client as instructed __[here](https://cds.climate.copernicus.eu/api-how-to)__: - a. Copy the URL and API key from the webpage into a file in your home directory name ~/.cdsapirc - url: https://cds.climate.copernicus.eu/api/v2 - key: your_key_here - __**Note**: the key represents the API key obtained upon the registration of CDS API, and should be replaced with the user's own information.__ - - b. Install the CDS API using pip: - pip install cdsapi - ___**Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER___ + a. Copy the URL and API key from the webpage into a file in your home directory name `~/.cdsapirc` + + url: https://cds.climate.copernicus.eu/api/v2 + key: your_key_here + + **Note**: the key represents the API key obtained upon the registration of CDS API, and should be replaced with the user's own information. + + b. Install the CDS API using pip + + pip install cdsapi + + **Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER + 4. You must accept the [license](https://cds.climate.copernicus.eu/cdsapp/#!/terms/licence-to-use-copernicus-products) for each product you wish to download. @@ -53,22 +56,23 @@ ECMWF requires a license agreement to be able to access, download, and use their 2. Confirm your email, etc. 3. Install the public API key and client as instructed __[here](https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key)__: - a. Copy the URL and API key from the webpage into a file in your home directory name ~/.ecmwfapirc - { - "url" : "https://api.ecmwf.int/v1", - "key" : your key here, - "email" : your email here - } + a. Copy the URL and API key from the webpage into a file in your home directory name `~/.ecmwfapirc` + + { + "url" : "https://api.ecmwf.int/v1", + "key" : your key here, + "email" : your email here + } - __**Note**: the email that is used to register the user account, and the key represents the API key obtained upon the registration of ECMWF API, and should be replaced with the user's own information.__ + **Note**: the email that is used to register the user account, and the key represents the API key obtained upon the registration of ECMWF API, and should be replaced with the user's own information. - b. Install the ECMWF API using pip: - ```pip install ecmwf-api-client``` - ___**Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER___ + b. Install the ECMWF API using pip: -[return to table of content](#contents) + pip install ecmwf-api-client` -### 3. NASA weather models (GMAO, MERRA2) + **Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER + +## 4. NASA weather models (GMAO, MERRA2) 1. The Global Modeling and Assimilation Office (__[GMAO](https://www.nccs.nasa.gov/services/data-collections/coupled-products/geos5-forecast#:~:text=The%20Global%20Modeling%20and%20Assimilation,near%2Dreal%2Dtime%20production.)__) at NASA generates reanalysis weather models. GMAO products can also be accessed without a license agreement through the pyDAP interface implemented in RAiDER. GMAO has a horizontal grid spacing of approximately 33 km, and its projection is EPSG code 4326 (WGS-84). @@ -85,7 +89,7 @@ Reference: __[The Modern-Era Retrospective Analysis for Research and Application login password - __**Note**: the username and password represent the user's username and password.__ + **Note**: the username and password represent the user's username and password. 4. Add the application `NASA GESDISC DATA ARCHIVE` by clicking on the `Applications->Authorized Apps` on the menu after logging into your Earthdata profile, and then scrolling down to the application `NASA GESDISC DATA ARCHIVE` to approve it. _This seems not required for GMAO for now, but recommended to do so for all OpenDAP-based weather models._ 5. Install the OpenDAP using pip: @@ -93,9 +97,6 @@ Reference: __[The Modern-Era Retrospective Analysis for Research and Application pip install pydap==3.2.1 - ___**Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER___ + **Note**: this step has been included in the conda install of RAiDER, thus can be omitted if one uses the recommended conda install of RAiDER - ___**Note**: PyDAP v3.2.1 is required for now (thus specified in the above pip install command) because the latest v3.2.2 (as of now) has a known [bug](https://colab.research.google.com/drive/1f_ss1Oa3VzgAOd_p8sgekdnLVE5NW6s5) in accessing and slicing the GMAO data. This bug is expected to be fixed in newer versions of PyDAP.___ - -[return to table of content](#contents) - + **Note**: PyDAP v3.2.1 is required for now (thus specified in the above pip install command) because the latest v3.2.2 (as of now) has a known [bug](https://colab.research.google.com/drive/1f_ss1Oa3VzgAOd_p8sgekdnLVE5NW6s5) in accessing and slicing the GMAO data. This bug is expected to be fixed in newer versions of PyDAP. diff --git a/docs/reference_pwm.md b/docs/reference_pwm.md deleted file mode 100644 index 2cc8ddfe9..000000000 --- a/docs/reference_pwm.md +++ /dev/null @@ -1,2 +0,0 @@ - -::: RAiDER.processWM \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index ba5cc4b4b..1d799865b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -11,7 +11,7 @@ nav: - raiderStats tutorial: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/raiderStats/raiderStats_tutorial.ipynb" target="_blank - Downloading GNSS tropospheric delays: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/raiderDownloadGNSS/raiderDownloadGNSS_tutorial.ipynb" target="_blank - GNSS delay manipulation with Pandas: https://nbviewer.org/github/dbekaert/RAiDER-docs/blob/main/notebooks/Pandas_tutorial/Pandas_tutorial.ipynb" target="_blank - - Weather Model Access: reference_pwm.md + - Weather Model Access: WeatherModels.md - Delay Calculation: reference_td.md - API Reference: reference.md From 9813b511b8357d93dc09508eb8ed727a741f92c6 Mon Sep 17 00:00:00 2001 From: Joseph H Kennedy Date: Mon, 19 Dec 2022 16:25:19 -0900 Subject: [PATCH 80/82] Fix typo --- docs/WeatherModels.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/WeatherModels.md b/docs/WeatherModels.md index 659155da0..ddbc87eea 100644 --- a/docs/WeatherModels.md +++ b/docs/WeatherModels.md @@ -18,7 +18,7 @@ ERA-5/ERA-I products require access to the ESA Copernicus servers. GMAO and MERR ------ ## 2. NOAA weather models (HRRR) -High-resolution rapid refresh (HRRR) weather model data products are generated by __[NOAA](https://rapidrefresh.noaa.gov/hrrr/)__ for the coninental US (CONUS) but not archived beyond three days. However a public __[archive](home.chpc.utah.edu/~u0553130/Brian_Blaylock/hrrr_FAQ.html)__ is available at the University of Utah. This archive does not require a license agreement. This model has the highest spatial resolution available in RAiDER, with a horizontal grid spacing of about 3 km, and is provided in a Lambert conformal conic projection. +High-resolution rapid refresh (HRRR) weather model data products are generated by __[NOAA](https://rapidrefresh.noaa.gov/hrrr/)__ for the coninental US (CONUS) but not archived beyond three days. However, a public __[archive](home.chpc.utah.edu/~u0553130/Brian_Blaylock/hrrr_FAQ.html)__ is available at the University of Utah. This archive does not require a license agreement. This model has the highest spatial resolution available in RAiDER, with a horizontal grid spacing of about 3 km, and is provided in a Lambert conformal conic projection. ------ @@ -52,7 +52,7 @@ The ECMWF provides access to both reanalysis and real-time prediction models. Yo ECMWF requires a license agreement to be able to access, download, and use their products. Instructions for completing this process is below. -1. Create an account on the ECMWF servers __[here](https://accounts.ecmwf.int/auth/realms/ecmwf/protocol/openid-connect/auth?response_type=code&scope=openid%20email&client_id=apache-www&state=sBYlpcTRPhat8d6uuM9swLCxuP8&redirect_uri=https%3A%2F%2Fwww.ecmwf.int%2Foidc.cgi&nonce=RyEzBUy4m6oo_HxRQEmJxbc5jrKY4KFZd1Usgi8cpnM)__. The ERA-I model is open-access, while HRES requires a special liscence agreement. +1. Create an account on the ECMWF servers __[here](https://accounts.ecmwf.int/auth/realms/ecmwf/protocol/openid-connect/auth?response_type=code&scope=openid%20email&client_id=apache-www&state=sBYlpcTRPhat8d6uuM9swLCxuP8&redirect_uri=https%3A%2F%2Fwww.ecmwf.int%2Foidc.cgi&nonce=RyEzBUy4m6oo_HxRQEmJxbc5jrKY4KFZd1Usgi8cpnM)__. The ERA-I model is open-access, while HRES requires a special licence agreement. 2. Confirm your email, etc. 3. Install the public API key and client as instructed __[here](https://confluence.ecmwf.int/display/WEBAPI/Access+ECMWF+Public+Datasets#AccessECMWFPublicDatasets-key)__: From 6a9bb1a523b43b73d8aa7bc5035eea16d051ad3d Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 19 Dec 2022 20:53:45 -0600 Subject: [PATCH 81/82] get rid of np dot float --- tools/RAiDER/cli/statsPlot.py | 5 ++--- tools/RAiDER/constants.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/tools/RAiDER/cli/statsPlot.py b/tools/RAiDER/cli/statsPlot.py index 9b0aa45ff..ff1b2866f 100755 --- a/tools/RAiDER/cli/statsPlot.py +++ b/tools/RAiDER/cli/statsPlot.py @@ -181,7 +181,7 @@ def convert_SI(val, unit_in, unit_out): # adjust if input isn't datetime, and assume it to be part of workflow # e.g. sigZTD filter, already extracted datetime object try: - return eval('val.apply(pd.to_datetime).dt.{}.astype(np.float).astype("Int32")'.format(unit_out)) + return eval('val.apply(pd.to_datetime).dt.{}.astype(float).astype("Int32")'.format(unit_out)) except BaseException: # TODO: Which error(s)? return val @@ -987,8 +987,7 @@ def create_DF(self): # estimate central longitude lines if '--time_lines' specified if self.time_lines and 'Datetime' in self.df.keys(): - self.df['Date_hr'] = self.df['Datetime'].dt.hour.astype( - np.float).astype("Int32") + self.df['Date_hr'] = self.df['Datetime'].dt.hour.astype(float).astype("Int32") # get list of unique times all_hrs = sorted(set(self.df['Date_hr'])) diff --git a/tools/RAiDER/constants.py b/tools/RAiDER/constants.py index 6f3b61575..7d7c5cbe7 100644 --- a/tools/RAiDER/constants.py +++ b/tools/RAiDER/constants.py @@ -11,7 +11,7 @@ _ZREF = np.float64(15000) # maximum requierd height _STEP = np.float64(15.0) # integration step size in meters -_CUBE_SPACING_IN_M = np.float(2000) # Horizontal spacing of cube +_CUBE_SPACING_IN_M = float(2000) # Horizontal spacing of cube _g0 = np.float64(9.80665) _RE = np.float64(6371008.7714) From c978ed9df76bae43a5b34b49004280bada5c6756 Mon Sep 17 00:00:00 2001 From: Jeremy Maurer Date: Mon, 19 Dec 2022 21:42:01 -0600 Subject: [PATCH 82/82] update changelog --- CHANGELOG.md | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adbb957cd..3919476ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,20 +6,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [PEP 440](https://www.python.org/dev/peps/pep-0440/) and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [0.3.2] -Added documentation for the Python library interface. -Added some unit tests. -Fixed some bugs and tweaked the CLI. - -## [0.3.1] -RAiDER package was refactored to use a __main__ file to allow calls to different functionality. -The default is `calcDelays` which maintains the original functionality of calling `raider.py`. -`raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ...` -`raider.py ++calcDelaysGUNW GUNWFILE` is enabled as a placeholder only. -Fix bugs to ensure working RAiDER_tutorial notebook -Upgraded ISCE3 to `>=v0.9.0` to fix a conda build issue as described in [#425](https://github.com/dbekaert/RAiDER/issues/425) -Allow user to specify --download_only or download_only=True in the configure file - ## [0.3.0] RAiDER package was refactored to expose the main functionality as a Python library, including the `prepareWeatherModel` and `tropo_delay` functions, as well as anciliarry functions needed for defining AOIs, look vectors, etc. @@ -27,10 +13,17 @@ and `tropo_delay` functions, as well as anciliarry functions needed for defining ### New/Updated Features + Python library access to main functions for accessing weather model data and calculating delays + Slant delay calculation through projection is supported for cubes with orbit files ++ Upgrade dem-stitcher to [`>=2.3.1`](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/blob/dev/CHANGELOG.md#231) so that the updated urls for the GLO-30 DEM are used. ++ `raider.py ++calcDelaysGUNW GUNWFILE` is enabled as a placeholder only. ++ Upgraded ISCE3 to `>=v0.9.0` to fix a conda build issue as described in [#425](https://github.com/dbekaert/RAiDER/issues/425) ++ Allow user to specify --download_only or download_only=True in the configure file ++ Added documentation for the Python library interface. ++ Added some unit tests. ++ Fixed some bugs and tweaked the CLI. ++ Added unit tests, docstrings, initial API reference ++ __main__ file to allow calls to different functionality. `raider.py ++process downloadGNSS ...` can now perform the functionality of `raiderDownloadGNSS.py ... -## [0.2.1] -* Upgrade dem-stitcher to [`>=2.3.1`](https://github.com/ACCESS-Cloud-Based-InSAR/dem-stitcher/blob/dev/CHANGELOG.md#231) so that the updated urls for the GLO-30 DEM are used. ## [0.2.0]