Merge branch 'dev' into formatter-github-action

.github/workflows/changelog.yml

@@ -14,5 +14,3 @@ on:
 jobs:
   call-changelog-check-workflow:
     uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2
-    secrets:
-      USER_TOKEN: ${{ secrets.GITHUB_TOKEN }}

CHANGELOG.md

@@ -8,11 +8,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
 ### Added
-* [650](https://github.com/dbekaert/RAiDER/pull/650) - add automatic code formatting
+* [650](https://github.com/dbekaert/RAiDER/pull/650) - Added automatic code formatting.
 ### Changed
+* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and add unit tests and bug fixes.
 * [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
-* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`.
-* [627](https://github.com/dbekaert/RAiDER/pull/627) - Make Python datetimes timezone-aware and add unit tests and bug fixes
+* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`.
+* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests.
+* [662](https://github.com/dbekaert/RAiDER/pull/662) - Ensures dem-stitcher to be >= v2.5.6, which updates the url for reading the Geoid EGM 2008.
 
 ## [0.5.1]
 ### Changed

README.md

@@ -36,7 +36,7 @@ RAiDER does **not** currently run on arm64 processors on Mac. We will update thi
 
 ### Installing With Conda
 
-RAiDER is available on [conda-forge](https://anaconda.org/conda-forge/raider). __[Conda](https://docs.conda.io/en/latest/index.html)__ is a cross-platform way to use Python that allows you to setup and use "virtual environments." These can help to keep dependencies for different sets of code separate. We recommend using [Miniforge](https://github.com/conda-forge/miniforge), a conda environment manager that uses conda-forge as its default code repo. Alternatively,see __[here](https://docs.anaconda.com/anaconda/install/)__ for help installing Anaconda and __[here](https://docs.conda.io/en/latest/miniconda.html)__ for installing Miniconda.
+RAiDER is available on [conda-forge](https://anaconda.org/conda-forge/raider). __[Conda](https://docs.conda.io/en/latest/index.html)__ is a cross-platform way to use Python that allows you to setup and use "virtual environments." These can help to keep dependencies for different sets of code separate. We recommend using [Miniforge](https://github.com/conda-forge/miniforge), a conda environment manager that uses conda-forge as its default code repo. Alternatively, see __[here](https://docs.anaconda.com/anaconda/install/)__ for help installing Anaconda and __[here](https://docs.conda.io/en/latest/miniconda.html)__ for installing Miniconda.
 
 Installing RAiDER:
 ```
@@ -85,8 +85,8 @@ RAiDER has the ability to download weather models from third-parties; some of wh
 ## 3. Running RAiDER and Documentation
 
 For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs).
-We welcome contributions of other examples on how to leverage the RAiDER  (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).
-``` raiderDelay.py -h ``` provides a help menu and list of example commands to get started.
+We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions).
+``` raider.py -h ``` provides a help menu and list of example commands to get started.
 The RAiDER scripts are highly modularized in Python and allows for building your own processing workflow.
 
 ------

environment.yml

@@ -17,7 +17,7 @@ dependencies:
  - cdsapi
  - cfgrib
  - dask
- - dem_stitcher>=2.3.1
+ - dem_stitcher>=2.5.6
  - ecmwf-api-client
  - h5netcdf
  - h5py

test/test_GUNW.py

@@ -24,7 +24,9 @@
     get_slc_ids_from_gunw,get_acq_time_from_slc_id
 )
 from RAiDER.cli.raider import calcDelaysGUNW
-from RAiDER.models.customExceptions import NoWeatherModelData
+from RAiDER.models.customExceptions import (
+     NoWeatherModelData, WrongNumberOfFiles,
+) 
 
 
 def compute_transform(lats, lons):
@@ -568,7 +570,7 @@ def test_GUNW_workflow_fails_if_a_download_fails(gunw_azimuth_test, orbit_dict_f
                '-interp', 'azimuth_time_grid'
                ]
 
-    with pytest.raises(RuntimeError):
+    with pytest.raises(WrongNumberOfFiles):
         calcDelaysGUNW(iargs_1)
     RAiDER.s1_azimuth_timing.get_s1_azimuth_time_grid.assert_not_called()
 

test/test_downloadGNSS.py

@@ -6,8 +6,8 @@
 from test import TEST_DIR, pushd
 from RAiDER.dem import download_dem
 from RAiDER.gnss.downloadGNSSDelays import (
-    check_url,read_text_file,in_box,fix_lons,get_ID,
-
+    check_url,in_box,fix_lons,get_ID,
+    download_UNR,main,
 )
 
 # Test check_url with a valid and invalid URL
@@ -23,21 +23,6 @@ def test_check_url_invalid():
     mock_head.return_value.status_code = 404  # Simulate not found response
     assert check_url(invalid_url) == ''
 
-# Test read_text_file with a valid and invalid filename
-def test_read_text_file_valid():
-  # Create a temporary test file with some content
-  with open("test_file.txt", "w") as f:
-    f.write("line1\nline2\n")
-  try:
-    lines = read_text_file("test_file.txt")
-    assert lines == ["line1", "line2"]
-  finally:
-    os.remove("test_file.txt")  # Cleanup the temporary file
-
-def test_read_text_file_invalid():
-  invalid_filename = "not_a_file.txt"
-  with pytest.raises(FileNotFoundError):
-    read_text_file(invalid_filename)
 
 # Test in_box with points inside and outside the box
 def test_in_box_inside():
@@ -82,4 +67,35 @@ def test_get_ID_valid():
 def test_get_ID_invalid():
   line = "ABCD 35.0"  # Missing longitude and height
   with pytest.raises(ValueError):
-    get_ID(line)
+    get_ID(line)
+
+
+def test_download_UNR():
+  statID = 'MORZ'
+  year = 2020
+  outDict = download_UNR(statID, year)
+  assert outDict['path'] == 'http://geodesy.unr.edu/gps_timeseries/trop/MORZ/MORZ.2020.trop.zip'
+
+def test_download_UNR_2():
+  statID = 'MORZ'
+  year = 2000
+  with pytest.raises(ValueError):
+    download_UNR(statID, year, download=True)
+
+def test_download_UNR_3():
+  statID = 'DUMY'
+  year = 2020
+  with pytest.raises(ValueError):
+    download_UNR(statID, year, download=True)
+
+def test_download_UNR_4():
+  statID = 'MORZ'
+  year = 2020
+  with pytest.raises(NotImplementedError):
+    download_UNR(statID, year, baseURL='www.google.com')
+
+
+def test_main():
+  # iargs = None 
+  # main(inps=iargs)
+  assert True

test/test_gnss.py

@@ -1,7 +1,7 @@
 from RAiDER.models.customExceptions import NoStationDataFoundError
 from RAiDER.gnss.downloadGNSSDelays import (
     get_stats_by_llh, get_station_list, download_tropo_delays,
-    filterToBBox,
+    filterToBBox
 )
 from RAiDER.gnss.processDelayFiles import (
     addDateTimeToFiles,
@@ -15,6 +15,7 @@
 import pandas as pd
 
 from test import pushd, TEST_DIR
+from unittest import mock
 SCENARIO2_DIR = os.path.join(TEST_DIR, "scenario_2")
 
 
@@ -163,3 +164,4 @@ def test_filterByBBox2():
         assert stat not in new_data['ID'].to_list()
     for stat in ['FGNW', 'JPLT', 'NVTP', 'WLHG', 'WORG']:
         assert stat in new_data['ID'].to_list()
+

tools/RAiDER/cli/raider.py

@@ -295,7 +295,7 @@ def calcDelays(iargs=None):
                 logger.error(e)
                 logger.error('Weather model files are: {}'.format(wfiles))
                 logger.error(msg)
-                raise
+                continue
 
         # dont process the delays for download only
         if dl_only:

tools/RAiDER/cli/statsPlot.py

@@ -10,7 +10,6 @@
 from RAiDER.utilFcns import WGS84_to_UTM
 from rasterio.transform import Affine
 from scipy import optimize
-from scipy import sum as scipy_sum
 from scipy.optimize import OptimizeWarning
 from shapely.strtree import STRtree
 from shapely.geometry import Point, Polygon
@@ -183,12 +182,12 @@ def convert_SI(val, unit_in, unit_out):
         # e.g. sigZTD filter, already extracted datetime object
         try:
             return eval('val.apply(pd.to_datetime).dt.{}.astype(float).astype("Int32")'.format(unit_out))
-        except BaseException:  # TODO: Which error(s)?
+        except AttributeError:
             return val
 
     # check if output spatial unit is supported
     if unit_out not in SI:
-        raise Exception("User-specified output unit {} not recognized.".format(unit_out))
+        raise ValueError("User-specified output unit {} not recognized.".format(unit_out))
 
     return val * SI[unit_in] / SI[unit_out]
 
@@ -199,6 +198,9 @@ def midpoint(p1, p2):
     '''
     import math
 
+    if p1[1] == p2[1]:
+        return p1[1]
+
     lat1, lon1, lat2, lon2 = map(math.radians, (p1[0], p1[1], p2[0], p2[1]))
     dlon = lon2 - lon1
     dx = math.cos(lat2) * math.cos(dlon)
@@ -243,19 +245,22 @@ def save_gridfile(df, gridfile_type, fname, plotbbox, spacing, unit,
         dst.update_tags(0, **metadata_dict)
         dst.write(df, 1)
 
-    return
+    return metadata_dict
 
 
 def load_gridfile(fname, unit):
     '''
         Function to load gridded-arrays saved from previous runs.
     '''
 
-    with rasterio.open(fname) as src:
-        grid_array = src.read(1).astype(float)
+    try:
+        with rasterio.open(fname) as src:
+            grid_array = src.read(1).astype(float)
+    except TypeError:
+        raise ValueError('fname is not a valid file')
 
-        # Read metadata variables needed for plotting
-        metadata_dict = src.tags()
+    # Read metadata variables needed for plotting
+    metadata_dict = src.tags()
 
     # Initiate no-data array to mask data
     nodat_arr = [0, np.nan, np.inf]
@@ -375,7 +380,7 @@ def _emp_vario(self, x, y, data, Nsamp=1000):
             y = y[mask]
 
         # deramp
-        temp1, temp2, x, y = WGS84_to_UTM(x, y, common_center=True)
+        _, _, x, y = WGS84_to_UTM(x, y, common_center=True)
         A = np.array([x, y, np.ones(len(x))]).T
         ramp = np.linalg.lstsq(A, data.T, rcond=None)[0]
         data = data - (np.matmul(A, ramp))
@@ -1487,7 +1492,7 @@ def fitfunc(t):
                     phsfit_c[station] = pcov[2, 2]**0.5
                     # pass RMSE of fit
                     seasonalfit_rmse[station] = yy - custom_sine_function_base(tt, *popt)
-                    seasonalfit_rmse[station] = (scipy_sum(seasonalfit_rmse[station]**2) /
+                    seasonalfit_rmse[station] = (np.sum(seasonalfit_rmse[station]**2) /
                                                  (seasonalfit_rmse[station].size - 2))**0.5
                 except FloatingPointError:
                     pass
@@ -2099,54 +2104,3 @@ def main():
         inps.variogram_per_timeslice,
         inps.variogram_errlimit
     )
-
-
-def main():
-    inps = cmd_line_parse()
-
-    stats_analyses(
-        inps.fname,
-        inps.col_name,
-        inps.unit,
-        inps.workdir,
-        inps.cpus,
-        inps.verbose,
-        inps.bounding_box,
-        inps.spacing,
-        inps.timeinterval,
-        inps.seasonalinterval,
-        inps.obs_errlimit,
-        inps.figdpi,
-        inps.user_title,
-        inps.plot_fmt,
-        inps.cbounds,
-        inps.colorpercentile,
-        inps.usr_colormap,
-        inps.densitythreshold,
-        inps.stationsongrids,
-        inps.drawgridlines,
-        inps.time_lines,
-        inps.plotall,
-        inps.station_distribution,
-        inps.station_delay_mean,
-        inps.station_delay_median,
-        inps.station_delay_stdev,
-        inps.station_seasonal_phase,
-        inps.phaseamp_per_station,
-        inps.grid_heatmap,
-        inps.grid_delay_mean,
-        inps.grid_delay_median,
-        inps.grid_delay_stdev,
-        inps.grid_seasonal_phase,
-        inps.grid_delay_absolute_mean,
-        inps.grid_delay_absolute_median,
-        inps.grid_delay_absolute_stdev,
-        inps.grid_seasonal_absolute_phase,
-        inps.grid_to_raster,
-        inps.min_span,
-        inps.period_limit,
-        inps.variogramplot,
-        inps.binnedvariogram,
-        inps.variogram_per_timeslice,
-        inps.variogram_errlimit
-    )