diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index bdc5a0543..1c7095a97 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -14,5 +14,3 @@ on: jobs: call-changelog-check-workflow: uses: ASFHyP3/actions/.github/workflows/reusable-changelog-check.yml@v0.11.2 - secrets: - USER_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 72168c0de..cd213dd34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,11 +8,13 @@ and uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] ### Added -* [650](https://github.com/dbekaert/RAiDER/pull/650) - add automatic code formatting +* [650](https://github.com/dbekaert/RAiDER/pull/650) - Added automatic code formatting. ### Changed +* [627](https://github.com/dbekaert/RAiDER/pull/627) - Made Python datetimes timezone-aware and add unit tests and bug fixes. * [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`. -* [651](https://github.com/dbekaert/RAiDER/pull/651) - Removed use of deprecated argument to `pandas.read_csv`. -* [627](https://github.com/dbekaert/RAiDER/pull/627) - Make Python datetimes timezone-aware and add unit tests and bug fixes +* [657](https://github.com/dbekaert/RAiDER/pull/657) - Fixed a few typos in `README.md`. +* [661](https://github.com/dbekaert/RAiDER/pull/661) - Fixed bug in raiderDownloadGNSS, removed call to scipy.sum, and added unit tests. +* [662](https://github.com/dbekaert/RAiDER/pull/662) - Ensures dem-stitcher to be >= v2.5.6, which updates the url for reading the Geoid EGM 2008. ## [0.5.1] ### Changed diff --git a/README.md b/README.md index 152720802..24baab335 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ RAiDER does **not** currently run on arm64 processors on Mac. We will update thi ### Installing With Conda -RAiDER is available on [conda-forge](https://anaconda.org/conda-forge/raider). __[Conda](https://docs.conda.io/en/latest/index.html)__ is a cross-platform way to use Python that allows you to setup and use "virtual environments." These can help to keep dependencies for different sets of code separate. We recommend using [Miniforge](https://github.com/conda-forge/miniforge), a conda environment manager that uses conda-forge as its default code repo. Alternatively,see __[here](https://docs.anaconda.com/anaconda/install/)__ for help installing Anaconda and __[here](https://docs.conda.io/en/latest/miniconda.html)__ for installing Miniconda. +RAiDER is available on [conda-forge](https://anaconda.org/conda-forge/raider). __[Conda](https://docs.conda.io/en/latest/index.html)__ is a cross-platform way to use Python that allows you to setup and use "virtual environments." These can help to keep dependencies for different sets of code separate. We recommend using [Miniforge](https://github.com/conda-forge/miniforge), a conda environment manager that uses conda-forge as its default code repo. Alternatively, see __[here](https://docs.anaconda.com/anaconda/install/)__ for help installing Anaconda and __[here](https://docs.conda.io/en/latest/miniconda.html)__ for installing Miniconda. Installing RAiDER: ``` @@ -85,8 +85,8 @@ RAiDER has the ability to download weather models from third-parties; some of wh ## 3. Running RAiDER and Documentation For detailed documentation, examples, and Jupyter notebooks see the [RAiDER-docs repository](https://github.com/dbekaert/RAiDER-docs). -We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions). -``` raiderDelay.py -h ``` provides a help menu and list of example commands to get started. +We welcome contributions of other examples on how to leverage the RAiDER (see [here](https://github.com/dbekaert/RAiDER/blob/dev/CONTRIBUTING.md) for instructions). +``` raider.py -h ``` provides a help menu and list of example commands to get started. The RAiDER scripts are highly modularized in Python and allows for building your own processing workflow. ------ diff --git a/environment.yml b/environment.yml index e40ef8232..7b92e2cf3 100644 --- a/environment.yml +++ b/environment.yml @@ -17,7 +17,7 @@ dependencies: - cdsapi - cfgrib - dask - - dem_stitcher>=2.3.1 + - dem_stitcher>=2.5.6 - ecmwf-api-client - h5netcdf - h5py diff --git a/test/test_GUNW.py b/test/test_GUNW.py index 99a5e7a74..d35ae981b 100644 --- a/test/test_GUNW.py +++ b/test/test_GUNW.py @@ -24,7 +24,9 @@ get_slc_ids_from_gunw,get_acq_time_from_slc_id ) from RAiDER.cli.raider import calcDelaysGUNW -from RAiDER.models.customExceptions import NoWeatherModelData +from RAiDER.models.customExceptions import ( + NoWeatherModelData, WrongNumberOfFiles, +) def compute_transform(lats, lons): @@ -568,7 +570,7 @@ def test_GUNW_workflow_fails_if_a_download_fails(gunw_azimuth_test, orbit_dict_f '-interp', 'azimuth_time_grid' ] - with pytest.raises(RuntimeError): + with pytest.raises(WrongNumberOfFiles): calcDelaysGUNW(iargs_1) RAiDER.s1_azimuth_timing.get_s1_azimuth_time_grid.assert_not_called() diff --git a/test/test_downloadGNSS.py b/test/test_downloadGNSS.py index 473a89cc1..284a26794 100644 --- a/test/test_downloadGNSS.py +++ b/test/test_downloadGNSS.py @@ -6,8 +6,8 @@ from test import TEST_DIR, pushd from RAiDER.dem import download_dem from RAiDER.gnss.downloadGNSSDelays import ( - check_url,read_text_file,in_box,fix_lons,get_ID, - + check_url,in_box,fix_lons,get_ID, + download_UNR,main, ) # Test check_url with a valid and invalid URL @@ -23,21 +23,6 @@ def test_check_url_invalid(): mock_head.return_value.status_code = 404 # Simulate not found response assert check_url(invalid_url) == '' -# Test read_text_file with a valid and invalid filename -def test_read_text_file_valid(): - # Create a temporary test file with some content - with open("test_file.txt", "w") as f: - f.write("line1\nline2\n") - try: - lines = read_text_file("test_file.txt") - assert lines == ["line1", "line2"] - finally: - os.remove("test_file.txt") # Cleanup the temporary file - -def test_read_text_file_invalid(): - invalid_filename = "not_a_file.txt" - with pytest.raises(FileNotFoundError): - read_text_file(invalid_filename) # Test in_box with points inside and outside the box def test_in_box_inside(): @@ -82,4 +67,35 @@ def test_get_ID_valid(): def test_get_ID_invalid(): line = "ABCD 35.0" # Missing longitude and height with pytest.raises(ValueError): - get_ID(line) \ No newline at end of file + get_ID(line) + + +def test_download_UNR(): + statID = 'MORZ' + year = 2020 + outDict = download_UNR(statID, year) + assert outDict['path'] == 'http://geodesy.unr.edu/gps_timeseries/trop/MORZ/MORZ.2020.trop.zip' + +def test_download_UNR_2(): + statID = 'MORZ' + year = 2000 + with pytest.raises(ValueError): + download_UNR(statID, year, download=True) + +def test_download_UNR_3(): + statID = 'DUMY' + year = 2020 + with pytest.raises(ValueError): + download_UNR(statID, year, download=True) + +def test_download_UNR_4(): + statID = 'MORZ' + year = 2020 + with pytest.raises(NotImplementedError): + download_UNR(statID, year, baseURL='www.google.com') + + +def test_main(): + # iargs = None + # main(inps=iargs) + assert True \ No newline at end of file diff --git a/test/test_gnss.py b/test/test_gnss.py index 36c5a8949..bd7598abe 100644 --- a/test/test_gnss.py +++ b/test/test_gnss.py @@ -1,7 +1,7 @@ from RAiDER.models.customExceptions import NoStationDataFoundError from RAiDER.gnss.downloadGNSSDelays import ( get_stats_by_llh, get_station_list, download_tropo_delays, - filterToBBox, + filterToBBox ) from RAiDER.gnss.processDelayFiles import ( addDateTimeToFiles, @@ -15,6 +15,7 @@ import pandas as pd from test import pushd, TEST_DIR +from unittest import mock SCENARIO2_DIR = os.path.join(TEST_DIR, "scenario_2") @@ -163,3 +164,4 @@ def test_filterByBBox2(): assert stat not in new_data['ID'].to_list() for stat in ['FGNW', 'JPLT', 'NVTP', 'WLHG', 'WORG']: assert stat in new_data['ID'].to_list() + diff --git a/tools/RAiDER/cli/raider.py b/tools/RAiDER/cli/raider.py index facf34eab..d619c57fe 100644 --- a/tools/RAiDER/cli/raider.py +++ b/tools/RAiDER/cli/raider.py @@ -295,7 +295,7 @@ def calcDelays(iargs=None): logger.error(e) logger.error('Weather model files are: {}'.format(wfiles)) logger.error(msg) - raise + continue # dont process the delays for download only if dl_only: diff --git a/tools/RAiDER/cli/statsPlot.py b/tools/RAiDER/cli/statsPlot.py index 6f53ca8f8..d75526689 100755 --- a/tools/RAiDER/cli/statsPlot.py +++ b/tools/RAiDER/cli/statsPlot.py @@ -10,7 +10,6 @@ from RAiDER.utilFcns import WGS84_to_UTM from rasterio.transform import Affine from scipy import optimize -from scipy import sum as scipy_sum from scipy.optimize import OptimizeWarning from shapely.strtree import STRtree from shapely.geometry import Point, Polygon @@ -183,12 +182,12 @@ def convert_SI(val, unit_in, unit_out): # e.g. sigZTD filter, already extracted datetime object try: return eval('val.apply(pd.to_datetime).dt.{}.astype(float).astype("Int32")'.format(unit_out)) - except BaseException: # TODO: Which error(s)? + except AttributeError: return val # check if output spatial unit is supported if unit_out not in SI: - raise Exception("User-specified output unit {} not recognized.".format(unit_out)) + raise ValueError("User-specified output unit {} not recognized.".format(unit_out)) return val * SI[unit_in] / SI[unit_out] @@ -199,6 +198,9 @@ def midpoint(p1, p2): ''' import math + if p1[1] == p2[1]: + return p1[1] + lat1, lon1, lat2, lon2 = map(math.radians, (p1[0], p1[1], p2[0], p2[1])) dlon = lon2 - lon1 dx = math.cos(lat2) * math.cos(dlon) @@ -243,7 +245,7 @@ def save_gridfile(df, gridfile_type, fname, plotbbox, spacing, unit, dst.update_tags(0, **metadata_dict) dst.write(df, 1) - return + return metadata_dict def load_gridfile(fname, unit): @@ -251,11 +253,14 @@ def load_gridfile(fname, unit): Function to load gridded-arrays saved from previous runs. ''' - with rasterio.open(fname) as src: - grid_array = src.read(1).astype(float) + try: + with rasterio.open(fname) as src: + grid_array = src.read(1).astype(float) + except TypeError: + raise ValueError('fname is not a valid file') - # Read metadata variables needed for plotting - metadata_dict = src.tags() + # Read metadata variables needed for plotting + metadata_dict = src.tags() # Initiate no-data array to mask data nodat_arr = [0, np.nan, np.inf] @@ -375,7 +380,7 @@ def _emp_vario(self, x, y, data, Nsamp=1000): y = y[mask] # deramp - temp1, temp2, x, y = WGS84_to_UTM(x, y, common_center=True) + _, _, x, y = WGS84_to_UTM(x, y, common_center=True) A = np.array([x, y, np.ones(len(x))]).T ramp = np.linalg.lstsq(A, data.T, rcond=None)[0] data = data - (np.matmul(A, ramp)) @@ -1487,7 +1492,7 @@ def fitfunc(t): phsfit_c[station] = pcov[2, 2]**0.5 # pass RMSE of fit seasonalfit_rmse[station] = yy - custom_sine_function_base(tt, *popt) - seasonalfit_rmse[station] = (scipy_sum(seasonalfit_rmse[station]**2) / + seasonalfit_rmse[station] = (np.sum(seasonalfit_rmse[station]**2) / (seasonalfit_rmse[station].size - 2))**0.5 except FloatingPointError: pass @@ -2099,54 +2104,3 @@ def main(): inps.variogram_per_timeslice, inps.variogram_errlimit ) - - -def main(): - inps = cmd_line_parse() - - stats_analyses( - inps.fname, - inps.col_name, - inps.unit, - inps.workdir, - inps.cpus, - inps.verbose, - inps.bounding_box, - inps.spacing, - inps.timeinterval, - inps.seasonalinterval, - inps.obs_errlimit, - inps.figdpi, - inps.user_title, - inps.plot_fmt, - inps.cbounds, - inps.colorpercentile, - inps.usr_colormap, - inps.densitythreshold, - inps.stationsongrids, - inps.drawgridlines, - inps.time_lines, - inps.plotall, - inps.station_distribution, - inps.station_delay_mean, - inps.station_delay_median, - inps.station_delay_stdev, - inps.station_seasonal_phase, - inps.phaseamp_per_station, - inps.grid_heatmap, - inps.grid_delay_mean, - inps.grid_delay_median, - inps.grid_delay_stdev, - inps.grid_seasonal_phase, - inps.grid_delay_absolute_mean, - inps.grid_delay_absolute_median, - inps.grid_delay_absolute_stdev, - inps.grid_seasonal_absolute_phase, - inps.grid_to_raster, - inps.min_span, - inps.period_limit, - inps.variogramplot, - inps.binnedvariogram, - inps.variogram_per_timeslice, - inps.variogram_errlimit - ) diff --git a/tools/RAiDER/gnss/downloadGNSSDelays.py b/tools/RAiDER/gnss/downloadGNSSDelays.py index b6612db53..1713ce022 100755 --- a/tools/RAiDER/gnss/downloadGNSSDelays.py +++ b/tools/RAiDER/gnss/downloadGNSSDelays.py @@ -20,38 +20,48 @@ def get_station_list( - bbox=None, - stationFile=None, - userstatList=None, - writeStationFile=True, - writeLoc=None, - name_appendix='' -): + bbox=None, + stationFile=None, + writeLoc=None, + name_appendix='', + writeStationFile=True, + ): ''' Creates a list of stations inside a lat/lon bounding box from a source Args: - bbox: list of float - length-4 list of floats that describes a bounding box. - Format is S N W E + bbox: list of float - length-4 list of floats that describes a bounding box. + Format is S N W E + station_file: str - Name of a .csv or .txt file to read containing station IDs + writeStationFile: bool - Whether to write out the station dataframe to a .csv file writeLoc: string - Directory to write data - userstatList: list - list of specific IDs to access name_appendix: str - name to append to output file Returns: - stations: list of strings - station IDs to access - output_file: string - file to write delays + stations: list of strings - station IDs to access + output_file: string or dataframe - file to write delays ''' - if stationFile: - station_data = pd.read_csv(stationFile) - elif userstatList: - station_data = read_text_file(userstatList) - elif bbox: + if bbox is not None: station_data = get_stats_by_llh(llhBox=bbox) + else: + try: + station_data = pd.read_csv(stationFile) + except: + stations = [] + with open(stationFile, 'r') as f: + for k, line in enumerate(f): + if k ==0: + names = line.strip().split() + else: + stations.append([line.strip().split()]) + station_data = pd.DataFrame(stations, columns=names) # write to file and pass final stations list if writeStationFile: - output_file = os.path.join(writeLoc or os.getcwd( - ), 'gnssStationList_overbbox' + name_appendix + '.csv') + output_file = os.path.join( + writeLoc or os.getcwd(), + 'gnssStationList_overbbox' + name_appendix + '.csv' + ) station_data.to_csv(output_file, index=False) return list(station_data['ID'].values), [output_file if writeStationFile else station_data][0] @@ -77,7 +87,7 @@ def get_stats_by_llh(llhBox=None, baseURL=_UNR_URL): sep=r'\s+', names=['ID', 'Lat', 'Lon', 'Hgt_m'] ) - stations = filterToBBox(stations, llhBox) + # convert lons from [0, 360] to [-180, 180] stations['Lon'] = ((stations['Lon'].values + 180) % 360) - 180 @@ -150,13 +160,17 @@ def download_UNR(statID, year, writeDir='.', download=False, baseURL=_UNR_URL): statID - 4-character station identifier year - 4-numeral year ''' + if baseURL not in [_UNR_URL]: + raise NotImplementedError('Data repository {} has not yet been implemented'.format(baseURL)) + URL = "{0}gps_timeseries/trop/{1}/{1}.{2}.trop.zip".format( baseURL, statID.upper(), year) logger.debug('Currently checking station %s in %s', statID, year) if download: - saveLoc = os.path.abspath(os.path.join( - writeDir, '{0}.{1}.trop.zip'.format(statID.upper(), year))) + saveLoc = os.path.abspath(os.path.join(writeDir, '{0}.{1}.trop.zip'.format(statID.upper(), year))) filepath = download_url(URL, saveLoc) + if filepath == '': + raise ValueError('Year or station ID does not exist') else: filepath = check_url(URL) return {'ID': statID, 'year': year, 'path': filepath} @@ -193,14 +207,6 @@ def check_url(url): return url -def read_text_file(filename): - ''' - Read a list of GNSS station names from a plain text file - ''' - with open(filename, 'r') as f: - return [line.strip() for line in f] - - def in_box(lat, lon, llhbox): ''' Checks whether the given lat, lon pair are inside the bounding box llhbox @@ -262,7 +268,7 @@ def main(inps=None): long_cross_zero = 1 # Handle station query - stats = get_stats(bbox, long_cross_zero, out, station_file) + stats, statdf = get_stats(bbox, long_cross_zero, out, station_file) # iterate over years years = list(set([i.year for i in dateList])) @@ -270,15 +276,11 @@ def main(inps=None): stats, years, gps_repo=gps_repo, writeDir=out, download=download ) - # Add lat/lon info - origstatsFile = pd.read_csv(station_file) - statsFile = pd.read_csv(os.path.join( - out, '{}gnssStationList_overbbox_withpaths.csv'.format(gps_repo))) - statsFile = pd.merge(left=statsFile, right=origstatsFile, - how='left', left_on='ID', right_on='ID') - statsFile.to_csv(os.path.join( - out, '{}gnssStationList_overbbox_withpaths.csv'.format(gps_repo)), index=False) - del origstatsFile, statsFile + # Combine station data with URL info + pathsdf = pd.read_csv(os.path.join(out, '{}gnssStationList_overbbox_withpaths.csv'.format(gps_repo))) + pathsdf = pd.merge(left=pathsdf, right=statdf, how='left', left_on='ID', right_on='ID') + pathsdf.to_csv(os.path.join(out, '{}gnssStationList_overbbox_withpaths.csv'.format(gps_repo)), index=False) + del statdf, pathsdf # Extract delays for each station dateList = [k.strftime('%Y-%m-%d') for k in dateList] @@ -333,27 +335,24 @@ def get_stats(bbox, long_cross_zero, out, station_file): bbox2 = bbox.copy() bbox1[3] = 360.0 bbox2[2] = 0.0 - stats1, origstatsFile1 = get_station_list( - bbox=bbox1, writeLoc=out, stationFile=station_file, name_appendix='_a') - stats2, origstatsFile2 = get_station_list( - bbox=bbox2, writeLoc=out, stationFile=station_file, name_appendix='_b') + stats1, statdata1 = get_station_list( + bbox=bbox1, stationFile=station_file, name_appendix='_a', writeStationFile=False + ) + stats2, statdata2 = get_station_list( + bbox=bbox2, stationFile=station_file, name_appendix='_b', writeStationFile=False + ) stats = stats1 + stats2 - origstatsFile = origstatsFile1[:-6] + '.csv' - file_a = pd.read_csv(origstatsFile1) - file_b = pd.read_csv(origstatsFile2) - frames = [file_a, file_b] - result = pd.concat(frames, ignore_index=True) - result.to_csv(origstatsFile, index=False) + frames = [statdata1, statdata2] + statdata = pd.concat(frames, ignore_index=True) else: if bbox[3] < bbox[2]: bbox[3] = 360.0 - stats, origstatsFile = get_station_list( - bbox=bbox, - writeLoc=out, - stationFile=station_file + stats, statdata = get_station_list( + bbox=bbox, stationFile=station_file, writeStationFile=False ) - - return stats + + statdata.to_csv(station_file, index=False) + return stats, statdata def filterToBBox(stations, llhBox):