From 9635cbe38877bafee778dbd4d101143db632892e Mon Sep 17 00:00:00 2001 From: Tyler Sutterley Date: Tue, 28 Sep 2021 22:15:34 -0700 Subject: [PATCH] test: added test for model definition files (#69) * test: added test for model definition files * fix: check that compressed attr is string * fix: use model class when retrieving files from s3 --- doc/source/user_guide/check_tide_points.md | 7 +- .../user_guide/compute_tide_corrections.md | 4 +- doc/source/user_guide/model.rst | 34 ++++- pyTMD/check_tide_points.py | 121 +++++------------- pyTMD/compute_tide_corrections.py | 1 + pyTMD/model.py | 71 ++++++---- scripts/compute_tidal_currents.py | 3 +- scripts/compute_tidal_elevations.py | 3 +- scripts/compute_tides_ICESat2_ATL03.py | 1 + scripts/compute_tides_ICESat2_ATL06.py | 1 + scripts/compute_tides_ICESat2_ATL07.py | 1 + scripts/compute_tides_ICESat2_ATL11.py | 1 + scripts/compute_tides_ICESat2_ATL12.py | 1 + scripts/compute_tides_ICESat_GLA12.py | 3 +- scripts/compute_tides_icebridge_data.py | 1 + scripts/reduce_OTIS_files.py | 1 + test/test_atlas_read.py | 54 ++++++-- test/test_download_and_read.py | 46 ++++++- test/test_fes_predict.py | 45 +++++-- test/test_perth3_read.py | 48 +++++-- 20 files changed, 286 insertions(+), 161 deletions(-) diff --git a/doc/source/user_guide/check_tide_points.md b/doc/source/user_guide/check_tide_points.md index a72625ea..c1d34c70 100644 --- a/doc/source/user_guide/check_tide_points.md +++ b/doc/source/user_guide/check_tide_points.md @@ -20,7 +20,12 @@ valid = check_tide_points(x, y, DIRECTORY=DIRECTORY, #### Keyword arguments - `DIRECTORY`: working data directory for tide models -- `MODEL`: Tide model to use in correction +- `MODEL`: Tide model to use +- `ATLAS_FORMAT`: ATLAS tide model format + * `'OTIS'` + * `'netcdf'` +- `GZIP`: Tide model files are gzip compressed +- `DEFINITION_FILE`: Tide model definition file for use - `EPSG`: input coordinate system * default: `3031` Polar Stereographic South, WGS84 - `METHOD`: interpolation method diff --git a/doc/source/user_guide/compute_tide_corrections.md b/doc/source/user_guide/compute_tide_corrections.md index afdb7003..c9efe3b2 100644 --- a/doc/source/user_guide/compute_tide_corrections.md +++ b/doc/source/user_guide/compute_tide_corrections.md @@ -22,7 +22,9 @@ tide = compute_tide_corrections(x, y, delta_time, DIRECTORY=DIRECTORY, #### Keyword arguments - `DIRECTORY`: working data directory for tide models - `MODEL`: Tide model to use in correction -- `ATLAS_FORMAT`: ATLAS tide model format (`'OTIS'`, `'netcdf'`) +- `ATLAS_FORMAT`: ATLAS tide model format + * `'OTIS'` + * `'netcdf'` - `GZIP`: Tide model files are gzip compressed - `DEFINITION_FILE`: Tide model definition file for use as correction - `EPOCH`: time period for calculating delta times diff --git a/doc/source/user_guide/model.rst b/doc/source/user_guide/model.rst index f53ae852..9c423b60 100644 --- a/doc/source/user_guide/model.rst +++ b/doc/source/user_guide/model.rst @@ -2,7 +2,7 @@ model.py ======== -Class with parameters for named tide models +Retrieves tide model parameters for named tide models and from model definition files `Source code`__ @@ -61,6 +61,10 @@ General Attributes and Methods Model grid file for ``OTIS`` and ``ATLAS`` models + .. attribute:: model.gzip + + Suffix if model is compressed + .. attribute:: model.long_name HDF5 ``long_name`` attribute string for output tide heights @@ -85,6 +89,34 @@ General Attributes and Methods Model scaling factor for converting to output units + .. attribute:: model.suffix + + Suffix if ATLAS model is ``'netcdf'`` format + .. attribute:: model.type Model type (``z``, ``u``, ``v``) + + .. attribute:: model.verify + + Verify that all model files exist + + .. attribute:: model.version + + Tide model version + + .. method:: model.pathfinder(model_file) + + Completes file paths and appends file and gzip suffixes + + .. method:: model.from_file(definition_file) + + Create a model object from an input definition file + + .. method:: model.from_dict(d) + + Create a model object from a python dictionary + + .. method:: model.to_bool(val) + + Converts strings of True/False to a boolean values diff --git a/pyTMD/check_tide_points.py b/pyTMD/check_tide_points.py index 8a0fa5f2..c08dddd6 100644 --- a/pyTMD/check_tide_points.py +++ b/pyTMD/check_tide_points.py @@ -1,7 +1,7 @@ #!/usr/bin/env python u""" check_tide_points.py -Written by Tyler Sutterley (07/2021) +Written by Tyler Sutterley (09/2021) Check if points are within a tide model domain OTIS format tidal solutions provided by Ohio State University and ESR @@ -17,7 +17,10 @@ OPTIONS: DIRECTORY: working data directory for tide models - MODEL: Tide model to use in correction + MODEL: Tide model to use + ATLAS_FORMAT: ATLAS tide model format (OTIS, netcdf) + GZIP: Tide model files are gzip compressed + DEFINITION_FILE: Tide model definition file for use EPSG: input coordinate system default: 3031 Polar Stereographic South, WGS84 METHOD: interpolation method @@ -40,6 +43,7 @@ https://pypi.org/project/pyproj/ PROGRAM DEPENDENCIES: + model.py: retrieves tide model parameters for named tide models convert_ll_xy.py: convert lat/lon points to and from projected coordinates read_tide_model.py: extract tidal harmonic constants from OTIS tide models read_netcdf_model.py: extract tidal harmonic constants from netcdf models @@ -48,6 +52,7 @@ bilinear_interp.py: bilinear interpolation of data to coordinates UPDATE HISTORY: + Updated 09/2021: refactor to use model class for files and attributes Updated 07/2021: added check that tide model directory is accessible Updated 06/2021: add try/except for input projection strings Written 05/2021 @@ -58,6 +63,7 @@ import pyproj import numpy as np import scipy.interpolate +import pyTMD.model import pyTMD.convert_ll_xy import pyTMD.read_tide_model import pyTMD.read_netcdf_model @@ -66,7 +72,9 @@ from pyTMD.bilinear_interp import bilinear_interp # PURPOSE: compute tides at points and times using tide model algorithms -def check_tide_points(x,y,DIRECTORY=None,MODEL=None,EPSG=3031,METHOD='spline'): +def check_tide_points(x, y, DIRECTORY=None, MODEL=None, + ATLAS_FORMAT='netcdf', GZIP=False, DEFINITION_FILE=None, + EPSG=3031, METHOD='spline'): """ Check if points are within a tide model domain @@ -78,7 +86,10 @@ def check_tide_points(x,y,DIRECTORY=None,MODEL=None,EPSG=3031,METHOD='spline'): Keyword arguments ----------------- DIRECTORY: working data directory for tide models - MODEL: Tide model to use in correction + MODEL: Tide model to use + ATLAS_FORMAT: ATLAS tide model format (OTIS, netcdf) + GZIP: Tide model files are gzip compressed + DEFINITION_FILE: Tide model definition file for use EPSG: input coordinate system default: 3031 Polar Stereographic South, WGS84 METHOD: interpolation method @@ -97,77 +108,12 @@ def check_tide_points(x,y,DIRECTORY=None,MODEL=None,EPSG=3031,METHOD='spline'): except: raise FileNotFoundError("Invalid tide directory") - # select between tide models - if (MODEL == 'CATS0201'): - grid_file = os.path.join(DIRECTORY,'cats0201_tmd','grid_CATS') - model_format = 'OTIS' - model_EPSG = '4326' - elif (MODEL == 'CATS2008'): - grid_file = os.path.join(DIRECTORY,'CATS2008','grid_CATS2008') - model_format = 'OTIS' - model_EPSG = 'CATS2008' - elif (MODEL == 'TPXO9-atlas'): - grid_file = os.path.join(DIRECTORY,'TPXO9_atlas','grid_tpxo9_atlas.nc.gz') - model_format = 'netcdf' - elif (MODEL == 'TPXO9-atlas-v2'): - grid_file = os.path.join(DIRECTORY,'TPXO9_atlas_v2','grid_tpxo9_atlas_30_v2.nc.gz') - model_format = 'netcdf' - elif (MODEL == 'TPXO9-atlas-v3'): - grid_file = os.path.join(DIRECTORY,'TPXO9_atlas_v3','grid_tpxo9_atlas_30_v3.nc.gz') - model_format = 'netcdf' - elif (MODEL == 'TPXO9-atlas-v4'): - grid_file = os.path.join(DIRECTORY,'TPXO9_atlas_v4','grid_tpxo9_atlas_30_v4') - model_format = 'OTIS' - model_EPSG = '4326' - elif (MODEL == 'TPXO9.1'): - grid_file = os.path.join(DIRECTORY,'TPXO9.1','DATA','grid_tpxo9') - model_format = 'OTIS' - model_EPSG = '4326' - elif (MODEL == 'TPXO8-atlas'): - grid_file = os.path.join(DIRECTORY,'tpxo8_atlas','grid_tpxo8atlas_30_v1') - model_format = 'ATLAS' - model_EPSG = '4326' - elif (MODEL == 'TPXO7.2'): - grid_file = os.path.join(DIRECTORY,'TPXO7.2_tmd','grid_tpxo7.2') - model_format = 'OTIS' - model_EPSG = '4326' - elif (MODEL == 'AODTM-5'): - grid_file = os.path.join(DIRECTORY,'aodtm5_tmd','grid_Arc5km') - model_format = 'OTIS' - model_EPSG = 'PSNorth' - elif (MODEL == 'AOTIM-5'): - grid_file = os.path.join(DIRECTORY,'aotim5_tmd','grid_Arc5km') - model_format = 'OTIS' - model_EPSG = 'PSNorth' - elif (MODEL == 'AOTIM-5-2018'): - grid_file = os.path.join(DIRECTORY,'Arc5km2018','grid_Arc5km2018') - model_format = 'OTIS' - model_EPSG = 'PSNorth' - elif (MODEL == 'GOT4.7'): - model_directory = os.path.join(DIRECTORY,'GOT4.7','grids_oceantide') - model_files = ['q1.d.gz','o1.d.gz','p1.d.gz','k1.d.gz','n2.d.gz', - 'm2.d.gz','s2.d.gz','k2.d.gz','s1.d.gz','m4.d.gz'] - model_format = 'GOT' - elif (MODEL == 'GOT4.8'): - model_directory = os.path.join(DIRECTORY,'got4.8','grids_oceantide') - model_files = ['q1.d.gz','o1.d.gz','p1.d.gz','k1.d.gz','n2.d.gz', - 'm2.d.gz','s2.d.gz','k2.d.gz','s1.d.gz','m4.d.gz'] - model_format = 'GOT' - elif (MODEL == 'GOT4.10'): - model_directory = os.path.join(DIRECTORY,'GOT4.10c','grids_oceantide') - model_files = ['q1.d.gz','o1.d.gz','p1.d.gz','k1.d.gz','n2.d.gz', - 'm2.d.gz','s2.d.gz','k2.d.gz','s1.d.gz','m4.d.gz'] - model_format = 'GOT' - elif (MODEL == 'FES2014'): - model_directory = os.path.join(DIRECTORY,'fes2014','ocean_tide') - model_files = ['2n2.nc.gz','eps2.nc.gz','j1.nc.gz','k1.nc.gz', - 'k2.nc.gz','l2.nc.gz','la2.nc.gz','m2.nc.gz','m3.nc.gz','m4.nc.gz', - 'm6.nc.gz','m8.nc.gz','mf.nc.gz','mks2.nc.gz','mm.nc.gz', - 'mn4.nc.gz','ms4.nc.gz','msf.nc.gz','msqm.nc.gz','mtm.nc.gz', - 'mu2.nc.gz','n2.nc.gz','n4.nc.gz','nu2.nc.gz','o1.nc.gz','p1.nc.gz', - 'q1.nc.gz','r2.nc.gz','s1.nc.gz','s2.nc.gz','s4.nc.gz','sa.nc.gz', - 'ssa.nc.gz','t2.nc.gz'] - model_format = 'FES' + #-- get parameters for tide model + if DEFINITION_FILE is not None: + model = pyTMD.model(DIRECTORY).from_file(DEFINITION_FILE) + else: + model = pyTMD.model(DIRECTORY, format=ATLAS_FORMAT, + compressed=GZIP).elevation(MODEL) # input shape of data idim = np.shape(x) @@ -184,39 +130,38 @@ def check_tide_points(x,y,DIRECTORY=None,MODEL=None,EPSG=3031,METHOD='spline'): np.atleast_1d(y).flatten()) # read tidal constants and interpolate to grid points - if model_format in ('OTIS','ATLAS'): + if model.format in ('OTIS','ATLAS'): # if reading a single OTIS solution - xi,yi,hz,mz,iob,dt = pyTMD.read_tide_model.read_tide_grid(grid_file) + xi,yi,hz,mz,iob,dt = pyTMD.read_tide_model.read_tide_grid(model.grid_file) # invert model mask mz = np.logical_not(mz) # adjust dimensions of input coordinates to be iterable # run wrapper function to convert coordinate systems of input lat/lon - X,Y = pyTMD.convert_ll_xy(lon,lat,model_EPSG,'F') - elif (model_format == 'netcdf'): + X,Y = pyTMD.convert_ll_xy(lon,lat,model.projection,'F') + elif (model.format == 'netcdf'): # if reading a netCDF OTIS atlas solution - xi,yi,hz = pyTMD.read_netcdf_model.read_netcdf_grid(grid_file, - GZIP=True, TYPE='z') + xi,yi,hz = pyTMD.read_netcdf_model.read_netcdf_grid(model.grid_file, + GZIP=model.compressed, TYPE=model.type) # copy bathymetry mask mz = np.copy(hz.mask) # copy latitude and longitude and adjust longitudes X,Y = np.copy([lon,lat]).astype(np.float64) lt0, = np.nonzero(X < 0) X[lt0] += 360.0 - elif (model_format == 'GOT'): + elif (model.format == 'GOT'): # if reading a NASA GOT solution - input_file = os.path.join(model_directory,model_files[0]) - hc,xi,yi,c = pyTMD.read_GOT_model.read_GOT_grid(input_file, GZIP=True) + hc,xi,yi,c = pyTMD.read_GOT_model.read_GOT_grid(model.model_file[0], + GZIP=model.compressed) # copy tidal constituent mask mz = np.copy(hc.mask) # copy latitude and longitude and adjust longitudes X,Y = np.copy([lon,lat]).astype(np.float64) lt0, = np.nonzero(X < 0) X[lt0] += 360.0 - elif (model_format == 'FES'): + elif (model.format == 'FES'): # if reading a FES netCDF solution - input_file = os.path.join(model_directory,model_files[0]) - hc,xi,yi = pyTMD.read_FES_model.read_netcdf_file(input_file,GZIP=True, - TYPE='z',VERSION='FES2014') + hc,xi,yi = pyTMD.read_FES_model.read_netcdf_file(model.model_file[0], + GZIP=model.compressed, TYPE=model.type, VERSION=model.version) # copy tidal constituent mask mz = np.copy(hc.mask) # copy latitude and longitude and adjust longitudes diff --git a/pyTMD/compute_tide_corrections.py b/pyTMD/compute_tide_corrections.py index 357e8ca5..8b89a9e2 100644 --- a/pyTMD/compute_tide_corrections.py +++ b/pyTMD/compute_tide_corrections.py @@ -57,6 +57,7 @@ PROGRAM DEPENDENCIES: time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models spatial: utilities for reading, writing and operating on spatial data utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes diff --git a/pyTMD/model.py b/pyTMD/model.py index 2cd5aa3e..2fa718be 100644 --- a/pyTMD/model.py +++ b/pyTMD/model.py @@ -2,8 +2,8 @@ u""" model.py Written by Tyler Sutterley (09/2021) -Retrieves tide model parameters for named models or - from a model definition file +Retrieves tide model parameters for named tide models and + from model definition files UPDATE HISTORY: Written 09/2021 @@ -22,6 +22,7 @@ def __init__(self, directory=os.getcwd(), **kwargs): # set default keyword arguments kwargs.setdefault('compressed',False) kwargs.setdefault('format','netcdf') + kwargs.setdefault('verify',True) # set initial attributes self.atl03 = None self.atl06 = None @@ -43,6 +44,7 @@ def __init__(self, directory=os.getcwd(), **kwargs): self.scale = None self.type = None self.variable = None + self.verify = copy.copy(kwargs['verify']) self.version = None def grid(self,m): @@ -904,6 +906,33 @@ def current(self,m): # return the model parameters return self + @property + def gzip(self): + """compression flag""" + return '.gz' if self.compressed else '' + + @property + def suffix(self): + """format suffix flag""" + return '.nc' if (self.format == 'netcdf') else '' + + def pathfinder(self,model_file): + """Completes file paths and appends file and gzip suffixes + """ + if isinstance(model_file,list): + output_file = [os.path.join(self.model_directory, + ''.join([f,self.suffix,self.gzip])) for f in model_file] + valid = all([os.access(f, os.F_OK) for f in output_file]) + elif isinstance(model_file,str): + output_file = os.path.join(self.model_directory, + ''.join([model_file,self.suffix,self.gzip])) + valid = os.access(output_file, os.F_OK) + #-- check that (all) output files exist + if self.verify and not valid: + raise FileNotFoundError(output_file) + #-- return the complete output path + return output_file + def from_file(self, definition_file): """Create a model object from an input definition file """ @@ -911,7 +940,7 @@ def from_file(self, definition_file): parameters = {} # Opening definition file and assigning file ID number if isinstance(definition_file,io.IOBase): - fid = open(definition_file, 'r') + fid = copy.copy(definition_file) else: fid = open(os.path.expanduser(definition_file), 'r') # for each line in the file will extract the parameter (name and value) @@ -955,6 +984,9 @@ def from_file(self, definition_file): # split type into list if currents u,v if re.search(r'[\s\,]+', temp.type): temp.type = re.split(r'[\s\,]+',temp.type) + # convert boolean strings + if isinstance(temp.compressed,str): + temp.compressed = self.to_bool(temp.compressed) # return the model parameters return temp @@ -966,29 +998,12 @@ def from_dict(self,d): # return the model parameters return self - @property - def gzip(self): - """compression flag""" - return '.gz' if self.compressed else '' - - @property - def suffix(self): - """format suffix flag""" - return '.nc' if (self.format == 'netcdf') else '' - - def pathfinder(self,model_file): - """Completes file paths and appends file and gzip suffixes + def to_bool(self,val): + """Converts strings of True/False to a boolean values """ - if isinstance(model_file,list): - output_file = [os.path.join(self.model_directory, - ''.join([f,self.suffix,self.gzip])) for f in model_file] - valid = all([os.access(f, os.F_OK) for f in output_file]) - elif isinstance(model_file,str): - output_file = os.path.join(self.model_directory, - ''.join([model_file,self.suffix,self.gzip])) - valid = os.access(output_file, os.F_OK) - #-- check that (all) output files exist - if not valid: - raise FileNotFoundError(output_file) - #-- return the complete output path - return output_file \ No newline at end of file + if val.lower() in ('y','yes','t','true','1'): + return True + elif val.lower() in ('n','no','f','false','0'): + return False + else: + raise ValueError('Invalid boolean string {0}'.format(val)) diff --git a/scripts/compute_tidal_currents.py b/scripts/compute_tidal_currents.py index f5ae6710..ca7fd92c 100755 --- a/scripts/compute_tidal_currents.py +++ b/scripts/compute_tidal_currents.py @@ -85,7 +85,8 @@ PROGRAM DEPENDENCIES: time.py: utilities for calculating time operations - spatial.py: utilities for reading and writing spatial data + model.py: retrieves tide model parameters for named tide models + spatial: utilities for reading, writing and operating on spatial data utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tidal_elevations.py b/scripts/compute_tidal_elevations.py index 35a5e84d..58f44cc1 100755 --- a/scripts/compute_tidal_elevations.py +++ b/scripts/compute_tidal_elevations.py @@ -94,7 +94,8 @@ PROGRAM DEPENDENCIES: time.py: utilities for calculating time operations - spatial.py: utilities for reading and writing spatial data + model.py: retrieves tide model parameters for named tide models + spatial: utilities for reading, writing and operating on spatial data utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat2_ATL03.py b/scripts/compute_tides_ICESat2_ATL03.py index f7d25e17..9b7e83e9 100644 --- a/scripts/compute_tides_ICESat2_ATL03.py +++ b/scripts/compute_tides_ICESat2_ATL03.py @@ -67,6 +67,7 @@ PROGRAM DEPENDENCIES: read_ICESat2_ATL03.py: reads ICESat-2 global geolocated photon data files time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat2_ATL06.py b/scripts/compute_tides_ICESat2_ATL06.py index 6997bf34..6f54b9b9 100644 --- a/scripts/compute_tides_ICESat2_ATL06.py +++ b/scripts/compute_tides_ICESat2_ATL06.py @@ -62,6 +62,7 @@ PROGRAM DEPENDENCIES: read_ICESat2_ATL06.py: reads ICESat-2 land ice along-track height data files time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat2_ATL07.py b/scripts/compute_tides_ICESat2_ATL07.py index f25ff76f..84c4ce15 100644 --- a/scripts/compute_tides_ICESat2_ATL07.py +++ b/scripts/compute_tides_ICESat2_ATL07.py @@ -62,6 +62,7 @@ PROGRAM DEPENDENCIES: read_ICESat2_ATL07.py: reads ICESat-2 sea ice height data files time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat2_ATL11.py b/scripts/compute_tides_ICESat2_ATL11.py index 19728b15..c77222b3 100644 --- a/scripts/compute_tides_ICESat2_ATL11.py +++ b/scripts/compute_tides_ICESat2_ATL11.py @@ -62,6 +62,7 @@ PROGRAM DEPENDENCIES: read_ICESat2_ATL11.py: reads ICESat-2 annual land ice height data files time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat2_ATL12.py b/scripts/compute_tides_ICESat2_ATL12.py index 92eec8e3..180240e9 100644 --- a/scripts/compute_tides_ICESat2_ATL12.py +++ b/scripts/compute_tides_ICESat2_ATL12.py @@ -62,6 +62,7 @@ PROGRAM DEPENDENCIES: read_ICESat2_ATL12.py: reads ICESat-2 ocean surface height data files time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_ICESat_GLA12.py b/scripts/compute_tides_ICESat_GLA12.py index 0cd5de58..b94c5e30 100644 --- a/scripts/compute_tides_ICESat_GLA12.py +++ b/scripts/compute_tides_ICESat_GLA12.py @@ -65,7 +65,8 @@ PROGRAM DEPENDENCIES: time.py: utilities for calculating time operations - spatial.py: utilities for reading, writing and operating on spatial data + model.py: retrieves tide model parameters for named tide models + spatial: utilities for reading, writing and operating on spatial data utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/compute_tides_icebridge_data.py b/scripts/compute_tides_icebridge_data.py index b7308ea3..f82e636a 100644 --- a/scripts/compute_tides_icebridge_data.py +++ b/scripts/compute_tides_icebridge_data.py @@ -69,6 +69,7 @@ PROGRAM DEPENDENCIES: time.py: utilities for calculating time operations + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files calc_astrol_longitudes.py: computes the basic astronomical mean longitudes calc_delta_time.py: calculates difference between universal and dynamic time diff --git a/scripts/reduce_OTIS_files.py b/scripts/reduce_OTIS_files.py index 2fe28552..7b7544af 100644 --- a/scripts/reduce_OTIS_files.py +++ b/scripts/reduce_OTIS_files.py @@ -22,6 +22,7 @@ https://pypi.org/project/pyproj/ PROGRAM DEPENDENCIES: + model.py: retrieves tide model parameters for named tide models utilities.py: download and management utilities for syncing files read_tide_model.py: extract tidal harmonic constants out of a tidal model convert_ll_xy.py: converts lat/lon points to and from projected coordinates diff --git a/test/test_atlas_read.py b/test/test_atlas_read.py index b8810c76..8246e77f 100644 --- a/test/test_atlas_read.py +++ b/test/test_atlas_read.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -test_atlas_read.py (03/2021) +test_atlas_read.py (09/2021) Tests that ATLAS compact and netCDF4 data can be downloaded from AWS S3 bucket Tests the read program to verify that constituents are being extracted @@ -16,6 +16,7 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/index.html UPDATE HISTORY: + Updated 09/2021: added test for model definition files Updated 03/2021: use pytest fixture to setup and teardown model data simplified netcdf inputs to be similar to binary OTIS read program replaced numpy bool/int to prevent deprecation warnings @@ -23,6 +24,7 @@ """ import os import re +import io import gzip import boto3 import shutil @@ -32,6 +34,7 @@ import posixpath import numpy as np import pyTMD.time +import pyTMD.model import pyTMD.utilities import pyTMD.read_tide_model import pyTMD.read_netcdf_model @@ -83,29 +86,32 @@ def download_TPXO9_v2(aws_access_key_id,aws_secret_access_key,aws_region_name): bucket = s3.Bucket('pytmd') #-- model parameters for TPXO9-atlas-v2 - model_directory = os.path.join(filepath,'TPXO9_atlas_v2') - model_files = ['grid_tpxo9_atlas_30_v2.nc.gz','h_2n2_tpxo9_atlas_30_v2.nc.gz', - 'h_k1_tpxo9_atlas_30_v2.nc.gz','h_k2_tpxo9_atlas_30_v2.nc.gz', - 'h_m2_tpxo9_atlas_30_v2.nc.gz','h_m4_tpxo9_atlas_30_v2.nc.gz', - 'h_mn4_tpxo9_atlas_30_v2.nc.gz','h_ms4_tpxo9_atlas_30_v2.nc.gz', - 'h_n2_tpxo9_atlas_30_v2.nc.gz','h_o1_tpxo9_atlas_30_v2.nc.gz', - 'h_p1_tpxo9_atlas_30_v2.nc.gz','h_q1_tpxo9_atlas_30_v2.nc.gz', - 'h_s2_tpxo9_atlas_30_v2.nc.gz'] + model = pyTMD.model(filepath,format='netcdf',compressed=True, + verify=False).elevation('TPXO9-atlas-v2') #-- recursively create model directory - os.makedirs(model_directory) + os.makedirs(model.model_directory) + #-- retrieve grid file from s3 + f = os.path.basename(model.grid_file) + obj = bucket.Object(key=posixpath.join('TPXO9_atlas_v2',f)) + response = obj.get() + #-- save grid data + with open(model.grid_file, 'wb') as destination: + shutil.copyfileobj(response['Body'], destination) + assert os.access(model.grid_file, os.F_OK) #-- retrieve each model file from s3 - for f in model_files: + for model_file in model.model_file: #-- retrieve constituent file + f = os.path.basename(model_file) obj = bucket.Object(key=posixpath.join('TPXO9_atlas_v2',f)) response = obj.get() #-- save constituent data - with open(os.path.join(model_directory,f), 'wb') as destination: + with open(model_file, 'wb') as destination: shutil.copyfileobj(response['Body'], destination) - assert os.access(os.path.join(model_directory,f), os.F_OK) + assert os.access(model_file, os.F_OK) #-- run tests yield #-- clean up model - shutil.rmtree(model_directory) + shutil.rmtree(model.model_directory) #-- parameterize interpolation method @pytest.mark.parametrize("METHOD", ['spline','nearest']) @@ -248,3 +254,23 @@ def test_Ross_Ice_Shelf(MODEL, METHOD, EXTRAPOLATE): EPOCH=(2000,1,1,0,0,0), TYPE='grid', TIME='TAI', EPSG=3031, METHOD=METHOD, EXTRAPOLATE=EXTRAPOLATE) assert np.any(tide) + +#-- PURPOSE: test definition file functionality +@pytest.mark.parametrize("MODEL", ['TPXO9-atlas-v2']) +def test_definition_file(MODEL): + #-- get model parameters + model = pyTMD.model(filepath,compressed=True).elevation(MODEL) + #-- create model definition file + fid = io.StringIO() + attrs = ['name','format','grid_file','model_file','compressed','type','scale'] + for attr in attrs: + val = getattr(model,attr) + if isinstance(val,list): + fid.write('{0}\t{1}\n'.format(attr,','.join(val))) + else: + fid.write('{0}\t{1}\n'.format(attr,val)) + fid.seek(0) + #-- use model definition file as input + m = pyTMD.model().from_file(fid) + for attr in attrs: + assert getattr(model,attr) == getattr(m,attr) diff --git a/test/test_download_and_read.py b/test/test_download_and_read.py index 73bac9e5..d259bedc 100644 --- a/test/test_download_and_read.py +++ b/test/test_download_and_read.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -test_download_and_read.py (05/2021) +test_download_and_read.py (09/2021) Tests that CATS2008 data can be downloaded from the US Antarctic Program (USAP) Tests that AOTIM-5-2018 data can be downloaded from the NSF ArcticData server Tests the read program to verify that constituents are being extracted @@ -19,6 +19,7 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/index.html UPDATE HISTORY: + Updated 09/2021: added test for model definition files Updated 07/2021: download CATS2008 and AntTG from S3 to bypass USAP captcha Updated 05/2021: added test for check point program Updated 03/2021: use pytest fixture to setup and teardown model data @@ -34,6 +35,7 @@ """ import os import re +import io import boto3 import shutil import pytest @@ -43,6 +45,7 @@ import posixpath import numpy as np import pyTMD.time +import pyTMD.model import pyTMD.utilities import pyTMD.read_tide_model import pyTMD.predict_tidal_ts @@ -554,6 +557,27 @@ def test_Ross_Ice_Shelf(self, METHOD, EXTRAPOLATE): EPSG=3031, METHOD=METHOD, EXTRAPOLATE=EXTRAPOLATE) assert np.any(tide) + #-- PURPOSE: test definition file functionality + @pytest.mark.parametrize("MODEL", ['CATS2008']) + def test_definition_file(self, MODEL): + #-- get model parameters + model = pyTMD.model(filepath).elevation(MODEL) + #-- create model definition file + fid = io.StringIO() + attrs = ['name','format','grid_file','model_file','type','projection'] + for attr in attrs: + val = getattr(model,attr) + if isinstance(val,list): + fid.write('{0}\t{1}\n'.format(attr,','.join(val))) + else: + fid.write('{0}\t{1}\n'.format(attr,val)) + fid.seek(0) + #-- use model definition file as input + m = pyTMD.model().from_file(fid) + for attr in attrs: + assert getattr(model,attr) == getattr(m,attr) + + #-- PURPOSE: Test and Verify AOTIM-5-2018 model read and prediction programs class Test_AOTIM5_2018: #-- PURPOSE: Download AOTIM-5-2018 from NSF ArcticData server @@ -737,3 +761,23 @@ def test_Arctic_Ocean(self, METHOD, EXTRAPOLATE): EPOCH=(2000,1,1,12,0,0), TYPE='grid', TIME='UTC', EPSG=3413, METHOD=METHOD, EXTRAPOLATE=EXTRAPOLATE) assert np.any(tide) + + #-- PURPOSE: test definition file functionality + @pytest.mark.parametrize("MODEL", ['AOTIM-5-2018']) + def test_definition_file(self, MODEL): + #-- get model parameters + model = pyTMD.model(filepath).elevation(MODEL) + #-- create model definition file + fid = io.StringIO() + attrs = ['name','format','grid_file','model_file','type','projection'] + for attr in attrs: + val = getattr(model,attr) + if isinstance(val,list): + fid.write('{0}\t{1}\n'.format(attr,','.join(val))) + else: + fid.write('{0}\t{1}\n'.format(attr,val)) + fid.seek(0) + #-- use model definition file as input + m = pyTMD.model().from_file(fid) + for attr in attrs: + assert getattr(model,attr) == getattr(m,attr) diff --git a/test/test_fes_predict.py b/test/test_fes_predict.py index 68773054..65a85788 100644 --- a/test/test_fes_predict.py +++ b/test/test_fes_predict.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -test_fes_predict.py (05/2021) +test_fes_predict.py (09/2021) Tests that FES2014 data can be downloaded from AWS S3 bucket Tests the read program to verify that constituents are being extracted Tests that interpolated results are comparable to FES2014 program @@ -17,12 +17,14 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/index.html UPDATE HISTORY: + Updated 09/2021: update check tide points to add compression flags Updated 05/2021: added test for check point program Updated 03/2021: use pytest fixture to setup and teardown model data Updated 02/2021: replaced numpy bool to prevent deprecation warning Written 08/2020 """ import os +import io import boto3 import shutil import pytest @@ -31,6 +33,7 @@ import posixpath import numpy as np import pyTMD.time +import pyTMD.model import pyTMD.utilities import pyTMD.read_FES_model import pyTMD.predict_tide_drift @@ -55,33 +58,31 @@ def download_model(aws_access_key_id,aws_secret_access_key,aws_region_name): bucket = s3.Bucket('pytmd') #-- model parameters for FES2014 - modelpath = os.path.join(filepath,'fes2014') - model_directory = os.path.join(modelpath,'ocean_tide') - model_files = ['2n2.nc.gz','k1.nc.gz','k2.nc.gz','m2.nc.gz','m4.nc.gz', - 'mf.nc.gz','mm.nc.gz','msqm.nc.gz','mtm.nc.gz','n2.nc.gz','o1.nc.gz', - 'p1.nc.gz','q1.nc.gz','s1.nc.gz','s2.nc.gz'] + model = pyTMD.model(filepath,compressed=True, + verify=False).elevation('FES2014') #-- recursively create model directory - os.makedirs(model_directory) + os.makedirs(model.model_directory) #-- retrieve each model file from s3 - for f in model_files: + for model_file in model.model_file: #-- retrieve constituent file + f = os.path.basename(model_file) obj = bucket.Object(key=posixpath.join('fes2014','ocean_tide',f)) response = obj.get() #-- save constituent data - with open(os.path.join(model_directory,f), 'wb') as destination: + with open(model_file, 'wb') as destination: shutil.copyfileobj(response['Body'], destination) - assert os.access(os.path.join(model_directory,f), os.F_OK) + assert os.access(model_file, os.F_OK) #-- run tests yield #-- clean up model - shutil.rmtree(modelpath) + shutil.rmtree(model.model_directory) #-- PURPOSE: Tests check point program def test_check_FES2014(): lons = np.zeros((10)) + 178.0 lats = -45.0 - np.arange(10)*5.0 obs = pyTMD.check_tide_points(lons, lats, DIRECTORY=filepath, - MODEL='FES2014', EPSG=4326) + MODEL='FES2014', GZIP=True, EPSG=4326) exp = np.array([True, True, True, True, True, True, True, True, False, False]) assert np.all(obs == exp) @@ -149,3 +150,23 @@ def test_verify_FES2014(): difference.mask = np.copy(tide.mask) if not np.all(difference.mask): assert np.all(np.abs(difference) <= eps) + +#-- PURPOSE: test definition file functionality +@pytest.mark.parametrize("MODEL", ['FES2014']) +def test_definition_file(MODEL): + #-- get model parameters + model = pyTMD.model(filepath,compressed=True).elevation(MODEL) + #-- create model definition file + fid = io.StringIO() + attrs = ['name','format','model_file','compressed','type','scale','version'] + for attr in attrs: + val = getattr(model,attr) + if isinstance(val,list): + fid.write('{0}\t{1}\n'.format(attr,','.join(val))) + else: + fid.write('{0}\t{1}\n'.format(attr,val)) + fid.seek(0) + #-- use model definition file as input + m = pyTMD.model().from_file(fid) + for attr in attrs: + assert getattr(model,attr) == getattr(m,attr) diff --git a/test/test_perth3_read.py b/test/test_perth3_read.py index ea43b308..d7ad75a0 100644 --- a/test/test_perth3_read.py +++ b/test/test_perth3_read.py @@ -1,6 +1,6 @@ #!/usr/bin/env python u""" -test_perth3_read.py (07/2021) +test_perth3_read.py (09/2021) Tests that GOT4.7 data can be downloaded from AWS S3 bucket Tests the read program to verify that constituents are being extracted Tests that interpolated results are comparable to NASA PERTH3 program @@ -15,6 +15,8 @@ https://boto3.amazonaws.com/v1/documentation/api/latest/index.html UPDATE HISTORY: + Updated 09/2021: added test for model definition files + update check tide points to add compression flags Updated 07/2021: added test for invalid tide model name Updated 05/2021: added test for check point program Updated 03/2021: use pytest fixture to setup and teardown model data @@ -22,6 +24,7 @@ Written 08/2020 """ import os +import io import gzip import boto3 import shutil @@ -31,6 +34,7 @@ import posixpath import numpy as np import pyTMD.time +import pyTMD.model import pyTMD.utilities import pyTMD.read_GOT_model import pyTMD.predict_tide_drift @@ -56,25 +60,24 @@ def download_model(aws_access_key_id,aws_secret_access_key,aws_region_name): bucket = s3.Bucket('pytmd') #-- model parameters for GOT4.7 - modelpath = os.path.join(filepath,'GOT4.7') - model_directory = os.path.join(modelpath,'grids_oceantide') - model_files = ['q1.d.gz','o1.d.gz','p1.d.gz','k1.d.gz','n2.d.gz', - 'm2.d.gz','s2.d.gz','k2.d.gz','s1.d.gz','m4.d.gz'] + model = pyTMD.model(filepath,compressed=True, + verify=False).elevation('GOT4.7') #-- recursively create model directory - os.makedirs(model_directory) + os.makedirs(model.model_directory) #-- retrieve each model file from s3 - for f in model_files: + for model_file in model.model_file: #-- retrieve constituent file + f = os.path.basename(model_file) obj = bucket.Object(key=posixpath.join('GOT4.7','grids_oceantide',f)) response = obj.get() #-- save constituent data - with open(os.path.join(model_directory,f), 'wb') as destination: + with open(model_file, 'wb') as destination: shutil.copyfileobj(response['Body'], destination) - assert os.access(os.path.join(model_directory,f), os.F_OK) + assert os.access(model_file, os.F_OK) #-- run tests yield #-- clean up model - shutil.rmtree(modelpath) + shutil.rmtree(model.model_directory) #-- parameterize interpolation method @pytest.mark.parametrize("METHOD", ['spline','linear','bilinear']) @@ -88,6 +91,7 @@ def test_verify_GOT47(METHOD): model_file = [os.path.join(model_directory,m) for m in model_files] constituents = ['q1','o1','p1','k1','n2','m2','s2','k2','s1'] model_format = 'GOT' + GZIP = True SCALE = 1.0 #-- read validation dataset @@ -114,7 +118,7 @@ def test_verify_GOT47(METHOD): #-- extract amplitude and phase from tide model amp,ph,cons = pyTMD.read_GOT_model.extract_GOT_constants(longitude, - latitude, model_file, METHOD=METHOD, SCALE=SCALE) + latitude, model_file, METHOD=METHOD, GZIP=GZIP, SCALE=SCALE) assert all(c in constituents for c in cons) #-- interpolate delta times from calendar dates to tide time delta_file = pyTMD.utilities.get_data_path(['data','merged_deltat.data']) @@ -149,7 +153,7 @@ def test_check_GOT47(): lons = np.zeros((10)) + 178.0 lats = -45.0 - np.arange(10)*5.0 obs = pyTMD.check_tide_points(lons, lats, DIRECTORY=filepath, - MODEL='GOT4.7', EPSG=4326) + MODEL='GOT4.7', GZIP=True, EPSG=4326) exp = np.array([True, True, True, True, True, True, True, True, False, False]) assert np.all(obs == exp) @@ -176,6 +180,26 @@ def test_Ross_Ice_Shelf(METHOD, EXTRAPOLATE): EPSG=3031, METHOD=METHOD, EXTRAPOLATE=EXTRAPOLATE) assert np.any(tide) +#-- PURPOSE: test definition file functionality +@pytest.mark.parametrize("MODEL", ['GOT4.7']) +def test_definition_file(MODEL): + #-- get model parameters + model = pyTMD.model(filepath,compressed=True).elevation(MODEL) + #-- create model definition file + fid = io.StringIO() + attrs = ['name','format','model_file','compressed','type','scale'] + for attr in attrs: + val = getattr(model,attr) + if isinstance(val,list): + fid.write('{0}\t{1}\n'.format(attr,','.join(val))) + else: + fid.write('{0}\t{1}\n'.format(attr,val)) + fid.seek(0) + #-- use model definition file as input + m = pyTMD.model().from_file(fid) + for attr in attrs: + assert getattr(model,attr) == getattr(m,attr) + #-- PURPOSE: test the catch in the correction wrapper function def test_unlisted_model(): ermsg = "Unlisted tide model"