diff --git a/scripts/iconform b/scripts/iconform index 24251039..dd4bae54 100755 --- a/scripts/iconform +++ b/scripts/iconform @@ -58,6 +58,8 @@ def parseArgs(argv = None): help='Output pathname for the output specification file(s).') parser.add_argument('-p', '--outdir', default=os.getcwd(), type=str, help='Output pathname for the conformer output files(s). This will be appended to each output file.') + parser.add_argument('-to', '--testoutput', default=False, type=bool, + help='Create test output for xconform.') return parser.parse_args(argv) @@ -127,17 +129,18 @@ def fill_missing_glob_attributes(attr, table, v, grids): attr["tracking_id"] = "hdl:21.14100/"+str(uuid.uuid4()) elif "variable_id" in a and "variable_id" in v.keys(): attr["variable_id"] = v["variable_id"] - if "none" not in attr["branch_method"]: - #if "branch_time_in_child" in attr.keys(): - # attr["branch_time_in_child"] = "Get correct date format" - #if "branch_time_in_parent" in attr.keys(): - # attr["branch_time_in_parent"] = "Get correct date format" - if "parent_mip_era" in attr.keys(): - attr["parent_mip_era"] = attr["mip_era"] - if "parent_source_id" in attr.keys(): - attr["parent_source_id"] = attr["source_id"] - if "parent_time_units" in attr.keys(): - attr["parent_time_units"] = "days since 0000-01-01 00:00:00" + if "branch_method" in attr.keys(): + if "none" not in attr["branch_method"]: + #if "branch_time_in_child" in attr.keys(): + # attr["branch_time_in_child"] = "Get correct date format" + #if "branch_time_in_parent" in attr.keys(): + # attr["branch_time_in_parent"] = "Get correct date format" + if "parent_mip_era" in attr.keys(): + attr["parent_mip_era"] = attr["mip_era"] + if "parent_source_id" in attr.keys(): + attr["parent_source_id"] = attr["source_id"] + if "parent_time_units" in attr.keys(): + attr["parent_time_units"] = "days since 0000-01-01 00:00:00" else: if "branch_time_in_child" in attr.keys(): attr["branch_time_in_child"] = "none" @@ -162,14 +165,32 @@ def fill_missing_glob_attributes(attr, table, v, grids): if "further_info_url" in attr.keys(): if "__FILL__" in attr["further_info_url"]: + if 'mip_era' in attr.keys(): mip_era = attr['mip_era'] + else: + mip_era = '' + if 'institution_id' in attr.keys(): institution_id = attr['institution_id'] + else: + institution_id = '' + if 'source_id' in attr.keys(): source_id = attr['source_id'] + else: + source_id = '' + if 'experiment_id' in attr.keys(): experiment_id = attr['experiment_id'] + else: + experiment_id = '' + if 'sub_experiment_id' in attr.keys(): sub_experiment_id = attr['sub_experiment_id'] + else: + sub_experiment_id = '' + if 'variant_label' in attr.keys(): ripf = attr["variant_label"] - info_url = "{0}.{1}.{2}.{3}.{4}.{5}".format(mip_era, institution_id, source_id, experiment_id, sub_experiment_id, ripf) - attr['further_info_url'] = "http://furtherinfo.es-doc.org/" + info_url + else: + ripf = '' + info_url = "{0}.{1}.{2}.{3}.{4}.{5}".format(mip_era, institution_id, source_id, experiment_id, sub_experiment_id, ripf) + attr['further_info_url'] = "http://furtherinfo.es-doc.org/" + info_url if "grid" in attr.keys(): if len(attr["realm"])>0: attr["grid"] = grids[attr["realm"].split()[0]] @@ -211,12 +232,30 @@ def defineVar(v, varName, attr, table_info, definition, experiment, out_dir): else: ripf = '' - mip_era = attributes['mip_era'] - activity_id = attributes['activity_id'] - institution_id = attributes['institution_id'] - source_id = attributes['source_id'] - grid = attributes['grid_label'] - sub_experiment_id = attributes['sub_experiment_id'] + if 'mip_era' in attributes.keys(): + mip_era = attributes['mip_era'] + else: + mip_era = '' + if 'activity_id' in attributes.keys(): + activity_id = attributes['activity_id'] + else: + activity_id = '' + if 'institution_id' in attributes.keys(): + institution_id = attributes['institution_id'] + else: + institution_id = '' + if 'source_id' in attributes.keys(): + source_id = attributes['source_id'] + else: + source_id = '' + if 'grid_labels' in attributes.keys(): + grid = attributes['grid_label'] + else: + grid = '' + if 'sub_experiment_id' in attributes.keys(): + sub_experiment_id = attributes['sub_experiment_id'] + else: + sub_experiment_id = '' f_format = attributes["netcdf_type"] valid_formats = ['NETCDF4','NETCDF4_CLASSIC','NETCDF3_CLASSIC','NETCDF3_64BIT_OFFSET','NETCDF3_64BIT_DATA'] @@ -343,7 +382,7 @@ def getUserVars(fn): #=================================================================================================== # create_output #=================================================================================================== -def create_output(exp_dict, definitions, attributes, output_path, args, experiment, out_dir): +def create_output(exp_dict, definitions, attributes, output_path, args, experiment, out_dir, testoutput): # create the output json files @@ -417,12 +456,22 @@ def create_output(exp_dict, definitions, attributes, output_path, args, experime # create json files per MIP+table if not os.path.exists(output_path): os.makedirs(output_path) - for n,t in TableSpec.iteritems(): - spec = {} - f = output_path + experiment + '_' + n + '_spec.json' - spec["variables"] = t - with open(f, 'w') as outfile: - json.dump(t, outfile, sort_keys=True, indent=4) + + if not testoutput: + for n,t in TableSpec.iteritems(): + f = output_path + "/" + experiment + '_' + n + '_spec.json' + with open(f, 'w') as outfile: + json.dump(t, outfile, sort_keys=True, indent=4) + else: + for n,t in TableSpec.iteritems(): + for vn,var in t.iteritems(): + varD={} + varD[vn]=var + for d in var["dimensions"]: + varD[d]=t[d] + f = output_path + "/" + experiment + '_' + n + '_' + vn + '_spec.json' + with open(f, 'w') as outfile: + json.dump(varD, outfile, sort_keys=True, indent=4) #f1 = output_path + '/MISSING_DEFS.json' #with open(f1, 'w') as outfile: @@ -554,7 +603,7 @@ def main(argv=None): if len(exp_dict.keys())>0: # Write the spec files out to disk - create_output(exp_dict, definitions, attributes, args.outputpath, args, exp, args.outdir) + create_output(exp_dict, definitions, attributes, args.outputpath, args, exp, args.outdir, args.testoutput) #=================================================================================================== diff --git a/source/pyconform/dataflow.py b/source/pyconform/dataflow.py index eea8cbbc..d5827ba5 100644 --- a/source/pyconform/dataflow.py +++ b/source/pyconform/dataflow.py @@ -81,16 +81,16 @@ def __init__(self, inpds, outds): defnodes = self._create_map_nodes_(defnodes, definfos) # Create the validate nodes for each valid output variable - valnodes = self._create_validate_nodes_(datnodes, defnodes) + self._valnodes = self._create_validate_nodes_(datnodes, defnodes) # Get the set of all sum-like dimensions (dimensions that cannot be broken into chunks) - self._sumlike_dimensions = self._find_sumlike_dimensions_(valnodes) + self._sumlike_dimensions = self._find_sumlike_dimensions_() # Create the WriteNodes for each time-series output file - self._writenodes = self._create_write_nodes_(valnodes) + self._writenodes = self._create_write_nodes_() # Compute the bytesizes of each output variable - varsizes = self._compute_variable_sizes_(valnodes) + varsizes = self._compute_variable_sizes_() # Compute the file sizes for each output file self._filesizes = self._compute_file_sizes(varsizes) @@ -226,8 +226,7 @@ def _create_validate_nodes_(self, datnodes, defnodes): vnode = datnodes[vname] if vname in datnodes else defnodes[vname] try: - validnode = ValidateNode(vname, vnode, dimensions=vdesc.dimensions.keys(), - attributes=vdesc.attributes, dtype=vdesc.dtype) + validnode = ValidateNode(vdesc, vnode) except Exception, err: vdef = vdesc.definition err_msg = 'Failure in variable {!r} with definition {!r}: {}'.format(vname, vdef, str(err)) @@ -236,10 +235,10 @@ def _create_validate_nodes_(self, datnodes, defnodes): valnodes[vname] = validnode return valnodes - def _find_sumlike_dimensions_(self, valnodes): + def _find_sumlike_dimensions_(self): unmapped_sumlike_dimensions = set() - for vname in valnodes: - vnode = valnodes[vname] + for vname in self._valnodes: + vnode = self._valnodes[vname] for nd in iter_dfs(vnode): if isinstance(nd, EvalNode): unmapped_sumlike_dimensions.update(nd.sumlike_dimensions) @@ -247,23 +246,23 @@ def _find_sumlike_dimensions_(self, valnodes): # Map the sum-like dimensions to output dimensions return set(self._i2omap[d] for d in unmapped_sumlike_dimensions if d in self._i2omap) - def _create_write_nodes_(self, valnodes): + def _create_write_nodes_(self): writenodes = {} for fname in self._ods.files: fdesc = self._ods.files[fname] - vmissing = tuple(vname for vname in fdesc.variables if vname not in valnodes) + vmissing = tuple(vname for vname in fdesc.variables if vname not in self._valnodes) if vmissing: warn('Skipping output file {} due to missing required variables: ' '{}'.format(fname, ', '.join(sorted(vmissing))), DefinitionWarning) else: - vnodes = tuple(valnodes[vname] for vname in fdesc.variables) + vnodes = tuple(self._valnodes[vname] for vname in fdesc.variables) wnode = WriteNode(fdesc, inputs=vnodes) writenodes[wnode.label] = wnode return writenodes - def _compute_variable_sizes_(self, valnodes): + def _compute_variable_sizes_(self): bytesizes = {} - for vname in valnodes: + for vname in self._valnodes: vdesc = self._ods.variables[vname] vsize = sum(ddesc.size for ddesc in vdesc.dimensions.itervalues()) vsize = 1 if vsize == 0 else vsize diff --git a/source/pyconform/datasets.py b/source/pyconform/datasets.py index e92d7153..d478163b 100644 --- a/source/pyconform/datasets.py +++ b/source/pyconform/datasets.py @@ -10,6 +10,7 @@ from os import linesep from os.path import exists +from copy import deepcopy from collections import OrderedDict from numpy import dtype from netCDF4 import Dataset as NC4Dataset @@ -61,7 +62,7 @@ def _is_list_of_type_(obj, typ): if not isinstance(obj, (list, tuple)): return False return all([isinstance(o, typ) for o in obj]) - + #=================================================================================================== # DimensionDesc @@ -85,7 +86,7 @@ def __init__(self, name, size=None, unlimited=False, stringlen=False): stringlen (bool): Whether the dimension represents a string length or not """ self._name = name - self._size = int(size) if size is not None else None + self._size = None if size is None else int(size) self._unlimited = bool(unlimited) self._stringlen = bool(stringlen) @@ -135,6 +136,7 @@ def set(self, dd): raise TypeError(err_msg) self._size = dd.size self._unlimited = dd.unlimited + self._stringlen = dd.stringlen def __eq__(self, other): if not isinstance(other, DimensionDesc): @@ -145,13 +147,16 @@ def __eq__(self, other): return False if self.unlimited != other.unlimited: return False + if self.stringlen != other.stringlen: + return False return True def __ne__(self, other): return not self.__eq__(other) def __str__(self): - return '{!r} [{}{}]'.format(self.name, self.size, '+' if self.unlimited else '') + set_str = '{}{}'.format(self.size, '+' if self.unlimited else '') if self.is_set() else '?' + return '{!r} [{}]'.format(self.name, set_str) @staticmethod def unique(descs): @@ -199,7 +204,7 @@ class VariableDesc(object): _DTYPES_ = (dtype('b'), dtype('u1'), dtype('S1'), dtype('i2'), dtype('u2'), dtype('i4'), dtype('u4'), dtype('i8'), dtype('u8'), dtype('f4'), dtype('f4'), dtype('f8')) - def __init__(self, name, datatype='float', dimensions=(), definition=None, attributes={}): + def __init__(self, name, datatype=None, dimensions=(), definition=None, attributes={}): """ Initializer @@ -212,7 +217,11 @@ def __init__(self, name, datatype='float', dimensions=(), definition=None, attri """ self._name = name - if isinstance(datatype, basestring) and datatype in VariableDesc._NTYPES_: + # Datatype is inheritable, in which case the datatype is set to None + if datatype is None: + self._ntype = None + self._dtype = None + elif isinstance(datatype, basestring) and datatype in VariableDesc._NTYPES_: self._ntype = datatype self._dtype = VariableDesc._DTYPES_[VariableDesc._NTYPES_.index(datatype)] elif isinstance(datatype, dtype) and datatype in VariableDesc._DTYPES_: @@ -221,18 +230,22 @@ def __init__(self, name, datatype='float', dimensions=(), definition=None, attri else: raise TypeError('Invalid variable datatype {} for variable {}'.format(datatype, name)) - self.definition = definition - + # Dimensions are required for all variables if not _is_list_of_type_(dimensions, DimensionDesc): err_msg = ('Dimensions for variable {!r} must be a list or tuple of type ' 'DimensionDesc').format(name) raise TypeError(err_msg) self._dimensions = DimensionDesc.unique(dimensions) + # Definition is required for output variables, but None for input variables + self._definition = definition + + # Attributes are modifiable after the variable descriptor is constructed if not isinstance(attributes, dict): raise TypeError('Attributes for variable {!r} not dict'.format(name)) - self._attributes = attributes + self._attributes = deepcopy(attributes) + # Initially, no files are associated with the variables, but it is modifiable after construction self._files = {} @property @@ -250,6 +263,10 @@ def dtype(self): """NumPy dtype of the variable data""" return self._dtype + @property + def definition(self): + return self._definition + @property def attributes(self): """Variable attributes dictionary""" @@ -281,7 +298,8 @@ def __ne__(self, other): def __str__(self): strvals = ['Variable: {!r}'.format(self.name)] - strvals += [' datatype: {!r}'.format(self.datatype)] + if self.datatype is not None: + strvals += [' datatype: {!r}'.format(self.datatype)] strvals += [' dimensions: {!s}'.format(self.dimensions.keys())] if self.definition is not None: strvals += [' definition: {!r}'.format(self.definition)] @@ -294,18 +312,34 @@ def __str__(self): for fname in self.files: strvals += [' {}'.format(fname)] return linesep.join(strvals) + + def __repr__(self): + dtyp_str = '' if self.datatype is None else ', datatype={!r}'.format(self.datatype) + dimn_str = ', dimensions={!s}'.format(tuple(self.dimensions.keys())) + defn_str = '' if self.definition is None else ', definition={!r}'.format(self.definition) + return '{!s}(name={!r}{}{}{})'.format(self.__class__.__name__, self.name, dtyp_str, dimn_str, defn_str) def units(self): - """Retrieve the units attribute, if it exists, otherwise 1""" - return self.attributes.get('units', 'no unit') + """Retrieve the units string otherwise None""" + ustr = str(self.attributes.get('units', '?')).split('since')[0].strip() + return None if ustr in ('', '?', 'unknown') else ustr + + def refdatetime(self): + """Retrieve the reference datetime string, otherwise None""" + lstr = str(self.attributes.get('units', '?')).split('since') + rstr = lstr[1].strip() if len(lstr) > 1 else '' + return None if rstr in ('', '?', 'unknown') else rstr def calendar(self): """Retrieve the calendar attribute, if it exists, otherwise None""" return self.attributes.get('calendar', None) + + def units_attr(self): + return self.attributes.get('units', None) def cfunits(self): """Construct a cf_units.Unit object from the units/calendar attributes""" - return Unit(self.units(), calendar=self.calendar()) + return Unit(self.units_attr(), calendar=self.calendar()) @staticmethod def unique(descs): @@ -380,7 +414,7 @@ def __init__(self, name, format='NETCDF4_CLASSIC', deflate=2, variables=(), attr dimensions = [] for vdesc in variables: - dimensions.extend(vdesc.dimensions.values()) + dimensions.extend(vdesc.dimensions.values()) self._dimensions = DimensionDesc.unique(dimensions) for vdesc in variables: @@ -395,7 +429,7 @@ def __init__(self, name, format='NETCDF4_CLASSIC', deflate=2, variables=(), attr err_msg = ('Attributes in file {!r} cannot be of type {!r}, needs to be a ' 'dict').format(name, type(attributes)) raise TypeError(err_msg) - self._attributes = attributes + self._attributes = deepcopy(attributes) @property def name(self): @@ -573,13 +607,13 @@ class InputDatasetDesc(DatasetDesc): parameter will contain the names of files from which the variable data can be read. """ - def __init__(self, name='input', filenames=[]): + def __init__(self, name='input', filenames=()): """ Initializer Parameters: name (str): String name to optionally give to a dataset - filenames (list): List of filenames in the dataset + filenames (tuple): List of filenames in the dataset """ files = [] diff --git a/source/pyconform/flownodes.py b/source/pyconform/flownodes.py index e07a32c6..ad11b1e1 100644 --- a/source/pyconform/flownodes.py +++ b/source/pyconform/flownodes.py @@ -36,6 +36,13 @@ class UnitsWarning(Warning): """Warning for units errors""" +#======================================================================================================================= +# DateTimeAutoParseWarning +#======================================================================================================================= +class DateTimeAutoParseWarning(Warning): + """Warning for not being able to autoparse new filename based on date-time in the file""" + + #======================================================================================================================= # iter_dfs - Depth-First Search Iterator #======================================================================================================================= @@ -433,56 +440,62 @@ class ValidateNode(FlowNode): This is a "non-source"/"non-sink" FlowNode. """ - def __init__(self, label, dnode, dimensions=None, dtype=None, attributes={}): + def __init__(self, vdesc, dnode): """ Initializer Parameters: - label: The label associated with this FlowNode + vdesc (VariableDesc): A variable descriptor object for the output variable dnode (FlowNode): FlowNode that provides input into this FlowNode - dimensions (tuple): The output dimensions to validate against - dtype (dtype): The NumPy dtype of the data to return - attributes (dict): Attributes to associate with the new variable """ - # Check FlowNode type + # Check Types + if not isinstance(vdesc, VariableDesc): + raise TypeError('ValidateNode requires a VariableDesc object as input') if not isinstance(dnode, FlowNode): raise TypeError('ValidateNode can only act on output from another FlowNode') # Call base class initializer - super(ValidateNode, self).__init__(label, dnode) - - # Save the data type - self._dtype = dtype - - # Check for dimensions - self._dimensions = None - if dimensions is not None: - if isinstance(dimensions, (list, tuple)): - self._dimensions = tuple(dimensions) - else: - raise TypeError('Dimensions must be a list or tuple') - - # Store the attributes given to the FlowNode - self._attributes = OrderedDict((k, v) for k, v in attributes.iteritems()) + super(ValidateNode, self).__init__(vdesc.name, dnode) - # Initialize the history attribute - info = self[None] + # Save the variable descriptor object + self._vdesc = vdesc + + # Initialize the history attribute, if necessary + info = dnode[None] if 'history' not in self.attributes: self.attributes['history'] = info.name + # Else inherit the units and calendar of the input data stream, if necessary + if 'units' in self.attributes: + if info.units.is_time_reference(): + ustr, rstr = [c.strip() for c in str(info.units).split('since')] + if self._vdesc.units() is not None: + ustr = self._vdesc.units() + if self._vdesc.refdatetime() is not None: + rstr = self._vdesc.refdatetime() + self.attributes['units'] = '{} since {}'.format(ustr, rstr) + + # Calendars must match as convertion between different calendars will fail + if self._vdesc.calendar() is None and info.units.calendar is not None: + self.attributes['calendar'] = info.units.calendar + + else: + if self._vdesc.units() is None: + self.attributes['units'] = str(info.units) + @property def attributes(self): """ Attributes dictionary of the variable returned by the ValidateNode """ - return self._attributes + return self._vdesc.attributes @property def dimensions(self): """ Dimensions tuple of the variable returned by the ValidateNode """ - return self._dimensions + return tuple(self._vdesc.dimensions.keys()) def __getitem__(self, index): """ @@ -493,21 +506,18 @@ def __getitem__(self, index): indata = self.inputs[0][index] # Check datatype, and cast as necessary - if self._dtype is None: + if self._vdesc.dtype is None: odtype = indata.dtype else: - odtype = self._dtype + odtype = self._vdesc.dtype if numpy.can_cast(indata.dtype, odtype, casting='same_kind'): indata = indata.astype(odtype) else: raise TypeError('Cannot cast datatype {!s} to {!s} in ValidateNode ' '{!r}').format(indata.dtype, odtype, self.label) - # Check that units match as expected + # Check that units match as expected, otherwise convert if 'units' in self.attributes: - if indata.units.is_time_reference(): - if 'calendar' not in self.attributes: - self.attributes['calendar'] = indata.units.calendar ounits = Unit(self.attributes['units'], calendar=self.attributes.get('calendar', None)) if ounits != indata.units: if index is None: @@ -518,9 +528,9 @@ def __getitem__(self, index): except Exception as err: err_msg = 'When validating output variable {}: {}'.format(self.label, err) raise err.__class__(err_msg) - + # Check that the dimensions match as expected - if self.dimensions is not None and self.dimensions != indata.dimensions: + if self.dimensions != indata.dimensions: indata = indata.transpose(self.dimensions) # Check the positive attribute, if specified @@ -653,16 +663,24 @@ def _autoparse_filename_(self, fname): possible_tvars = [] for var in self._filedesc.variables: - vobj = self._filedesc.variables[var] - if vobj.cfunits().is_time_reference() and len(vobj.dimensions) == 1: + vdesc = self._filedesc.variables[var] + if var in ('time', 'time1', 'time2', 'time3'): + possible_tvars.append(var) + elif vdesc.cfunits().is_time_reference() and len(vdesc.dimensions) == 1: + possible_tvars.append(var) + elif 'standard_name' in vdesc.attributes and vdesc.attributes['standard_name'] == 'time': + possible_tvars.append(var) + elif 'axis' in vdesc.attributes and vdesc.attributes['axis'] == 'T': possible_tvars.append(var) if len(possible_tvars) == 0: - raise ValueError('Could not find time variable in file {!r}'.format(fname)) + msg = 'Could not identify a time variable to autoparse filename {!r}'.format(fname) + warn(msg, DateTimeAutoParseWarning) + return fname + tvar = 'time' if 'time' in possible_tvars else possible_tvars[0] - tnodes = [vnode for vnode in self.inputs if vnode.label == tvar] if len(tnodes) == 0: - raise ValueError('Time variable input missing in file {!r}'.format(fname)) + raise ValueError('Time variable input missing for file {!r}'.format(fname)) tnode = tnodes[0] t1 = tnode[0:1] t2 = tnode[-1:] diff --git a/source/pyconform/physarray.py b/source/pyconform/physarray.py index 74614e37..9de9bfc0 100644 --- a/source/pyconform/physarray.py +++ b/source/pyconform/physarray.py @@ -284,8 +284,7 @@ def transpose(self, *dims): new_dims = tuple(self.dimensions[i] for i in dims) axes = dims else: - raise DimensionsError(('Cannot transpose dimensions/axes {} to ' - '{}').format(self.dimensions, dims)) + raise DimensionsError(('Cannot transpose dimensions/axes {} to {}').format(self.dimensions, dims)) if new_dims == self.dimensions: return self else: diff --git a/source/pyconform/version.py b/source/pyconform/version.py index 426dd660..75340b86 100644 --- a/source/pyconform/version.py +++ b/source/pyconform/version.py @@ -1,2 +1,2 @@ # Single place for version information -__version__ = '0.2.2' +__version__ = '0.2.3' diff --git a/source/pyconform/test/__init__.py b/source/test/__init__.py similarity index 100% rename from source/pyconform/test/__init__.py rename to source/test/__init__.py diff --git a/source/pyconform/test/climIOTests.py b/source/test/climIOTests.py similarity index 100% rename from source/pyconform/test/climIOTests.py rename to source/test/climIOTests.py diff --git a/source/pyconform/test/dataflowTests.py b/source/test/dataflowTests.py similarity index 98% rename from source/pyconform/test/dataflowTests.py rename to source/test/dataflowTests.py index 964d3410..2c5163d2 100644 --- a/source/pyconform/test/dataflowTests.py +++ b/source/test/dataflowTests.py @@ -48,7 +48,8 @@ def setUp(self): ('time', {'units': 'days since 1979-01-01 0:0:0', 'calendar': 'noleap', 'standard_name': 'time'}), - ('time_bnds', {'units': 'days since 1979-01-01 0:0:0'}), + ('time_bnds', {'units': 'days since 1979-01-01 0:0:0', + 'calendar': 'noleap'}), ('cat', {'standard_name': 'categories'}), ('u1', {'units': 'km', 'standard_name': 'u variable 1'}), @@ -157,7 +158,10 @@ def setUp(self): vdicts['T_bnds']['datatype'] = 'double' vdicts['T_bnds']['dimensions'] = ('t', 'd') vdicts['T_bnds']['definition'] = 'time_bnds' - vdicts['T_bnds']['attributes'] = OrderedDict() + vattribs = OrderedDict() + vattribs['units'] = 'days since 0001-01-01 00:00:00' + vattribs['calendar'] = 'noleap' + vdicts['T_bnds']['attributes'] = vattribs vdicts['V1'] = OrderedDict() vdicts['V1']['datatype'] = 'double' @@ -320,7 +324,7 @@ def test_dimension_map(self): def test_execute_all(self): testname = 'DataFlow().execute()' df = dataflow.DataFlow(self.inpds, self.outds) - df.execute() + df.execute(history=True) actual = all(exists(f) for f in self.outfiles.itervalues()) expected = True print_test_message(testname, actual=actual, expected=expected) diff --git a/source/pyconform/test/datasetsTests.py b/source/test/datasetsTests.py similarity index 97% rename from source/pyconform/test/datasetsTests.py rename to source/test/datasetsTests.py index a6dc8e6e..964a99d8 100644 --- a/source/pyconform/test/datasetsTests.py +++ b/source/test/datasetsTests.py @@ -235,7 +235,7 @@ def test_name(self): def test_datatype_default(self): vdesc = VariableDesc('x') actual = vdesc.datatype - expected = 'float' + expected = None print_test_message('VariableDesc.datatype == float', actual=actual, expected=expected) self.assertEqual(actual, expected, 'Default VariableDesc.datatype is not float') @@ -348,7 +348,7 @@ def test_equals_diff_dims(self): def test_units_default(self): vdesc = VariableDesc('x') actual = vdesc.units() - expected = Unit('no unit') + expected = None print_test_message('VariableDesc.units() == nounit', actual=actual, expected=expected) self.assertEqual(actual, expected, 'Default VariableDesc.units() not None') @@ -360,6 +360,21 @@ def test_units(self): print_test_message('VariableDesc.units()', indata=indata, actual=actual, expected=expected) self.assertEqual(actual, expected, 'Default VariableDesc.units() not {}'.format(indata)) + def test_refdatetime_default(self): + vdesc = VariableDesc('x') + actual = vdesc.refdatetime() + expected = None + print_test_message('VariableDesc.refdatetime() == None', actual=actual, expected=expected) + self.assertEqual(actual, expected, 'Default VariableDesc.refdatetime() not None') + + def test_refdatetime(self): + indata = '0001-01-01 00:00:00 -05:00' + vdesc = VariableDesc('x', attributes={'units': 'days since {}'.format(indata)}) + actual = vdesc.refdatetime() + expected = indata + print_test_message('VariableDesc.refdatetime()', indata=indata, actual=actual, expected=expected) + self.assertEqual(actual, expected, 'Default VariableDesc.refdatetime() not {}'.format(indata)) + def test_calendar_default(self): vdesc = VariableDesc('x') actual = vdesc.calendar() @@ -379,7 +394,7 @@ def test_calendar(self): def test_cfunits_default(self): vdesc = VariableDesc('time') actual = vdesc.cfunits() - expected = Unit('no unit') + expected = Unit('?') print_test_message('VariableDesc.cfunits() == None', actual=actual, expected=expected) self.assertEqual(actual, expected, 'Default VariableDesc.cfunits() not None') @@ -536,7 +551,7 @@ def test_attributes_invalid(self): self.assertRaises(expected, FileDesc, 'test.nc', attributes=indata) def test_variables_scalar(self): - indata = [VariableDesc('a'), VariableDesc('b')] + indata = (VariableDesc('a'), VariableDesc('b')) fdesc = FileDesc('test.nc', variables=indata) actual = fdesc.variables expected = OrderedDict((d.name, d) for d in indata) @@ -548,8 +563,8 @@ def test_variables_scalar(self): self.assertEqual(actual, expected, 'FileDesc.dimensions failed') def test_variables_same_dims(self): - indata = [VariableDesc('a', dimensions=[DimensionDesc('x', 4)]), - VariableDesc('b', dimensions=[DimensionDesc('x', 4)])] + indata = (VariableDesc('a', dimensions=[DimensionDesc('x', 4)]), + VariableDesc('b', dimensions=[DimensionDesc('x', 4)])) fdesc = FileDesc('test.nc', variables=indata) actual = fdesc.variables expected = OrderedDict((d.name, d) for d in indata) @@ -640,6 +655,7 @@ def setUp(self): vdicts['X']['dimensions'] = ('x',) vdicts['X']['definition'] = 'lon' vattribs = OrderedDict() + vattribs['axis'] = 'X' vattribs['standard_name'] = 'longitude' vattribs['units'] = 'degrees_east' vdicts['X']['attributes'] = vattribs @@ -649,6 +665,7 @@ def setUp(self): vdicts['Y']['dimensions'] = ('y',) vdicts['Y']['definition'] = 'lat' vattribs = OrderedDict() + vattribs['axis'] = 'Y' vattribs['standard_name'] = 'latitude' vattribs['units'] = 'degrees_north' vdicts['Y']['attributes'] = vattribs @@ -659,7 +676,8 @@ def setUp(self): vdicts['T']['definition'] = 'time' vattribs = OrderedDict() vattribs['standard_name'] = 'time' - vattribs['units'] = 'days since 01-01-0001 00:00:00' + vattribs['axis'] = 'T' + vattribs['units'] = 'days since 0001-01-01 00:00:00' vattribs['calendar'] = 'noleap' vdicts['T']['attributes'] = vattribs diff --git a/source/pyconform/test/flownodesTests.py b/source/test/flownodesTests.py similarity index 80% rename from source/pyconform/test/flownodesTests.py rename to source/test/flownodesTests.py index 69bde74c..f9cfcb37 100644 --- a/source/pyconform/test/flownodesTests.py +++ b/source/test/flownodesTests.py @@ -397,8 +397,9 @@ class ValidateNodeTests(unittest.TestCase): def test_nothing(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) + V1 = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),)) testname = 'OK: ValidateNode().__getitem__(:)' - N1 = ValidateNode('validate(x)', N0) + N1 = ValidateNode(V1, N0) actual = N1[:] expected = N0[:] print_test_message(testname, actual=actual, expected=expected) @@ -408,10 +409,9 @@ def test_nothing(self): def test_units_ok(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'units': 'm'} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'm'}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -420,12 +420,10 @@ def test_units_ok(self): self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) def test_time_units_ok(self): - N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', - dimensions=('x',))) - indata = {'units': 'days since 2000-01-01 00:00:00', 'calendar': 'gregorian'} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', dimensions=('x',))) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'days since 2000-01-01 00:00:00', 'calendar': 'gregorian'}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -435,10 +433,9 @@ def test_time_units_ok(self): def test_dimensions_ok(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'dimensions': ('x',)} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, **indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),)) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -448,10 +445,9 @@ def test_dimensions_ok(self): def test_min_ok(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'valid_min': 0} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'valid_min': 0}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -461,10 +457,9 @@ def test_min_ok(self): def test_max_ok(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'valid_max': 10} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'valid_max': 10}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -474,10 +469,9 @@ def test_max_ok(self): def test_minmax_getitem_none(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'valid_min': 0, 'valid_max': 2} - testname = ('OK: ValidateNode({}).__getitem__(None)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'valid_min': 0, 'valid_max': 2}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[None] expected = N0[None] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -487,10 +481,9 @@ def test_minmax_getitem_none(self): def test_min_mean_abs_ok(self): N0 = DataNode(PhysArray(numpy.arange(-5, 10), name='x', units='m', dimensions=('x',))) - indata = {'ok_min_mean_abs': 3} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'ok_min_mean_abs': 3}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -500,10 +493,9 @@ def test_min_mean_abs_ok(self): def test_max_mean_abs_ok(self): N0 = DataNode(PhysArray(numpy.arange(-5, 10), name='x', units='m', dimensions=('x',))) - indata = {'ok_max_mean_abs': 5} - testname = ('OK: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'ok_max_mean_abs': 5}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -513,10 +505,9 @@ def test_max_mean_abs_ok(self): def test_units_convert(self): N0 = DataNode(PhysArray(numpy.arange(10.0), name='x', units='m', dimensions=('x',))) - indata = {'units': 'km'} - testname = ('CONVERT: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'km'}) + testname = 'CONVERT: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = (Unit('m').convert(N0[:], Unit('km'))).astype(numpy.float64) expected.name = 'convert(x, from=m, to=km)' @@ -527,24 +518,33 @@ def test_units_convert(self): self.assertEqual(actual.units, expected.units, '{} failed'.format(testname)) self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) + def test_units_inherit(self): + N0 = DataNode(PhysArray(numpy.arange(10.0), name='x', units='m', dimensions=('x',))) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': '?'}) + testname = 'INHERIT: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) + actual = N1[:] + expected = N0[:] + print_test_message(testname, indata=indata, actual=actual, expected=expected) + numpy.testing.assert_array_equal(actual, expected, '{} failed'.format(testname)) + self.assertEqual(actual.units, expected.units, '{} failed'.format(testname)) + self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) + def test_dimensions_transpose(self): N0 = DataNode(PhysArray([[1.,2.],[3.,4.]], name='a', units='m', dimensions=('x', 'y'))) - indata = {'dimensions': ('y', 'x')} - testname = ('TRANSPOSE: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(a)', N0, **indata) + indata = VariableDesc('validate(a)', dimensions=(DimensionDesc('y'), DimensionDesc('x'))) + testname = 'TRANSPOSE: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:].dimensions - expected = indata['dimensions'] + expected = tuple(indata.dimensions.keys()) print_test_message(testname, indata=indata, actual=actual, expected=expected) self.assertEqual(actual, expected, '{} failed'.format(testname)) def test_time_units_convert(self): - N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', - dimensions=('x',))) - indata = {'units': 'hours since 2000-01-01 00:00:00', 'calendar': 'gregorian'} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', dimensions=('x',))) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'hours since 2000-01-01 00:00:00', 'calendar': 'gregorian'}) + testname = 'CONVERT: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:].convert(Unit('hours since 2000-01-01 00:00:00')) print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -552,13 +552,36 @@ def test_time_units_convert(self): self.assertEqual(actual.units, expected.units, '{} failed'.format(testname)) self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) + def test_time_units_inherit(self): + N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', dimensions=('x',))) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': ''}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) + actual = N1[:] + expected = N0[:] + print_test_message(testname, indata=indata, actual=actual, expected=expected) + numpy.testing.assert_array_equal(actual, expected, '{} failed'.format(testname)) + self.assertEqual(actual.units, expected.units, '{} failed'.format(testname)) + self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) + + def test_time_units_inherit_refdatetime(self): + N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', dimensions=('x',))) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'hours since ?'}) + testname = 'OK: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) + actual = N1[:] + expected = N0[:].convert('hours since 2000-01-01 00:00:00') + print_test_message(testname, indata=indata, actual=actual, expected=expected) + numpy.testing.assert_array_equal(actual, expected, '{} failed'.format(testname)) + self.assertEqual(actual.units, expected.units, '{} failed'.format(testname)) + self.assertEqual(actual.dimensions, expected.dimensions, '{} failed'.format(testname)) + def test_time_units_convert_nocal(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', dimensions=('x',), units=Unit('days since 2000-01-01 00:00:00', calendar='noleap'))) - indata = {'units': 'hours since 2000-01-01 00:00:00'} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'hours since 2000-01-01 00:00:00'}) + testname = 'CONVERT: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:].convert(Unit('hours since 2000-01-01 00:00:00', calendar='noleap')) print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -568,28 +591,26 @@ def test_time_units_convert_nocal(self): def test_time_units_error_calendar(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='days since 2000-01-01 00:00:00', dimensions=('x',))) - indata = {'units': 'days since 2000-01-01 00:00:00', 'calendar': 'noleap'} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'units': 'days since 2000-01-01 00:00:00', 'calendar': 'noleap'}) + testname = 'FAIL: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) print_test_message(testname, indata=indata, expected=UnitsError) self.assertRaises(UnitsError, N1.__getitem__, slice(None)) def test_dimensions_error(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'dimensions': ('y',)} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('y'),)) + testname = 'FAIL: ValidateNode({!r}).__getitem__(:)'.format(indata) expected = DimensionsError + V = ValidateNode(indata, N0) print_test_message(testname, indata=indata, expected=expected) - self.assertRaises(expected, ValidateNode, 'validate(x)', N0, **indata) + self.assertRaises(expected, V.__getitem__, slice(None)) def test_min_warn(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'valid_min': 2} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'valid_min': 2}) + testname = 'WARN: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -599,10 +620,9 @@ def test_min_warn(self): def test_max_warn(self): N0 = DataNode(PhysArray(numpy.arange(10), name='x', units='m', dimensions=('x',))) - indata = {'valid_max': 8} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'valid_max': 8}) + testname = 'WARN: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -612,10 +632,9 @@ def test_max_warn(self): def test_min_mean_abs_warn(self): N0 = DataNode(PhysArray(numpy.arange(-5, 10), name='x', units='m', dimensions=('x',))) - indata = {'ok_min_mean_abs': 5} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'ok_min_mean_abs': 5}) + testname = 'WARN: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -625,10 +644,9 @@ def test_min_mean_abs_warn(self): def test_max_mean_abs_warn(self): N0 = DataNode(PhysArray(numpy.arange(-5, 10), name='x', units='m', dimensions=('x',))) - indata = {'ok_max_mean_abs': 3} - testname = ('WARN: ValidateNode({}).__getitem__(:)' - '').format(', '.join('{!s}={!r}'.format(k, v) for k, v in indata.iteritems())) - N1 = ValidateNode('validate(x)', N0, attributes=indata) + indata = VariableDesc('validate(x)', dimensions=(DimensionDesc('x'),), attributes={'ok_max_mean_abs': 3}) + testname = 'WARN: ValidateNode({!r}).__getitem__(:)'.format(indata) + N1 = ValidateNode(indata, N0) actual = N1[:] expected = N0[:] print_test_message(testname, indata=indata, actual=actual, expected=expected) @@ -648,31 +666,32 @@ class WriteNodeTests(unittest.TestCase): def setUp(self): NX = 15 X0 = -5 - xdata = PhysArray(numpy.arange(X0, X0+NX, dtype='f'), - name='X', units='m', dimensions=('x',)) + xdata = PhysArray(numpy.arange(X0, X0+NX, dtype='d'), name='X', units='m', dimensions=('x',)) NY = 8 Y0 = 0 - ydata = PhysArray(numpy.arange(Y0, Y0+NY, dtype='f'), - name='Y', units='m', dimensions=('y',)) + ydata = PhysArray(numpy.arange(Y0, Y0+NY, dtype='d'), name='Y', units='m', dimensions=('y',)) - vdata = PhysArray(numpy.arange(0, NX*NY, dtype='d').reshape(NX,NY), - name='V', units='K', dimensions=('x', 'y')) + NT = 3 + tunits = Unit('days since 2000-01-01', calendar='noleap') + tdata = PhysArray(numpy.arange(0, NT, dtype='d'), name='T', units=tunits, dimensions=('t',)) + + vdata = PhysArray(numpy.arange(0, NX*NY*NT, dtype='f').reshape(NX,NY,NT), name='V', units='K', + dimensions=('x', 'y', 't')) - self.data = {'X': xdata, 'Y': ydata, 'V': vdata} - self.atts = {'X': {'xa1': 'x attribute 1', 'xa2': 'x attribute 2', 'axis': 'X'}, + self.data = {'X': xdata, 'Y': ydata, 'T': tdata, 'V': vdata} + self.atts = {'X': {'xa1': 'x attribute 1', 'xa2': 'x attribute 2', 'axis': 'X', 'units': str(xdata.units)}, 'Y': {'ya1': 'y attribute 1', 'ya2': 'y attribute 2', 'axis': 'Y', - 'direction': 'decreasing'}, - 'V': {'va1': 'v attribute 1', 'va2': 'v attribute 2'}} - self.nodes = {n:ValidateNode(n, DataNode(self.data[n]), attributes=self.atts[n]) - for n in self.data} + 'direction': 'decreasing', 'units': str(ydata.units)}, + 'T': {'axis': 'T', 'ta1': 'time attribute', 'units': str(tdata.units), + 'calendar': tdata.units.calendar}, + 'V': {'va1': 'v attribute 1', 'va2': 'v attribute 2', 'units': str(vdata.units)}} - dimdescs = {n:DimensionDesc(n, s) for x in self.data.itervalues() - for n, s in zip(x.dimensions, x.shape)} + dimdescs = {n:DimensionDesc(n, s) for x in self.data.itervalues() for n, s in zip(x.dimensions, x.shape)} vardescs = {n:VariableDesc(n, datatype=self.data[n].dtype, attributes=self.atts[n], - dimensions=[dimdescs[d] for d in self.data[n].dimensions]) - for n in self.data} + dimensions=[dimdescs[d] for d in self.data[n].dimensions]) for n in self.data} self.vardescs = vardescs + self.nodes = {n:ValidateNode(self.vardescs[n], DataNode(self.data[n])) for n in self.data} def tearDown(self): for fname in glob('*.nc'): @@ -772,6 +791,35 @@ def test_execute_simple_default(self): self.assertEqual(actual, expected, '{} failed'.format(testname)) print_ncfile(filename) + def test_execute_simple_autoparse(self): + filename = 'v.{%Y%m%d-%Y%m%d}.nc' + testname = 'WriteNode({}).execute()'.format(filename) + filedesc = FileDesc(filename, variables=self.vardescs.values(), attributes={'ga': 'global attribute'}) + N = WriteNode(filedesc, inputs=self.nodes.values()) + N.enable_history() + N.execute() + newfname = 'v.20000101-20000103.nc' + actual = exists(newfname) + expected = True + print_test_message(testname, actual=actual, expected=expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + print_ncfile(newfname) + + def test_execute_simple_autoparse_fail(self): + filename = 'v.{%Y%m%d-%Y%m%d}.nc' + testname = 'WriteNode({}).execute()'.format(filename) + vdescs = {n:self.vardescs[n] for n in self.vardescs if n != 'T'} + filedesc = FileDesc(filename, variables=vdescs.values(), attributes={'ga': 'global attribute'}) + vnodes = {n:self.nodes[n] for n in self.nodes if n != 'T'} + N = WriteNode(filedesc, inputs=vnodes.values()) + N.enable_history() + N.execute() + actual = exists(filename) + expected = True + print_test_message(testname, actual=actual, expected=expected) + self.assertEqual(actual, expected, '{} failed'.format(testname)) + print_ncfile(filename) + def test_execute_simple_nc3(self): filename = 'v_x_y_simple_nc3.nc' testname = 'WriteNode({}).execute()'.format(filename) diff --git a/source/pyconform/test/functionsTests.py b/source/test/functionsTests.py similarity index 100% rename from source/pyconform/test/functionsTests.py rename to source/test/functionsTests.py diff --git a/source/pyconform/test/indexingTests.py b/source/test/indexingTests.py similarity index 100% rename from source/pyconform/test/indexingTests.py rename to source/test/indexingTests.py diff --git a/source/pyconform/test/makeTestData.py b/source/test/makeTestData.py similarity index 100% rename from source/pyconform/test/makeTestData.py rename to source/test/makeTestData.py diff --git a/source/pyconform/test/mapdatesTests.py b/source/test/mapdatesTests.py similarity index 62% rename from source/pyconform/test/mapdatesTests.py rename to source/test/mapdatesTests.py index 2f94ba1d..b980fe83 100644 --- a/source/pyconform/test/mapdatesTests.py +++ b/source/test/mapdatesTests.py @@ -7,48 +7,51 @@ import numpy as np from cStringIO import StringIO -from pyconform.test.makeTestData import DataMaker +from makeTestData import DataMaker from pyconform import mapdates import unittest -files=['test1.nc', 'test2.nc', 'test3.nc'] +files = ['test1.nc', 'test2.nc', 'test3.nc'] -#=============================================================================== +#========================================================================= # dateMapTests -#=============================================================================== +#========================================================================= + + class dateMapTests(unittest.TestCase): def test_get_files_in_order(self): # Create some data that should be contiguous - t = {'time': [3,3,3], 'space': 2} + t = {'time': [3, 3, 3], 'space': 2} dm = DataMaker(dimensions=t, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values - ordered_files,counts,error = mapdates.get_files_in_order(files) + ordered_files, counts, error = mapdates.get_files_in_order(files) self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], "get_files_in_order, ordered_list".format()) - self.assertTrue(counts == [3,3,3], + self.assertTrue(counts == [3, 3, 3], "get_files_in_order, counts".format()) self.assertTrue(error == 0, "get_files_in_order, error".format()) dm.clear() def test_get_files_out_of_order(self): - + # Create some data that is not contiguous - a = np.asarray([1,2,3]) - b = np.asarray([7,8,9]) - c = np.asarray([4,5,6]) - t = {'time': [3,3,3], 'space': 2} - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + a = np.asarray([1, 2, 3]) + b = np.asarray([7, 8, 9]) + c = np.asarray([4, 5, 6]) + t = {'time': [3, 3, 3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values - ordered_files,counts,error = mapdates.get_files_in_order(files) + ordered_files, counts, error = mapdates.get_files_in_order(files) self.assertTrue(ordered_files == ['test1.nc', 'test3.nc', 'test2.nc'], "get_files_in_order, ordered_list".format()) - self.assertTrue(counts == [3,3,3], + self.assertTrue(counts == [3, 3, 3], "get_files_in_order, counts".format()) self.assertTrue(error == 0, "get_files_in_order, error".format()) dm.clear() @@ -56,14 +59,16 @@ def test_get_files_out_of_order(self): def test_get_files_gap_between_files(self): # Create some data that is not contiguous - a = np.asarray([1,2,3]) - b = np.asarray([4,5,6]) - c = np.asarray([10,11,12]) - t = {'time': [3,3,3], 'space': 2} - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + a = np.asarray([1, 2, 3]) + b = np.asarray([4, 5, 6]) + c = np.asarray([10, 11, 12]) + t = {'time': [3, 3, 3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() - # Call get_files_in_order and evaluate the return values, return value should fail + # Call get_files_in_order and evaluate the return values, return value + # should fail _ordered_files, _counts, error = mapdates.get_files_in_order(files) self.assertTrue(error == 1, "get_files_in_order, error".format()) dm.clear() @@ -71,14 +76,16 @@ def test_get_files_gap_between_files(self): def test_get_files_gap_in_a_file(self): # Create some data that is not contiguous - a = np.asarray([1,2,4]) - b = np.asarray([5,6,7]) - c = np.asarray([8,9,10]) - t = {'time': [3,3,3], 'space': 2} - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + a = np.asarray([1, 2, 4]) + b = np.asarray([5, 6, 7]) + c = np.asarray([8, 9, 10]) + t = {'time': [3, 3, 3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() - # Call get_files_in_order and evaluate the return values, return value should fail + # Call get_files_in_order and evaluate the return values, return value + # should fail _ordered_files, _counts, error = mapdates.get_files_in_order(files) self.assertTrue(error == 1, "get_files_in_order, error".format()) dm.clear() @@ -88,18 +95,19 @@ def test_get_files_6hr(self): from collections import OrderedDict # Create some data that is not contiguous - a = np.asarray([1,1.25,1.50,1.75]) - b = np.asarray([2,2.25,2.50,2.75]) - c = np.asarray([3,3.25,3.50,3.75]) - t = OrderedDict([('time', [4,4,4]),('space', 2)]) - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + a = np.asarray([1, 1.25, 1.50, 1.75]) + b = np.asarray([2, 2.25, 2.50, 2.75]) + c = np.asarray([3, 3.25, 3.50, 3.75]) + t = OrderedDict([('time', [4, 4, 4]), ('space', 2)]) + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values - ordered_files,counts,error = mapdates.get_files_in_order(files) + ordered_files, counts, error = mapdates.get_files_in_order(files) self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], "get_files_in_order, ordered_list".format()) - self.assertTrue(counts == [4,4,4], + self.assertTrue(counts == [4, 4, 4], "get_files_in_order, counts".format()) self.assertTrue(error == 0, "get_files_in_order, error".format()) dm.clear() @@ -108,24 +116,26 @@ def test_get_files_monthly(self): from collections import OrderedDict - # Create some data - a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334, 365]) - b1=[] - c1=[] - for i in range(0,12): - b1.append(a[i]+365) - c1.append(a[i]+730) + # Create some data + a = np.asarray([31, 59, 90, 120, 151, 181, + 212, 243, 273, 304, 334, 365]) + b1 = [] + c1 = [] + for i in range(0, 12): + b1.append(a[i] + 365) + c1.append(a[i] + 730) b = np.asarray(b1) - c = np.asarray(c1) - t = OrderedDict([('time', [12,12,12]),('space', 2)]) - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + c = np.asarray(c1) + t = OrderedDict([('time', [12, 12, 12]), ('space', 2)]) + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values - ordered_files,counts,error = mapdates.get_files_in_order(files) + ordered_files, counts, error = mapdates.get_files_in_order(files) self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], "get_files_in_order, ordered_list".format()) - self.assertTrue(counts == [12,12,12], + self.assertTrue(counts == [12, 12, 12], "get_files_in_order, counts".format()) self.assertTrue(error == 0, "get_files_in_order, error".format()) dm.clear() @@ -136,15 +146,16 @@ def test_get_files_monthly_gap_between_files(self): # Create some data that is not contiguous a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]) - b1=[] - c1=[] - for i in range(0,11): - b1.append(a[i]+365) - c1.append(a[i]+730) + b1 = [] + c1 = [] + for i in range(0, 11): + b1.append(a[i] + 365) + c1.append(a[i] + 730) b = np.asarray(b1) c = np.asarray(c1) - t = OrderedDict([('time', [11,11,11]),('space', 2)]) - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + t = OrderedDict([('time', [11, 11, 11]), ('space', 2)]) + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values @@ -158,15 +169,16 @@ def test_get_files_monthly_gap_in_a_file(self): # Create some data that is not contiguous a = np.asarray([31, 59, 90, 120, 151, 181, 212, 243, 273, 334, 365]) - b1=[] - c1=[] - for i in range(0,11): - b1.append(a[i]+365) - c1.append(a[i]+730) + b1 = [] + c1 = [] + for i in range(0, 11): + b1.append(a[i] + 365) + c1.append(a[i] + 730) b = np.asarray(b1) c = np.asarray(c1) - t = OrderedDict([('time', [11,11,11]),('space', 2)]) - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + t = OrderedDict([('time', [11, 11, 11]), ('space', 2)]) + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values @@ -176,26 +188,28 @@ def test_get_files_monthly_gap_in_a_file(self): def test_get_files_yearly(self): - # Create some data + # Create some data a = np.asarray([365, 730, 1095]) b = np.asarray([1460, 1825, 2190]) c = np.asarray([2555, 2920, 3285]) - t = {'time': [3,3,3], 'space': 2} - dm = DataMaker(dimensions=t, vardata={'time': [a,b,c]}, filenames=files) + t = {'time': [3, 3, 3], 'space': 2} + dm = DataMaker(dimensions=t, vardata={ + 'time': [a, b, c]}, filenames=files) dm.write() # Call get_files_in_order and evaluate the return values - ordered_files,counts,error = mapdates.get_files_in_order(files) + ordered_files, counts, error = mapdates.get_files_in_order(files) self.assertTrue(ordered_files == ['test1.nc', 'test2.nc', 'test3.nc'], "get_files_in_order, ordered_list".format()) - self.assertTrue(counts == [3,3,3], + self.assertTrue(counts == [3, 3, 3], "get_files_in_order, counts".format()) self.assertTrue(error == 0, "get_files_in_order, error".format()) dm.clear() -#=============================================================================== + +#========================================================================= # Command-Line Operation -#=============================================================================== +#========================================================================= if __name__ == "__main__": hline = '=' * 70 print hline @@ -210,5 +224,3 @@ def test_get_files_yearly(self): print 'TESTS RESULTS:' print hline print str(mystream.getvalue()) - - diff --git a/source/pyconform/test/parsingTests.py b/source/test/parsingTests.py similarity index 100% rename from source/pyconform/test/parsingTests.py rename to source/test/parsingTests.py diff --git a/source/pyconform/test/physarrayTests.py b/source/test/physarrayTests.py similarity index 100% rename from source/pyconform/test/physarrayTests.py rename to source/test/physarrayTests.py diff --git a/source/pyconform/test/testutils.py b/source/test/testutils.py similarity index 100% rename from source/pyconform/test/testutils.py rename to source/test/testutils.py