Merge pull request #53 from NCAR/devel

Updating master with latest devel branch and docs

.gitignore

@@ -1,3 +1,5 @@
 *.spec
 *.pyc
 *.nc
+.DS_Store
+temp

.pydevproject

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <?eclipse-pydev version="1.0"?><pydev_project>
-
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
 <pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Homebrew Python 2.7</pydev_property>
 <pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">

CHANGES.rst

@@ -1,26 +1,35 @@
 PyConform ChangeLog
 ===================
 
-Copyright 2015, University Corporation for Atmospheric Research
+Copyright 2017, University Corporation for Atmospheric Research
 See the LICENSE.rst file for details
 
 VERSION 0.0.1
 -------------
-06 Jan 2016:
  - Add date check/mapping ability.
-
-19 Nov 2015:
  - Added initial CESM to CMIP6 JSON tables to the examples directory.
  - Added the code within examples/CESM/CMIP6/src used to generate these tables.
-
-18 Nov 2015:
  - Added initial CESM to CMIP5 JSON tables to examples directory
-
-17 Nov 2015:
  - Add initial version of mip_table_parser.py.
-
-03 Nov 2015:
  - Add initial versions of climIO.py and its unit test.
-
-30 Oct 2015:
  - Working repository created.  Template in place.
+
+VERSION 0.1.0
+-------------
+ - Pre-release of Version 0.1.0
+ - Many improvements and features
+ - Demo with CMIP5 Amon table with the b40.rcp4_5-1deg.006 experiment data
+
+VERSION 0.2.0
+-------------
+ - Major refactor of the graph data structure and dependent objects
+ - Includes allowances for 'chunking' (serialization) of the data
+ - Uses the SimpleComm interface for parallelism
+ - Many other improvements and simplifications
+
+VERSION 0.2.1
+-------------
+ - Automatic interpretation of "direction" attribute in coordinate variables
+ - Simplified PhysArray API
+ - More powerful Dataset Descriptors
+

README.rst

@@ -5,7 +5,7 @@ A package for transforming a NetCDF dataset into a defined format
 suitable for publication according to a defined publication standard.
 
 :AUTHORS: Sheri Mickelson, Kevin Paul
-:COPYRIGHT: 2016, University Corporation for Atmospheric Research
+:COPYRIGHT: 2017, University Corporation for Atmospheric Research
 :LICENSE: See the LICENSE.rst file for details
 
 Send questions and comments to Kevin Paul (kpaul@ucar.edu) or
@@ -23,16 +23,16 @@ Dependencies
 
 The PyConform package directly depends upon ...
 
-Currently the explicit dependencies are known to be:
+The major dependencies are known to be:
 
-* ASAPTools (>=0.4)
-* PyNIO (>=1.4.1)
-* mpi4py (>=1.3)
+* ASAPTools (>=0.6)
+* netCDF4-python
 
-This implies the dependencies:
+These dependencies imply the dependencies:
 
-* PyNIO depends upon numpy (>=1.5) and netCDF
-* mpi4py depends on numpy (>=1.4) and MPI
+* numpy (>=1.5)
+* netCDF4
+* MPI
 
 Additionally, the entire package is designed to work with Python v2.7 and up
 to (but not including) Python v3.0.

docs/_config.yml

@@ -0,0 +1,40 @@
+# Name of your blog (this will show up at the top of your page and in the RSS feed)
+name: PyConform
+
+# Short description (goes below the title; it will also be used in the RSS feed)
+description: A tool for standardizing NetCDF datasets for Model Intercomparison Projects
+
+# Your name, as you want it to appear underneath each post and in the footer
+author: Kevin Paul
+
+# Your email if you want it to be linked on the contact page
+author_email: kpaul@ucar.edu
+
+# The directory for category index pages. Change it to something else if
+# for example you want links like /categories/category1 instead of /category1
+category_dir: /
+
+# Uncomment if you are planning to run the blog in a subdirectory
+# Note - if you enable this, and attempt to view your site locally you have to use the baseurl in your local path.
+# Example, you must use http://localhost:4000/path/to/blog
+#baseurl: /path/to/blog 
+baseurl: 
+
+# The URL of your actual domain. This will be used to make absolute links in the RSS feed.
+#url: http://yourdomain.com/
+
+#### Under the Hood Stuff #####
+
+# Use rdiscount as the markdown engine because it generates html5 compliant code for stuff like footnotes
+# If you use maroku (default engine) some of your generated pages may not validate or lint as html5
+# If you don't have it install it via gem install rdiscount
+markdown: rdiscount
+
+# Makes pretty (descriptive) permalinks. See Jekyll docs for alternatives.
+permalink: pretty
+
+# How many articles do you wish to appear on the front page:
+paginate: 3
+
+# Exclude metadata and development time dependencies (like Grunt plugins)
+exclude: [README.markdown, package.json, grunt.js, Gruntfile.js, Gruntfile.coffee, node_modules]

examples/CESM/CMIP5/cmip5funcs.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+
+from Ngl import vinth2p
+from pyconform.physarray import PhysArray, UnitsError, DimensionsError
+from pyconform.functions import Function, is_constant
+from cf_units import Unit
+from numpy import diff, empty
+
+
+#=======================================================================================================================
+# BoundsFunction
+#=======================================================================================================================
+class BoundsFunction(Function):
+    key = 'bounds'
+
+    def __init__(self, data, bdim='bnds', location=1, endpoints=1, idata=None):
+        super(BoundsFunction, self).__init__(data, bdim=bdim, location=location, endpoints=endpoints, idata=idata)
+        data_info = data if is_constant(data) else data[None]
+        if not isinstance(data_info, PhysArray):
+            raise TypeError('bounds: data must be a PhysArray')
+        if not isinstance(bdim, basestring):
+            raise TypeError('bounds: bounds dimension name must be a string')
+        if location not in [0,1,2]:
+            raise ValueError('bounds: location must be 0, 1, or 2')
+        if len(data_info.dimensions) != 1:
+            raise DimensionsError('bounds: data can only be 1D')
+        self._mod_end = bool(endpoints)
+        self.add_sumlike_dimensions(data_info.dimensions[0])
+        if idata is None:
+            self._compute_idata = True
+        else:
+            self._compute_idata = False
+            idata_info = idata if is_constant(idata) else idata[None]
+            if not isinstance(idata_info, PhysArray):
+                raise TypeError('bounds: interface-data must be a PhysArray')
+            if len(idata_info.dimensions) != 1:
+                raise DimensionsError('bounds: interface-data can only be 1D')
+            self.add_sumlike_dimensions(idata_info.dimensions[0])
+
+    def __getitem__(self, index):
+        data = self.arguments[0][index]
+        bdim = self.keywords['bdim']
+        location = self.keywords['location']
+
+        bnds = PhysArray([1, 1], dimensions=(bdim,))
+        new_data = PhysArray(data * bnds, name='bounds({})'.format(data.name))
+        if index is None:
+            return new_data
+
+        if self._compute_idata:
+            dx = diff(data.data)
+            if location == 0:
+                new_data[:-1,1] = data.data[:-1] + dx
+                if self._mod_end:
+                    new_data[-1,1] = data.data[-1] + dx[-1]
+            elif location == 1:
+                hdx = 0.5 * dx
+                new_data[1:,0] = data.data[1:] - hdx
+                new_data[:-1,1] = data.data[:-1] + hdx
+                if self._mod_end:
+                    new_data[0,0] = data.data[0] - hdx[0]
+                    new_data[-1,1] = data.data[-1] + hdx[-1]
+            elif location == 2:
+                new_data[1:,0] = data.data[1:] - dx
+                if self._mod_end:
+                    new_data[0,0] = data.data[0] - dx[0]
+            return new_data
+
+        else:
+            ddim = data.dimensions[0]
+            dslice = index[ddim] if ddim in index else slice(None)
+            islice = slice(None, None, dslice.step)
+            idata = self.keywords['idata'][islice]
+
+            ifc_len = len(data) + 1
+            ifc_data = empty(ifc_len, dtype=data.dtype)
+            if len(idata) == ifc_len:
+                ifc_data[:] = idata.data[:]
+            elif len(idata) == ifc_len - 2:
+                ifc_data[1:-1] = idata.data[:]
+                if location == 0:
+                    ifc_data[0] = data.data[0]
+                    ifc_data[-1] = 2*data.data[-1] - data.data[-2]
+                elif location == 1:
+                    ifc_data[0] = 2*data.data[0] - idata.data[0]
+                    ifc_data[-1] = 2*data.data[-1] - idata.data[-1]
+                else:
+                    ifc_data[0] = 2*data.data[0] - data.data[1]
+                    ifc_data[-1] = data.data[-1]
+            else:
+                raise ValueError('bounds: interface-data length is {} but should be {} or '
+                                 '{}'.format(len(idata), ifc_len, ifc_len-2))
+
+            new_data[:,0] = ifc_data[:-1]
+            new_data[:,1] = ifc_data[1:]
+
+        return new_data
+
+
+#=======================================================================================================================
+# VertInterpFunction
+#=======================================================================================================================
+class VertInterpFunction(Function):
+    key = 'vinth2p'
+
+    def __init__(self, datai, hbcofa, hbcofb, plevo, psfc, p0, intyp=1, ixtrp=0):
+        super(VertInterpFunction, self).__init__(datai, hbcofa, hbcofb, plevo, psfc, p0, intyp=intyp, ixtrp=ixtrp)
+        datai_info = datai if is_constant(datai) else datai[None]
+        hbcofa_info = hbcofa if is_constant(hbcofa) else hbcofa[None]
+        hbcofb_info = hbcofb if is_constant(hbcofb) else hbcofb[None]
+        plevo_info = plevo if is_constant(plevo) else plevo[None]
+        psfc_info = psfc if is_constant(psfc) else psfc[None]
+        p0_info = p0 if is_constant(p0) else p0[None]
+
+        if not all(isinstance(obj, PhysArray)
+                   for obj in (datai_info, hbcofa_info, hbcofb_info, plevo_info, psfc_info, p0_info)):
+            raise TypeError('vinth2p: arrays must be PhysArrays')
+
+        if len(datai_info.dimensions) != 3 and len(datai_info.dimensions) != 4:
+            raise DimensionsError('vinth2p: interpolated data must be 3D or 4D')
+        if len(hbcofa_info.dimensions) != 1 or len(hbcofb_info.dimensions) != 1:
+            raise DimensionsError('vinth2p: hybrid a/b coefficients must be 1D')
+        if len(plevo_info.dimensions) != 1:
+            raise DimensionsError('vinth2p: output pressure levels must be 1D')
+        if len(p0_info.dimensions) != 0:
+            raise DimensionsError('vinth2p: reference pressure must be scalar')
+
+        dlevi = hbcofa_info.dimensions[0]
+        if dlevi != hbcofb_info.dimensions[0]:
+            raise DimensionsError('vinth2p: hybrid a/b coefficients do not have same dimensions')
+        dlevo = plevo_info.dimensions[0]
+        self.add_sumlike_dimensions(dlevi, dlevo)
+
+        for d in psfc_info.dimensions:
+            if d not in datai_info.dimensions:
+                raise DimensionsError(('vinth2p: surface pressure dimension {!r} not found '
+                                       'in input data dimensions').format(d))
+        dlat, dlon = psfc_info.dimensions[-2:]
+
+        if (dlevi, dlat, dlon) != datai_info.dimensions[-3:]:
+            raise DimensionsError(('vinth2p: input data dimensions {} inconsistent with the '
+                                   'dimensions of surface pressure {} and hybrid coefficients {}'
+                                   '').format(datai_info.dimensions, psfc_info.dimensions, hbcofa_info.dimensions))
+
+        ilev = datai_info.dimensions.index(dlevi)
+
+        new_dims = [d for d in datai_info.dimensions]
+        new_dims[ilev] = dlevo
+        self._new_dims = tuple(new_dims)
+
+        self._new_name = 'vinth2p({}, plevs={})'.format(datai_info.name, plevo_info.name)
+
+    def __getitem__(self, index):
+        datai = self.arguments[0][index]
+        hbcofa = self.arguments[1][index]
+        hbcofb = self.arguments[2][index]
+        plevo = self.arguments[3][index].convert('mbar')
+        psfc = self.arguments[4][index].convert('Pa')
+        p0 = self.arguments[5][index].convert('mbar')
+        intyp = self.keywords['intyp']
+        ixtrp = self.keywords['ixtrp']
+
+        return PhysArray(vinth2p(datai.data, hbcofa.data, hbcofb.data, plevo.data,
+                                 psfc.data, intyp, p0.data, 1, bool(ixtrp)), name=self._new_name,
+                         dimensions=self._new_dims, units=datai.units, positive=datai.positive)

examples/CESM/CMIP5/definitions/CMIP5_3hr.def

@@ -0,0 +1,22 @@
+clt = CLDTOT
+hfls = LHFLX
+hfss = SHFLX
+huss = QREFHT
+lat = lat
+lat_bnds = bounds(lat, endpoints=0)
+lon = lon
+lon_bnds = bounds(lon)
+pr = PRECC + PRECL
+prc = PRECC
+prsn = PRECSC + PRECSL
+ps = PS
+rlds = FLDS
+rldscs = FLDSC
+rlus = FLDS + FLNS
+rsds = FSDS
+rsdscs = FSDSC
+rsus = FSDS - FSNS
+rsuscs = FSDSC - FSNSC
+tas = TREFHT
+time = mean(time, "tbnd")
+time_bnds = time_bnds

examples/CESM/CMIP5/definitions/CMIP5_Amon.def

@@ -0,0 +1,64 @@
+a = hyam
+a_bnds = bounds(hyam)
+b = hybm
+b_bnds = bounds(hybm)
+ch4 = vinth2p(CH4, hyam, hybm, plev, PS, P0)
+ci = chunits(FREQZM, units="1")
+cl = chunits(CLOUD, units="%")
+cli = CLDICE
+clivi = TGCLDIWP
+clt = chunits(CLDTOT, units="1")
+clw = CLDLIQ
+clwvi = TGCLDLWP
+co2 = vinth2p(CO2, hyam, hybm, plev, PS, P0) * (28.966/44.0)
+evspsbl = QFLX
+hfls = up(LHFLX)
+hfss = up(SHFLX)
+hur = vinth2p(RELHUM, hyam, hybm, plev, PS, P0)
+hurs = chunits(RHREFHT, units="%")
+hus = vinth2p(Q, hyam, hybm, plev, PS, P0)
+huss = QREFHT
+lat = lat
+lat_bnds = bounds(lat, endpoints=0)
+lev = lev
+lev_bnds = bounds(lev)
+lon = lon
+lon_bnds = bounds(lon)
+mc = up(CMFMC + CMFMCDZM)
+n2o = vinth2p(N2O, hyam, hybm, plev, PS, P0)
+p0 = P0
+pr = chunits(PRECC + PRECL, units="kg m-2 s-1")
+prc = chunits(PRECC, units="kg m-2 s-1")
+prsn = chunits(PRECSC + PRECSL, units="kg m-2 s-1")
+prw = TMQ
+ps = PS
+psl = PSL
+rlds = down(FLDS)
+rldscs = down(FLDSC)
+rlus = up(FLDS + FLNS)
+rlut = up(FSNTOA-FSNT+FLNT)
+rlutcs = up(FLUTC)
+rsds = down(FSDS)
+rsdscs = down(FSDSC)
+rsdt = down(SOLIN)
+rsus = up(FSDS - FSNS)
+rsuscs = up(FSDSC - FSNSC)
+rsut = up(SOLIN - FSNTOA)
+rsutcs = up(SOLIN - FSNTOAC)
+rtmt = down(FSNT - FLNT)
+sci = chunits(FREQSH, units="1")
+sfcWind = U10
+ta = vinth2p(T, hyam, hybm, plev, PS, P0)
+tas = TREFHT
+tasmax = TREFMXAV
+tasmin = TREFMNAV
+tauu = up(TAUX)
+tauv = up(TAUY)
+time = chunits(mean(time_bnds, "tbnd"), units=time)
+time_bnds = time_bnds
+tro3 = vinth2p(O3, hyam, hybm, plev, PS, P0)
+ts = TS
+ua = vinth2p(U, hyam, hybm, plev, PS, P0)
+va = vinth2p(V, hyam, hybm, plev, PS, P0)
+wap = vinth2p(OMEGA, hyam, hybm, plev, PS, P0)
+zg = vinth2p(Z3, hyam, hybm, plev, PS, P0)