arcapi.py

"""
#-------------------------------------------------------------------------------
# Name:        arcapi.arcapi
# Purpose:     Core module for convenient API for arcpy
#
# Authors:     Filip Kral, Caleb Mackey
#
# Created:     01/02/2014
# Licence:     LGPL v3
#-------------------------------------------------------------------------------
# The core module of the arcapi package.
#
# All this content gets imported when you import the arcapi package.
# See arcapi __init__.py for more notes.
#
#-------------------------------------------------------------------------------
"""

import os
import sys
import time
import httplib
import urllib
import urllib2
import json
import urlparse
from contextlib import closing
import datetime

try:
    import arcpy
except ImportError:
    from ArcpyMockup import ArcpyMockup
    arcpy = ArcpyMockup()


__version__ = '0.3.0'
"""Version number of arcapi"""


def names(x, filterer = None):
    """Return list of column names of a table.

    Required:
    x -- input table or table view

    Optional:
    filterer -- function, only fields where filterer returns True are listed

    Example:
    >>> names('c:\\foo\\bar.shp', lambda f: f.name.startswith('eggs'))
    """
    flds = arcpy.ListFields(x)
    if filterer is None: filterer = lambda a: True
    return [f.name for f in flds if filterer(f)]


def types(x, filterer = None):
    """Return list of column types of a table.

    Required:
    x -- input table or table view

    Optional:
    filterer -- function, only fields where filterer returns True are listed

    Example:
    >>> types('c:\\foo\\bar.shp', lambda f: f.name.startswith('eggs'))
    """
    flds = arcpy.ListFields(x)
    if filterer is None: filterer = lambda a: True
    return [f.type for f in flds if filterer(f)]


def nrow(x):
    """Return number of rows in a table as integer.

    Required:
    x -- input table or table view

    Example:
    >>> nrow('c:\\foo\\bar.shp')
    """
    return int(arcpy.GetCount_management(x).getOutput(0))


def values(tbl, col, w='', o=None):
    """Return a list of all values in column col in table tbl.

    If col is a single column, returns a list of values, otherwise returns
    a list of tuples of values where each tuple is one row.

    Columns included in the o parameter must be included in the col parameter!

    Required:
    tbl -- input table or table view
    col -- input column name(s) as string or a list; valid options are:
        col='colA'
        col=['colA']
        col=['colA', 'colB', 'colC']
        col='colA,colB,colC'
        col='colA;colB;colC'

    Optional:
    w -- where clause
    o -- order by clause like '"OBJECTID" ASC, "Shape_Area" DESC',
        default is None, which means order by object id if exists

    Example:
    >>> values('c:\\foo\\bar.shp', 'Shape_Length')
    >>> values('c:\\foo\\bar.shp', 'SHAPE@XY')
    >>> values('c:\\foo\\bar.shp', 'SHAPE@XY;Shape_Length', 'Shape_Length ASC')
    >>> # columns in 'o' must be in 'col', otherwise RuntimeError is raised:
    >>> values('c:\\foo\\bar.shp', 'SHAPE@XY', 'Shape_Length DESC') # Error!
    """

    # unpack column names
    if isinstance(col, (list, tuple)):
        cols = col
    else:
        col = str(col)
        separ = ';' if ';' in col else ','
        cols = [c.strip() for c in col.split(separ)]

    # indicate whether one or more than one columns were specified
    multicols = False
    if len(cols) > 1:
        multicols = True

    # construct order by clause
    if o is not None:
        o = 'ORDER BY ' + str(o)
    else:
        pass

    # retrieve values with search cursor
    ret = []
    with arcpy.da.SearchCursor(tbl, cols, where_clause = w, sql_clause=(None, o)) as sc:
        for row in sc:
            if multicols:
                ret.append(row)
            else:
                ret.append(row[0])

    return ret


def frequency(x):
    """Return a dict of counts of each value in iterable x.

    Values in x must be hashable in order to work as dictionary keys.

    Required:
    x -- input iterable object like list or tuple

    Example:
    >>> frequency([1,1,2,3,4,4,4]) # {1: 2, 2: 1, 3: 1, 4: 3}
    >>> frequency(values('c:\\foo\\bar.shp', 'STATE'))
    """
    x.sort()
    fq = {}
    for i in x:
        if i in fq: #has_key deprecated in 3.x
            fq[i] += 1
        else:
            fq[i] = 1
    return fq


def distinct(tbl, col, w=''):
    """Return a list of distinct values in column col in table tbl.

    Required:
    tbl -- input table or table view
    col -- input column name as string

    Optional:
    w -- where clause

    Example:
    >>> distinct('c:\\foo\\bar.shp', "CATEGORY")
    >>> distinct('c:\\foo\\bar.shp', "SHAPE@XY")
    """
    return list(set(values(tbl, col, w)))


def print_tuples(x, delim=" ", tbl=None, geoms=None, fillchar=" ",  padding=1, verbose=True, returnit = False):
    """Print and/or return list of tuples formatted as a table.


    Intended for quick printing of lists of tuples in the terminal.
    Returns None or the formatted table depending on value of returnit.


    Required:
    x -- input list of tuples to print (can be tuple of tuples, list of lists).


    Optional:
    delim -- delimiter to use between columns
    tbl -- table or list of arcpy.Field objects to take column headings from (default is None)
    geoms -- if None (default), print geometries 'as is', else as str(geom).
        Works only is valid tbl is specified.
    filchar -- string to be used to pad values
    padding -- how many extra fillchars to use in cells
    verbose -- suppress printing when False, default is True
    returnit -- if True, return the formatted table, else return None (default)
    """
    lpadding, rpadding = padding, padding
    fch = fillchar
    # find column widths
    gi = None
    if tbl is None:
        nms = ["V" + str(a) for a in range(len(x[0]))]
        tps = ["LONG" if str(ti).isdigit() else "TEXT" for ti in x[0]]
        geoms = None
    else:
        nms,tps = [],[]
        i = 0
        if isinstance(tbl, list) or isinstance(tbl, tuple):
            fields = tbl
        else:
            fields = arcpy.ListFields(tbl)
        for f in fields:
            nms.append(f.name)
            tps.append(f.type)
            if f.type.lower() == "geometry" and geoms is not None:
                gi = i # index of geometry column
            i += 1
    nmirange = range(len(nms))
    toLeft = []
    leftTypes = ("STRING", "TEXT") # field types to be left justified
    for nmi in range(len(nms)):
        if tps[nmi].upper() in leftTypes:
            toLeft.append(nmi)
    widths = []
    for nmi in range(len(nms)):
        widths.append(len(str(nms[nmi])))
    for tpl in x:
        for nmi in range(len(nms)):
            if geoms is not None and nmi == gi:
                clen = len(str(geoms))
            else:
                clen = len(str(tpl[nmi]))
            if clen > widths[nmi]:
                widths[nmi] = clen


    sbuilder = []
    frmtd = []
    for nmi in range(len(nms)):
        pad = widths[nmi] + lpadding + rpadding
        frmtd.append(str(nms[nmi]).center(pad, fch))


    hdr = delim.join(frmtd)
    if verbose: print hdr # print header
    sbuilder.append(hdr)
    for r in x:
        frmtd = []
        for nmi in range(len(nms)):
            if nmi in toLeft:
                if geoms is not None and nmi == gi:
                    pad = widths[nmi] + rpadding
                    padfull = pad + lpadding
                    valf = str(geoms).ljust(pad, fch).rjust(padfull, fch)
                else:
                    pad = widths[nmi] + rpadding
                    padfull = pad + lpadding
                    valf = str(r[nmi]).ljust(pad, fch).rjust(padfull, fch)
            else:
                if geoms is not None and nmi == gi:
                    pad = widths[nmi] + lpadding
                    padfull = pad + rpadding
                    valf = str(geoms).rjust(pad, fch).ljust(padfull, fch)
                else:
                    pad = widths[nmi] + lpadding
                    padfull = pad + rpadding
                    valf = str(r[nmi]).rjust(pad, fch).ljust(padfull, fch)
            frmtd.append(valf)
        rw = delim.join(frmtd)


        if verbose:
            print rw # print row
        sbuilder.append(rw)


    ret = "\n".join(sbuilder) if returnit else None
    return ret


def head(tbl, n=10, t=True, delimiter="; ", geoms=None, cols=["*"], w="", verbose=True):
    """Return top rows of table tbl.


    Returns a list where the first element is a list of tuples representing
    first n rows of table tbl, second element is a dictionary like:
    {i: {"name":f.name, "values":[1,2,3,4 ...]}} for each field index i.


    Optional:
    n -- number of rows to read, default is 10
    t -- if True (default), columns are printed as rows, otherwise as columns
    delimiter -- string to be used to separate values (if t is True)
    geoms -- if None (default), print geometries 'as is', else as str(geom).
    cols -- list of columns to include, include all by default, case insensitive
    w, where clause to limit selection from tbl
    verbose -- suppress printing if False, default is True


    Example:
    >>> tmp = head('c:\\foo\\bar.shp', 5, True, "|", " ")
    """
    allcols = ['*', ['*'], ('*'), [], ()]
    colslower = [c.lower() for c in cols]
    flds = arcpy.ListFields(arcpy.Describe(tbl).catalogPath)
    if cols not in allcols:
        flds = [f for f in flds if f.name.lower() in colslower]
    fs = {}
    nflds = len(flds)
    fieldnames = []
    for i in range(nflds):
        f = flds[i]
        if cols in allcols or f.name in cols:
            fieldnames.append(f.name)
            fs.update({i: {"name":f.name, "values":[]}})
    i = 0
    hd = []
    with arcpy.da.SearchCursor(tbl, fieldnames, where_clause = w) as sc:
        for row in sc:
            i += 1
            if i > n: break
            hd.append(row)
            for j in range(nflds):
                fs[j]["values"].append(row[j])


    if t:
        labels = []
        values = []
        for fld in range(nflds):
            f = fs[fld]
            fl = flds[fld]
            labels.append(str(fl.name) + " (" + str(fl.type) +  "," + str(fl.length) + ")")
            if fl.type.lower() == 'geometry' and (geoms is not None):
                values.append(delimiter.join(map(str, len(f["values"]) * [geoms])))
            else:
                values.append(delimiter.join(map(str, f["values"])))
        longestLabel = max(map(len, labels))
        for l,v in zip(labels, values):
            toprint = l.ljust(longestLabel, ".") +  ": " + v
            arcpy.AddMessage(toprint)
            if verbose:
                print toprint
    else:
        if verbose:
            print_tuples(hd, delim=delimiter, tbl=flds, geoms=geoms, returnit=False)
    return [hd, fs]
def chart(x, out_file='c:\\temp\\chart.jpg', texts={}, template=None, resolution=95, openit=True):
    """Create and open a map (JPG) showing x and return path to the figure path.

    Required:
    x -- input feature class, raster dataset, or a layer

    Optional:
    out_file -- path to output jpeg file, default is 'c:\\temp\\chart.jpg'
    texts -- dict of strings to include in text elements on the map (by name)
    template -- path to the .mxd to be used, default None points to mxd with
        a single text element called "txt"
    resolution -- output resolution in DPI (dots per inch)
    openit -- if True (default), exported jpg is opened in a webbrowser

    Example:
    >>> chart('c:\\foo\\bar.shp')
    >>> chart('c:\\foo\\bar.shp', texts = {'txt': 'A Map'}, resolution = 300)
    """
    todel = []
    import re
    if template is None: template = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'chart.mxd')
    if not re.findall(".mxd", template, flags=re.IGNORECASE): template += ".mxd"
    if not re.findall(".jpe?g", out_file, flags=re.IGNORECASE): out_file += ".jpg"

    mxd = arcpy.mapping.MapDocument(template)
    if not arcpy.Exists(x):
        x = arcpy.CopyFeatures_management(x, arcpy.CreateScratchName('tmp', workspace = 'in_memory')).getOutput(0)
        todel = [x]
    dtype = arcpy.Describe(x).dataType
    df = arcpy.mapping.ListDataFrames(mxd)[0]

    lr = "chart" + tstamp(tf = "%H%M%S")
    if arcpy.Exists(lr) and arcpy.Describe(lr).dataType in ('FeatureLayer', 'RasterLayer'):
        arcpy.Delete_management(lr)
    if "raster" in dtype.lower():
        arcpy.MakeRasterLayer_management(x, lr)
    else:
        arcpy.MakeFeatureLayer_management(x, lr)

    lyr = arcpy.mapping.Layer(lr)
    arcpy.mapping.AddLayer(df, lyr)

    # try to update text elements if any requested:
    for tel in texts.iterkeys():
        try:
            texel = arcpy.mapping.ListLayoutElements(mxd, "TEXT_ELEMENT", tel)[0]
            texel.text = str(texts[tel])
        except Exception, e:
            arcpy.AddMessage("Error when updating text element " + str(tel) + ": "+ str(e))
    arcpy.RefreshActiveView()
    arcpy.mapping.ExportToJPEG(mxd, out_file, resolution=resolution)

    # cleanup
    arcpy.Delete_management(lr)
    del mxd
    if todel: arcpy.Delete_management(todel[0])

    # open the chart in a browser if requested
    if openit:
        import webbrowser
        webbrowser.open_new_tab(out_file)

    return arcpy.Describe(out_file).catalogPath


def plot(x, y=None, out_file="c:\\temp\\plot.png", main="Arcapi Plot", xlab="X", ylab="Y", pch="+", color="r", openit=True):
    """
    Create and display a plot (PNG) showing x (and y).

    Uses matplotlib.pyplot.scatter.

    Required:
    x -- values to plot on x axis

    Optional:
    y -- values to plot on y axis or None (default), then x will be plotted
        on y axis, using index for x axis.
    out_file -- path to output file, default is 'c:\\temp\\plot.png'
    main -- title of the plot
    xlab -- label for x axis
    ylab -- label for y axis
    pch
    color -- color of points:
        'r': red (default), 'b': blue, 'g': green, 'c': cyan, 'm': magenta,
        'y': yellow, 'k': black, 'w': white, hexadecimal code like '#eeefff',
        shades of grey as '0.75', 3-tuple like (0.1, 0.9, 0.5) for (R, G, B).
    pch -- character for matplotlib plot marks, default is '+', can also be:
        +: plus sign, .: dot, o: circle, *: star, p: pentagon, s:square, x: X,
        D: diamond, h: hexagon, ^: triangle
    openit -- if True (default), exported figure is opened in a webbrowser

    Example:
    >>> x = xrange(20)
    >>> plot(x)
    >>> plot(x, out_file='c:\\temp\\pic.png')
    >>> y = xrange(50,70)
    >>> plot(x, y, 'c:\\temp\\pic.png', 'Main', 'X [m]', 'Y [m]', 'o', 'k')
    """
    import re
    if not re.findall(".png", out_file, flags=re.IGNORECASE): out_file += ".png"

    if y is None:
        y = x
        len(x)
        x = xrange(len(y))
    lx = len(x)
    ly = len(y)
    if lx != ly:
        raise ArcapiError('x and y have different length, %s and %s' % (lx, ly))

    from matplotlib import pyplot as plt
    plt.scatter(x, y, c=color, marker=pch)
    plt.title(str(main))
    plt.xlabel(str(xlab))
    plt.ylabel(str(ylab))
    plt.savefig(out_file)
    plt.close()
    if openit:
        import webbrowser
        webbrowser.open_new_tab("file://" + out_file)
    return


def hist(x, out_file='c:\\temp\\hist.png', openit=True, **args):
    """
    Create and display a plot (PNG) showing histogram of x and return computed
    histogram of values, breaks, and patches.

    Uses matplotlib.pyplot.hist, for details see help(matplotlib.pyplot.hist).
    Draws an empty plot if x is empty.

    Required:
    x -- Input data (not empty!); histogram is computed over the flattened array.

    Optional:
    bins -- int or sequence of scalars defining the number of equal-width bins.
        or the bin edges including the rightmost edge. Default is 10.
    range -- (float, float), the lower and upper range of the bins,
        default is (min(x), max(x))
    normed -- counts normalized to form a probability density if True, default False
    weights -- array_like array of weights for each element of x.
    cumulative -- cumulative counts from left are calculated if True, default False
    histtype -- 'bar'(default)|'barstacked'|'step'|'stepfilled'
    align -- 'left'|'mid'(default)|'right' to align bars to bin edges.
    orientation -- 'horizontal'(default)|'vertical'
    rwidth -- relative width [0.0 to 1.0] of the bars, default is None (automatic)
    log -- if True, empty bins are filtered out and log scale set; default False
    color -- scalar or array-like, the colors of the bar faces:
        'r': red (default), 'b': blue, 'g': green, 'c': cyan, 'm': magenta,
        'y': yellow, 'k': black, 'w': white, hexadecimal code like '#eeefff',
        shades of grey as '0.75', 3-tuple like (0.1, 0.9, 0.5) for (R, G, B).
        Can also be full color and style specification.
    label -- label for legend if x has multiple datasets
    out_file -- path to output file, default is 'c:\\temp\\hist.png'
    main -- string, histogram main title
    xlab -- string, label for the ordinate (independent) axis
    openit -- if True (default), exported figure is opened in a webbrowser

    Example:
    >>> x = numpy.random.randn(10000)
    >>> hist(x)
    >>> hist(x, bins=20, color='r', main='A Title", xlab='Example')
    """
    import matplotlib.pyplot as plt

    # sort out parameters
    extras =  ('main', 'xlab', 'ylab')
    pars = dict([(k,v) for k,v in args.iteritems() if k not in extras])

    h = plt.hist(x, **pars)

    plt.title(str(args.get('main', 'Histogram')))
    xlab = str(args.get('xlab', 'Value'))
    ylab = 'Count'
    if args.get('Density', False):
        ylab = 'Density'

    if args.get('orientation', 'horizontal') == 'vertical':
        lab = xlab
        xlab = ylab
        ylab = lab

    plt.xlabel(str(xlab))
    plt.ylabel(str(ylab))
    plt.savefig(out_file)
    plt.close()
    if openit:
        import webbrowser
        webbrowser.open_new_tab("file://" + out_file)
    return h


def bars(x, out_file='c:\\temp\\hist.png', openit=True, **args):
    """
    Create and display a plot (PNG) showing barchart of x.

    Uses matplotlib.plt.bar, draws an empty plot if x is empty.
    Parameter width is always 1.0.

    Use matplotlib colors for coloring;
        'r': red, 'b': blue (default), 'g': green, 'c': cyan, 'm': magenta,
        'y': yellow, 'k': black, 'w': white, hexadecimal code like '#eeefff',
        shades of grey as '0.75', 3-tuple like (0.1, 0.9, 0.5) for (R, G, B).

    Required:
    x -- Input data. list-like of bar heights.

    Optional:
    color -- scalar or array-like, the colors of the bar faces
    edgecolor -- scalar or array-like, the colors of the bar edges
    linewidth -- scalar or array-like, default: None width of bar edge(s).
        If None, use default linewidth; If 0, don't draw edges.
    xerr -- scalar or array-like, use to generate errorbar(s) if not None (default)
    yerr -- scalar or array-like, use to generate errorbar(s) if not None (default)
    ecolor -- scalar or array-like, use as color of errorbar(s) if not None (default)
    capsize -- integer, length of error bar caps in points, default: 3
    orientation -- 'vertical'(default)|'horizontal', orientation of the bars
    log -- boolean, set the axis to be log scale if True, default is False
    # other
    out_file -- path to output file, default is 'c:\\temp\\hist.png'
    labels -- list-like of labels for each bar to display on x axis
    main -- string, histogram main title
    xlab -- string, label for the ordinate (independent) axis
    openit -- if True (default), exported figure is opened in a webbrowser

    Example:
    >>> x = [1,2,3,4,5]
    >>> lb = ['A','B','C','D','E']
    >>> bars(x)
    >>> bars(x, labels=lb, color='r', main='A Title', orientation='vertical')
    """
    import matplotlib.pyplot as plt
    import numpy

    width = 1.0
    # unpack arguments
    bpars = ['width', 'color', 'edgecolor', 'linewidth', 'xerr', 'yerr',
    'ecolor', 'capsize','error_kw', 'orientation', 'log']
    barpars = dict([(i, args.get(i, None)) for i in args if i in bpars])
    barpars['align'] = 'center'
    center = range(len(x))
    labels = args.get('labels', center)
    barpars['width'] = width
    orientation = barpars.get('orientation', 'vertical')

    fig, ax = plt.subplots()
    fig.canvas.draw()

    # the orientation parameter seems to have no effect on pyplot.bar, therefore
    # it is handled by calling barh instead of bar if orientation is horizontal
    if orientation == 'horizontal':
        a = barpars.pop('width', None)
        a = barpars.pop('orientation', None)
        plt.barh(center, x, **barpars)
    else:
        plt.bar(center, x, **barpars)

    xlab = str(args.get('xlab', 'Item'))
    ylab = str(args.get('ylab', 'Value'))
    if orientation == 'horizontal':
        lab = xlab
        xlab = ylab
        ylab = lab
        ax.set_yticks(center)
        ax.set_yticklabels(labels)
    else:
        ax.set_xticks(center)
        ax.set_xticklabels(labels)

    ax.set_xlabel(xlab)
    ax.set_ylabel(str(ylab))
    ax.set_title(str(args.get('main', 'Barplot')))
    plt.savefig(out_file)
    plt.close()
    if openit:
        import webbrowser
        webbrowser.open_new_tab("file://" + out_file)

    return


def pie(x, y=None, **kwargs):
    """
    Create and display a plot (PNG) showing pie chart of x.

    Uses matplotlib.pyplot.pie, draws an empty plot if x is empty.
    The fractional area of each wedge is given by x/sum(x).  If sum(x) <= 1,
    then the values of x will be used as the fractional area directly.

    Use matplotlib colors for coloring;
        'r': red, 'b': blue, 'g': green, 'c': cyan, 'm': magenta,
        'y': yellow, 'k': black, 'w': white, hexadecimal code like '#eeefff',
        shades of grey as '0.75', 3-tuple like (0.1, 0.9, 0.5) for (R, G, B).

    Required:
    x -- Input data. list-like of bar heights.

    Optional keyword arguments (see matplotlib.pyplot.pie for further details):
    y -- Groupping data - list of factor values, len(x) == len(y),
       values of x will be groupped by y and before the pie chart is plotted.
       If y is specified, labels will include the relevant y value.
    out_file -- output file, default is 'c:\\temp\\hist.png'
    color -- scalar or array-like, the colors of the bar faces
    labels -- list-like of labels for each wedge, or None for default labels
    explode -- scalar or array like for offsetting wedges, default None (0.0)
    main -- string, main title of the plot
    openit -- Open exported figure in a webbrowser? True(default)|False.
    autopct -- None, format string or function for labelling wedges.
    pctdistance -- scalar or array like for fine tuning placment of text
    labeldistance -- labels will be drawn this far from the pie (default is 1.1)
    shadow -- Shadow beneath the pie? False(default)|True.
    mainbox -- bbox properties of the main title, ignored if main is None
    tight -- Apply tight layout? True(default)|False

    Example:
    >>> x = [1,2,3,4,5]
    >>> lb = ['A','B','C','D','E']
    >>> pie(x)
    >>> pie(x, labels=lb, main='A Title')
    >>> pie([1,2,3,4,5,6,7], y=[1,1,2,2,3,3,3], autopct='%1.1f%%')
    >>> pie([1,2,3,4,5,6], y=[(1,'a'),(1,'a'),2,2,'b','b'], autopct='%1.1f%%')
    """
    import matplotlib.pyplot as plt

    # unpack arguments
    #y = kwargs.get('y', None) # more convenient to get as a named argument
    out_file =kwargs.get('out_file', 'c:\\temp\\hist.png')
    openit = kwargs.get('openit', True)
    #
    explode = kwargs.get('explode', None)
    labels = kwargs.get('labels', None)
    colors = kwargs.get('colors', ('b', 'g', 'r', 'c', 'm', 'y', 'k', 'w'))
    autopct = kwargs.get('autopct', None)
    pctdistance = kwargs.get('pctdistance', 0.6)
    labeldistance = kwargs.get('labeldistance', 1.1)
    shadow = kwargs.get('shadow', False)
    startangle = kwargs.get('startangle', 90)
    #
    main = kwargs.get('main', None)
    mainbox = kwargs.get('mainbox', None)
    legend = kwargs.get('legend', True)
    legloc = kwargs.get('legloc', 'best')
    tight = kwargs.get('tight', True)

    # handle the cases when y parameter is supplied
    # i.e. summarize x by y, construct labels etc.
    n = len(x)
    if y is not None:
        if n != len(y):
            raise ArcapiError("Lenghts of x and y must match, %s != %s" %
                (n, len(y))
            )

        freqs = {}
        for xi,yi in zip(x,y):
            if yi in freqs:
                freqs[yi] += xi
            else:
                freqs[yi] = xi

        x,y = [],[]
        for k,v in freqs.iteritems():
            x.append(v)
            y.append(k)
        labels = y

    # expand explode, labels, colors, etc. to the right length
    n = len(x)
    if explode is not None:
        if isinstance(explode, list) or isinstance(explode, tuple):
            if len(explode) != n:
                explode = ( explode * n )[0:n]
        else:
            explode = [explode] * n
    if labels is not None:
        if isinstance(labels, list) or isinstance(labels, tuple):
            if len(labels) != n:
                labels = ( labels * n )[0:n]
        else:
            labels = [labels] * n
    if colors is not None:
        if isinstance(colors, list) or isinstance(colors, tuple):
            if len(colors) != n:
                colors = ( colors * n )[0:n]
        else:
            colors = [colors] * n

    plt.figure(1)
    plt.subplot(1,1,1)
    pieresult = plt.pie(
        x,
        explode=explode,
        labels=labels,
        colors=colors,
        autopct=autopct,
        pctdistance=pctdistance,
        shadow=shadow,
        labeldistance=labeldistance
    )
    patches = pieresult[0]
    texts = pieresult[1]

    # add title
    if main is not None:
        plt.title(main, bbox=mainbox)

    # add legend
    if legend:
        if labels is None:
            labels = map(str, x)
            plt.legend(patches, labels, loc=legloc)

    # make output square and tight
    plt.axis('equal')
    if tight:
        plt.tight_layout()

    # save and display
    plt.savefig(out_file)
    plt.close()
    if openit:
        import webbrowser
        webbrowser.open_new_tab("file://" + out_file)

    return


def rename_col(tbl, col, newcol, alias = ''):
    """Rename column in table tbl and return the new name of the column.

    This function first adds column newcol, re-calculates values of col into it,
    and deletes column col.
    Uses arcpy.ValidateFieldName to adjust newcol if not valid.
    Raises ArcapiError if col is not found or if newcol already exists.

    Required:
    tbl -- table with the column to rename
    col -- name of the column to rename
    newcol -- new name of the column

    Optional:
    alias -- field alias for newcol, default is '' to use newcol for alias too
    """
    if col != newcol:
        d = arcpy.Describe(tbl)
        dcp = d.catalogPath
        flds = arcpy.ListFields(tbl)
        fnames = [f.name.lower() for f in flds]
        newcol = arcpy.ValidateFieldName(newcol, tbl) #os.path.dirname(dcp))
        if col.lower() not in fnames:
            raise ArcapiError("Field %s not found in %s." % (col, dcp))
        if newcol.lower() in fnames:
            raise ArcapiError("Field %s already exists in %s" % (newcol, dcp))
        oldF = [f for f in flds if f.name.lower() == col.lower()][0]
        if alias == "": alias = newcol
        arcpy.AddField_management(tbl, newcol, oldF.type, oldF.precision, oldF.scale, oldF.length, alias, oldF.isNullable, oldF.required, oldF.domain)
        arcpy.CalculateField_management(tbl, newcol, "!" + col + "!", "PYTHON_9.3")
        arcpy.DeleteField_management(tbl, col)
    return newcol


def tlist_to_table(x, out_tbl, cols, nullNumber=None, nullText=None):
    """Save a list of tuples as table out_tbl and return catalog path to it.

    Required:
    x -- list of tuples (no nesting!), can be list of lists or tuple of tuples
    out_tbl -- path to the output table
    cols -- list of tuples defining columns of x. Can be defined as:
        [('colname1', 'type1'), ('colname2', 'type2'), ...]
        ['colname1:type1:lgt1', 'colname2:type2', ('colname3', 'type3')]
        [('colname1', 'type1'), 'colname2:type2:lgt2, ...]
        where types are case insensitive members of:
        ('SHORT', 'SMALLINTEGER', 'LONG', 'INTEGER', 'TEXT', 'STRING', 'DOUBLE',
        'FLOAT')
        Each column definition can have third element for length of the field,
        e.g.: ('ATextColumn', 'TEXT', 250).
        To leave out length, simply leave it out or set to '#'

    Optional:
    nullNumber -- a value to replace null (None) values in numeric columns, default is None and does no replacement
    nullText -- a value to replace null (None) values in text columns, default is None and does no replacement

    Example:
    >>> x = [(...),(...),(...),(...),(...), ...]
    >>> ot = 'c:\\temp\\foo.dbf'
    >>> tlist_to_table(x, ot, [('IDO', 'SHORT'), ('NAME', 'TEXT', 200)]
    >>> tlist_to_table(x, ot, ['IDO:SHORT', 'NAME:TEXT:200']
    """
    # decode column names, types, and lengths
    cols = [tuple(c.split(":")) if type(c) not in (tuple, list) else c for c in cols]
    # remember what indexes to replace if values are null
    replaceNumbers, replacesText = [], []
    for i in range(len(cols)):
        if cols[i][1].upper() in ('TEXT', 'STRING'):
            replacesText.append(i)
        else:
            replaceNumbers.append(i)

    doReplaceNumber = False if nullNumber is None else True
    doReplaceText = False if nullText is None else True
    doReplace = doReplaceNumber or doReplaceText

    dname = os.path.dirname(out_tbl)
    if dname in('', u''): dname = arcpy.env.workspace
    r = arcpy.CreateTable_management(dname, os.path.basename(out_tbl))
    out_tbl = r.getOutput(0)
    # add the specified fields
    for f in cols:
        fname = f[0]
        ftype = f[1].upper()
        flength = '#'
        if len(f) > 2:
            flength = int(f[2]) if str(f[2]).isdigit() else '#'
        arcpy.AddField_management(out_tbl, fname, ftype, '#', '#', flength)
    # rewrite all tuples
    fields = [c[0] for c in cols]

    with arcpy.da.InsertCursor(out_tbl, fields) as ic:
        for rw in x:
            if doReplace:
                rw = list(rw)
                if i in replaceNumbers:
                    if rw[i] is None:
                        rw[i] = nullNumber
                if i in replacesText:
                    if rw[i] is None:
                        rw[i] = nullText
                rw = tuple(rw)
            ic.insertRow(rw)
    return out_tbl


def docu(x, n = None):
    """Print x.__doc__ string of the argument line by line using Python's print.

    Similar to builtin help() but allows to limit number of rows printed.

    Optional:
    n -- print only n first rows (or everything if n > number of rows or None)
    """
    dc = x.__doc__
    dc = dc.split("\n")
    nrows = len(dc)
    n = nrows if n is None else n
    n = min(n, nrows)
    j = 0
    for i in dc:
        print i
        j += 1
        if j == n: break
    return


def meta(datasource, mode="PREPEND", **args):
    """Read/write metadata of ArcGIS Feature Class, Raster Dataset, Table, etc.

    Returns a dictionary of all accessible (if readonly) or all editted entries.

    *** This function may irreversibly alter metadata, see details below! ***

    The following entries (XML elements) can be read or updated:
    Title ("dataIdInfo/idCitation/resTitle")
    Purpose ("dataIdInfo/idPurp")
    Abstract ("dataIdInfo/idAbs")

    This function exports metadata of the datasource to XML file using template
    'Metadata\Stylesheets\gpTools\exact copy of.xslt' from ArcGIS installation
    directory. Then it loads the exported XML file into memory using Pythons
    xml.etree.ElementTree, modifies supported elements, writes a new XML file,
    and imports this new XML file as metadata to the datasource.
    If the content of exported metada does not contain element dataInfo,
    it is assumend the metadata is not up to date with current ArcGIS version
    and UpgradeMetadata_conversion(datasource, 'ESRIISO_TO_ARCGIS') is applied!
    Try whether this function is appropriate for your work flows on dummy data.

    Required:
    datasource -- path to the data source to update metadata for
    mode -- {PREPEND|APPEND|OVERWRITE}, indicates whether new entries will be
        prepended or appended to existing entries, or whether new entries will
        overwrite existing entries. Case insensitive.
    **args, keyword arguments of type string indicating what entries to update:
        title, string to use in Title
        purpose, string to use in Purpose
        abstract, string to use in Abstract
        If no keyword argument is specifed, metadata are read only, not edited.

    Example:
    >>> fc = 'c:\\foo\\bar.shp'
    >>> meta(fc) # reads existing entries
    >>> meta(fc, 'OVERWRITE', title="Bar") # updates title
    >>> meta(fc, 'append', purpose='example', abstract='Column Spam means eggs')
    """
    import xml.etree.ElementTree as ET
    xslt = None # metadata template, could be exposed as a parameter
    sf = arcpy.env.scratchFolder
    tmpmetadatafile = arcpy.CreateScratchName('tmpmetadatafile', workspace=sf)

    # checks
    if xslt is None:
        template = 'Metadata\Stylesheets\gpTools\exact copy of.xslt'
        arcdir = arcpy.GetInstallInfo()['InstallDir']
        xslt = os.path.join(arcdir, template)
    if not os.path.isfile(xslt):
        raise ArcapiError("Cannot find xslt file " + str(xslt))
    mode = mode.upper()

    lut_name_by_node = {
        'dataIdInfo/idCitation/resTitle': 'title',
        'dataIdInfo/idPurp': 'purpose',
        'dataIdInfo/idAbs': 'abstract'
    }

    # work
    r = arcpy.XSLTransform_conversion(datasource, xslt, tmpmetadatafile)
    tmpmetadatafile = r.getOutput(0)
    with file(tmpmetadatafile, "r") as f:
        mf = f.read()
    tree = ET.fromstring(mf)

    # check if read-only access requested (no entries supplied)
    readonly = True if len(args) == 0 else False
    reader = {}
    if readonly:
        args = {'title':'', 'purpose':'', 'abstract': ''}
    else:
        # Update the metadata version if it is not up to date
        if tree.find('dataIdInfo') is None:
            arcpy.conversion.UpgradeMetadata(datasource, 'ESRIISO_TO_ARCGIS')
            os.remove(tmpmetadatafile)
            r = arcpy.XSLTransform_conversion(datasource, xslt, tmpmetadatafile)
            tmpmetadatafile = r.getOutput(0)
            with file(tmpmetadatafile, "r") as f:
                mf = f.read()
            tree = ET.fromstring(mf)

    # get what user wants to update
    entries = {}
    if args.get('title', None) is not None:
        entries.update({'dataIdInfo/idCitation/resTitle': args.get('title')})
    if args.get('purpose', None) is not None:
        entries.update({'dataIdInfo/idPurp': args.get('purpose')})
    if args.get('abstract', None) is not None:
        entries.update({'dataIdInfo/idAbs': args.get('abstract')})

    # update entries
    for p,t in entries.iteritems():
        el = tree.find(p)
        if el is None:
            if not readonly:
                wm = "Element %s not found, creating it from scratch." % str(p)
                arcpy.AddWarning(wm)
                pparent = "/".join(p.split("/")[:-1])
                parent = tree.find(pparent)
                if parent is None:
                    em = "Could not find parent %s as parent of %s in %s " % \
                        (pparent, p, str(datasource))
                    raise ArcapiError(em)
                subel = ET.SubElement(parent, p.split("/")[-1])
                subel.text = str(t)
                el = subel
                del subel
        else:
            if not readonly:
                pre, mid, post = ('', '', '')
                if mode != "OVERWRITE":
                    # remember existing content if not overwrite
                    mid = '' if el.text is None else el.text
                    joiner = '&lt;br/&gt;'
                else:
                    mid = str('' if t is None else t)
                    joiner = ''
                if mode == 'APPEND': post = str('' if t is None else t)
                if mode == 'PREPEND': pre = str('' if t is None else t)
                el.text = joiner.join((pre, mid, post))
        reader.update({lut_name_by_node[p]: getattr(el, 'text', None)})

    # write a new xml file to be imported
    mf = ET.tostring(tree)
    with file(tmpmetadatafile, "w") as f:
        f.write(mf)

    # import new xml file as metadata
    r = arcpy.MetadataImporter_conversion(tmpmetadatafile, datasource)
    msg("Updated metadata for " +  str(datasource))

    # try to clean up
    try: os.remove(tmpmetadatafile)
    except: pass

    return reader


def msg(x, timef='%Y-%m-%d %H:%M:%S', verbose=True, log=None, level='message'):
    """Print (and optionally log) a message using print and arcpy.AddMessage.

    In python console, arcpy.AddMessage does not work but print does.
    A message like 'P:2014-02-16 20:44:35: foo' is printed.
    In geoprocessing windows, print does not work but arcpy.AddMessage does,
    A message like 'T:2014-02-16 20:44:35: foo' is printed.
    In Windows command line, both messages are printed.

    arcpy.AddWarning is used if level is 'warning'
    arcpy.AddError is used if level is 'error', sys.exit() is called then.

    If log file does not exist, it is created, otherwise message is appended.

    Required:
    x -- content of the message

    Optional:
    timef -- time format, default is "%Y-%m-%d %H:%M:%S" (YYYY-MM-DD HH:MM:SS)
    verbose -- if True (default) print the message to the console
    log -- file to append the message to, the default is None (i.e. no appending)
    level -- one of 'message'|'warning'|'error' or 0|1|2 respectively

    Example:
    >>> msg('foo') # P:2014-02-16 20:44:35: foo
    >>> msg('foo', '%H%M%S') # P:204503: foo
    >>> msg('foo', '%H%M%S', True, 'c:\\temp\\log.txt') # P:204531: foo
    """
    x = str(x)
    level = str(level).lower()
    doexit = False
    tstamp = time.strftime(timef, time.localtime())
    if verbose:
        m = tstamp + ": " + x
        if level in ('message', '0'):
            print("P:" + m)
            arcpy.AddMessage("T:" + m)
        elif level in ('warning', '1'):
            print("W:" + m)
            arcpy.AddWarning("T:" + m)
        elif level in ('error', '2'):
            print("E:" + m)
            arcpy.AddError("T:" + m)
            doexit = True
        else:
            em = "Level %s not in 'message'|'warning'|'error'|0|1|2." % (level)
            raise ArcapiError(em)

    if log not in ("", None):
        with open(log, "a") as fl:
            fl.write("P:" + tstamp + ": " + x + "\n")

    if doexit:
        try: sys.exit()
        except: pass


def list_environments(x=[], printit=False):
    """Return a list of 2-tuples of all arcgis environments.

    Optional:
    x -- list of names of environment settings, default is empty list, i.e. all
    printit -- if True, a readable representation of the dictionary is printed using Python's print function.

    Example:
    >>> tmp = list_environments(['snapRaster', 'extent'], 1)
    >>> tmp = list_environments([], 1)
    """
    envs = [en for en in dir(arcpy.env) if not en.startswith("_") and en not in ('items', 'keys', 'iteritems', 'iterkeys', 'values')]
    if len(x) > 0:
        x = [i.lower() for i in x]
        envs = [en for en in envs if en.lower() in x]
    ret = []
    for en in envs:
        env = getattr(arcpy.env, en)
        if printit:
            print str(str(en) + " ").ljust(30, ".") + ": " + str(env)
        ret.append((en, env))
    return ret


def oidF(table):
    """Return name of the object ID field in table table"""
    return arcpy.Describe(table).OIDFieldName


def shpF(fc):
    """Return name of the Shape (Geometry) field in feature class fc"""
    return arcpy.Describe(fc).ShapeFieldName


def tstamp(p = "", tf="%Y%m%d%H%M%S", d="_", m=False, s=()):
    """Returns time stamped string.

    Return string like p + time in tf + d + s[0] + d + s[1] + d + ... s[n]
    If m is True, it will print a message too.

    Optional:
    p -- prefix
    tf -- fime format, default is "%Y%m%d%H%M%S" (i.e. YYYYMMDDHHMMSS)
    d -- delimiter between elements of s
    s -- tuple or list of postfixes

    Example:
    >>> ap.tstamp() # '20140216184029'
    >>> ap.tstamp("lr") # 'lr20140216184045'
    >>> ap.tstamp("lr", "%H%M%S") # 'lr184045'
    >>> ap.tstamp("lr", "%H%M%S") # 'lr184045'
    >>> ap.tstamp("lr", "%H%M%S", s=('run',1)) # 'lr184527_run_1'
    """
    bits = str(d).join(map(str, s))
    if bits: bits = d + bits
    stamp = str(p) + time.strftime(tf, time.localtime()) + bits
    if m: msg(stamp, "")
    return stamp


def dlt(x):
    """arcpy.Delete_management(x) if arcpy.Exists(x).

    Return False if x does not exist, True if x exists and was deleted.
    """
    deletted = False
    if arcpy.Exists(x):
        arcpy.Delete_management(x)
        deletted = True
    return deletted


def cleanup(x, verbose=False, **args):
    """Delete items in x and return number of items that could not be deleted.

    This function uses the dlt function, which in turn uses
    arcpy. Exists and arcpy.management.Delete. The deletion is wrapped in a try
    statement so failed deletions are skipped silently.

    Required:
    x -- iterable of items to delete

    Optional:
    verbose -- suppress messages if False (default), otherwise print messages
    **args -- keyword arguments 'timef' and 'log' for function 'msg'

    Example:
    >>> cleanup(['c:\\foo\\bar.shp', 'lyr', 'c:\\foo\\eggs.tif'])
    """
    cnt = 0
    for i in x:
        try:
            deleted = dlt(i)
        except:
            deleted = False
        if deleted:
            m = "Cleanup deleted " + str(i)
        else:
            m = "Cleanup could not delete " + str(i)
        msg(m, args.get('timef', '%Y-%m-%d %H:%M:%S'), verbose, args.get('log', None))
    return cnt


def to_points(tbl, out_fc, xcol, ycol, sr, zcol='#', w=''):
    """Convert table to point feature class, return path to the feature class.

    Required:
    tbl -- input table or table view
    out_fc -- path to output feature class
    xcol -- name of a column in tbl that stores x coordinates
    ycol -- name of a column in tbl that stores y coordinates
    sr -- spatial reference for out_fc
        sr can be either arcpy.SpatialReference object or a well known id as int

    Optional:
    zcol -- name of a column in tbl that stores y coordinates, default is '#'
    w -- where clause to limit the rows of tbl considered, default is ''

    Example:
    >>> t = 'c:\\foo\\bar.shp'
    >>> o = 'c:\\foo\\bar_pts.shp'
    >>> table_to_points(t, o, "XC", "YC", 4326, zcol='#', w='"FID" < 10')
    >>> table_to_points(t, o, "XC", "YC", arcpy.SpatialReference(27700))
    >>> table_to_points(t, o, "XC", "YC", arcpy.describe(tbl).spatialReference)
    """
    lrnm = tstamp('lr', '%m%d%H%M%S', '')
    if type(sr) != arcpy.SpatialReference:
        sr = arcpy.SpatialReference(sr)
    lr = arcpy.MakeXYEventLayer_management(tbl, xcol, ycol, lrnm, sr, zcol).getOutput(0)
    if str(w) not in ('', '*'):
        arcpy.SelectLayerByAttribute_management(lr, "NEW_SELECTION", w)
    out_fc = arcpy.CopyFeatures_management(lr, out_fc).getOutput(0)
    dlt(lr)
    return (arcpy.Describe(out_fc).catalogPath)


def update_col_from_dict(x, y, xcol, xidcol=None, xw='', na=None):
    """Update column in a table with values from a dictionary.

    Return number of updated records.

    Required:
    x -- table to update
    y -- dictionary with new values
    xcol -- name of the column of x to update

    Optional:
    xidcol -- column of x to be used as keys to look up values in y,
        default is None, which means to use object id field.
    xw -- where clause to select rows to update from x
    na -- value to be used instead of new value for non-matching records,
        default is None, use (1,1) to leave original value if match is not found

    Example:
    >>> fc = 'c:\\foo\\bar.shp'
    >>> d = {1: 'EN', 2:'ST', 3:'WL', 4:'NI'}
    >>> update_col_from_dict(fc, d, 'country_code')
    >>> update_col_from_dict(fc, d, 'country_num', 'country_code', na='Other')
    """
    if xidcol is None:
        xidcol = arcpy.Describe(x).OIDFieldName

    # indicate whether to leave nonmatching values unchenged or set to na
    identity = False if na != (1,1) else True

    if xcol == xidcol:
        selfupdate = True
        cols = [xidcol]
    else:
        selfupdate = False
        cols = [xidcol, xcol]

    cnt = 0
    with arcpy.da.UpdateCursor(x, cols, where_clause = xw) as uc:
        for row in uc:
            ido = row[0]

            if identity:

                # branch that leaves unmatched records unchanged
                if ido in y:
                    newval = y[ido]
                    if selfupdate:
                        row[0] = newval
                    else:
                        row[1] = newval
                    uc.updateRow(row)
                    cnt += 1
                else:
                    pass

            else:
                # branch that sets unmatched records to na
                newval = y.get(ido, na)
                if selfupdate:
                    row[0] = newval
                else:
                    row[1] = newval
                uc.updateRow(row)
                cnt += 1

    return cnt


def to_scratch(name, enforce=False):
    """Return path to a dataset called name in scratch workspace.

    LIMITATION: Reliable for geodatabases only! Does not handle extensions.

    Returns os.path.join(arcpy.env.scratchWorkspace, name).
    If scratchWorkspace is None, it tries workspace, then scratchGDB.

    This function 'to_scratch' has also an alias 'tos'!

    Required:
    name -- basename of the output dataset

    Optional:
    enforce -- if True, arcpy.CreateScratchName is used to ensure name does not
        exist in scratch workspace, otherwise returns basename equal to name.

    Example:
    >>> to_scratch('foo', 0) # '...\\scratch.gdb\\foo'
    >>> to_scratch('foo', 1) # '...\\scratch.gdb\\foo0'
    >>> to_scratch('foo.shp', 0) # '...\\scratch.gdb\\foo_shp'
    >>> to_scratch('foo.shp', 1) # '...\\scratch.gdb\\foo_shp0'
    >>> tos('foo', 0) # '...\\scratch.gdb\\foo'
    """
    ws = arcpy.env.scratchWorkspace
    if ws is None: ws = arcpy.env.workspace
    if ws is None: ws = arcpy.env.scratchGDB

    if arcpy.Describe(ws).workspaceType.lower() == 'filesystem':
        m = "Scratch workspace is a folder, scratch names may be incorrect."
        msg(m)
        arcpy.AddWarning(m)

    nm = os.path.basename(name)
    nm = arcpy.ValidateTableName(nm, ws)
    if enforce:
        nm = arcpy.CreateScratchName(nm, workspace=ws)
    else:
        nm = os.path.join(ws, nm)
    return nm


def wsp(ws = None):
    """Get or set arcpy.env.workspace and return its path.

    If ws is None and arcpy.env.workspace is None, this function will set
    arcpy.env.workspace to arcpy.env.scratchGDB and return its path.

    Optional:
    ws -- path to workspace, default is None.
        If ws is a non-existing file geodatabse, it will be created.

    Example:
    >>> # if executed in order
    >>> ev = arcpy.env
    >>> wsp() # sets ev.workspace = ec.scratchGDB if ev.workspace is None
    >>> wsp('c:\\temp') # sets ev.workspace = 'c:\\temp', returns 'c:\\temp'
    >>> wsp() # now returns 'c:\\temp'
    """
    if ws is None:
        ws = arcpy.env.workspace
        if ws is None:
            ws = arcpy.env.scratchGDB
            arcpy.env.workspace = ws
    else:
        if ws[-4:].lower() == '.gdb' and not arcpy.Exists(ws):
            import re
            ws = arcpy.management.CreateFileGDB(os.path.dirname(ws), re.sub(".gdb", "", os.path.basename(ws), re.IGNORECASE), "CURRENT").getOutput(0)
        arcpy.env.workspace = ws
    return arcpy.env.workspace


def swsp(ws = None):
    """Get or set arcpy.env.scratchWorkspace and return its path.

    If ws is None and arcpy.env.scratchWorkspace is None, this function will set
    arcpy.env.scratchWorkspace to arcpy.env.scratchGDB and return its path.

    This function 'swsp' has also an alias 'wsps'!

    Optional:
    ws -- path to scratch workspace, default is None
        If ws is a non-existing file geodatabse, it will be created.

    Example:
    >>> # if executed in order
    >>> ev = arcpy.env
    >>> swsp() # sets ev.scratchWorkspace = ec.scratchGDB if ev.scratchWorkspace is None
    >>> swsp('c:\\temp') # sets ev.scratchWorkspace = 'c:\\temp', returns 'c:\\temp'
    >>> swsp() # now returns 'c:\\temp'
    """
    if ws is None:
        ws = arcpy.env.scratchWorkspace
        if ws is None:
            ws = arcpy.env.scratchGDB
            arcpy.env.scratchWorkspace = ws
    else:
        if ws[-4:].lower() == '.gdb' and not arcpy.Exists(ws):
            import re
            ws = arcpy.management.CreateFileGDB(os.path.dirname(ws), re.sub(".gdb", "", os.path.basename(ws), re.IGNORECASE), "CURRENT").getOutput(0)
        arcpy.env.scratchWorkspace = ws
    return arcpy.env.scratchWorkspace


def summary(tbl, cols=['*'], modes=None, maxcats=10, w='', verbose=True):
    """Summary statistics about columns of a table.

    Required:
    tbl -- table

    Optional:
    cols -- list of columns to look at or ['*'] for all columns (default).
    modes -- list of columns of the same length of cols.
        allowed values are "NUM", "CAT", "IGNORE"
        mode_i indicates if column_i should be treated as numeric value,
        categorical variable, or if it should be ignored.
        Default is None, if which case mode is determined as follows:
            CAT for columns of type TEXT or STRING
            NUM for SHORT, SMALLINTEGER, LONG, INTEGER, DOUBLE, FLOAT
            IGNORE for all other types.
    maxcats -- maximum number of categories to keep track of for CAT columns
        Records of superfluous categories are counted together as ('...').
    w -- where clause to limit the rows of tbl considered, default is ''
    verbose -- suppress printing if False, default is True

    Example:
    >>> summary('c:\\foo\\bar.shp')
    >>> summary('c:\\foo\\bar.shp', ['smap', 'eggs'], ['NUM', 'CAT'])
    """
    cattypes = ('TEXT', 'STRING')
    numtypes = ('SHORT', 'SMALLINTEGER', 'LONG', 'INTEGER', 'DOUBLE', 'FLOAT')
    modetypes = ("NUM", "CAT", "IGNORE")
    fields = arcpy.ListFields(tbl)
    fields = dict([(f.name, f) for f in fields])
    if cols in([], ['*'], None):
        cols = fields.keys()

    if modes is None:
        modes = []
        for c in cols:
            fld = fields.get(c, None)
            if fld is None:
                raise ArcapiError("Column %s not found." % (c))
            fldtype = fld.type.upper()
            if fldtype in numtypes:
                modes.append("NUM")
            elif fldtype in cattypes:
                modes.append("CAT")
            else:
                modes.append("IGNORE")
    else:
        modes = [str(m).upper() for m in modes]
        if not set(modes).issubset(set(modetypes)):
            raise ArcapiError("modes can only be one of %s" % (str(modetypes)))

    nc = len(cols)
    cixs = range(nc)
    stats = {}
    for ci in cixs:
        colname = cols[ci]
        stats[ci] = {
            "col": colname,
            "type": getattr(fields.get(colname, None), 'type', None),
            "cats": {}, "min":None, "max":None, "n": 0, "na": 0
        }

    with arcpy.da.SearchCursor(tbl, cols, where_clause = w) as sc:
        for row in sc:
            for ci in cixs:
                mode = modes[ci]
                statsci = stats[ci]
                v = row[ci]
                if mode == "CAT":
                    cats = statsci["cats"]
                    if cats is not None:
                        ncats = len(cats)
                        if v in cats:
                            cats[v] += 1
                        else:
                            if ncats < maxcats:
                                cats[v] = 1
                            else:
                                cats[('...')] = cats.get(('...'), 0) + 1
                elif mode == "NUM":
                    if v is None:
                        statsci["na"] += 1
                    else:
                        statsci["n"] += 1
                        m = statsci["min"]
                        if m is None or v < m:
                            statsci["min"] = v
                        m = statsci["max"]
                        if m is None or v > m:
                            statsci["max"] = v
                        statsci["sum"] = statsci.get("sum", 0) + v
                else:
                    # mode is IGNORE
                    pass

        # calculate means
        for i in cixs:
            sm = stats[i].get('sum', None)
            n = stats[i]['n']
            if n > 0 and sm is not None:
                stats[i]['mean'] = sm / n

        if verbose:
            width = 10
            fulline = '-' * 40
            print fulline
            print str(tbl)
            print str(arcpy.Describe(tbl).catalogPath)
            print fulline
            for j,i in stats.iteritems():
                mode = modes[j]
                print 'COLUMN'.ljust(width) + ": " + str(i.get('col', None))
                print 'type'.ljust(width) + ": "+ str(i.get('type', None))
                if mode == "NUM":
                    print 'min'.ljust(width) + ": " + str(i.get('min', None))
                    print 'max'.ljust(width) + ": " + str(i.get('max', None))
                    print 'mean'.ljust(width) + ": " + str(i.get('mean', None))
                    print 'sum'.ljust(width) + ": " + str(i.get('sum', None))
                    print 'n'.ljust(width) + ": " + str(i.get('n', None))
                    print 'na'.ljust(width) + ": " + str(i.get('na', None))
                elif mode == "CAT":
                    cats = i["cats"]
                    if len(cats) > 0:
                        print "CATEGORIES:"
                        catable = sorted(zip(cats.keys(), cats.values()), key = lambda a: a[1], reverse = True)
                        print_tuples(catable)
                else:
                    pass
                print fulline
    return stats


def remap_sa(st, stop, step, n=1):
    """Create a spatial analyst format reclassify remap range (list)
    [[start value, end value, new value]...]

    >>> # ex: make range groups from 50 - 80
    >>> remap_sa(50, 80, 10)
    [[50, 60, 1], [60, 70, 2], [70, 80, 3]]

    st   -- start value (int)
    stop -- stop value (int)
    step -- step value for range (int)

    Optional:
    n -- new value interval, default is 1 (int)
    """

    tups = [[i,i+step] for i in range(st, stop, step)]
    return [[t] + [(tups.index(t)+1)*n] for t in tups]


def remap_3d(st, stop, step, n=1):
    """Create a 3D analyst format reclassify remap range (str)
    "start end new;..."

    Required:
    st --   start value (int)
    stop -- stop value (int)
    step -- step value for range (int)

    Optional:
    n -- new value interval, default is 1 (int)

    Example:
    >>> # make range groups from 50 - 80
    >>> remap_3d(50, 80, 10)
    '50 60 1;60 70 2;70 80 3'
    """

    tups = [[i,i+step] for i in range(st, stop, step)]
    return ';'.join(' '.join([str(i) for i in t] + [str((tups.index(t)+1)*n)]) for t in tups)


def find(pattern, path, sub_dirs=True):
    """Find files matching a wild card pattern.

    Parameters:
    pattern -- wild card search (str)
    path -- root directory to search
    sub_dirs -- search through all sub directories? default is True (bool)

    Example:
    >>> # find SQL databases (.mdf files)
    >>> find('*.mdf', r'\\ArcServer1\SDE')
    \\arcserver1\SDE\ALBT\Albertville.mdf
    \\arcserver1\SDE\ARLI\Arlington.mdf
    \\arcserver1\SDE\BELL\BellePlaine.mdf
    \\arcserver1\SDE\BGLK\BigLake.mdf
    """
    import fnmatch

    theFiles = []
    for path, dirs, files in os.walk(path):
        for filename in files:
            if fnmatch.fnmatch(filename, pattern):
                theFiles.append(os.path.abspath(os.path.join(path, filename)))
        if sub_dirs in [False, 'false', 0]:
            break
    return theFiles


def int_to_float(raster, out_raster, decimals):
    """Convert an Integer Raster to a Float Raster
    *** Requires spatial analyst extension ***

    E.g., for a cell with a value of 45750, using this tool with 3
    decimal places will give this cell a value of 45.750

    Required:
    raster -- input integer raster
    out_raster -- new float raster
    decimals -- number of places to to move decimal for each cell

    Example:
    >>> convertIntegerToFloat(r'C:\Temp\ndvi_int', r'C:\Temp\ndvi_float', 4)
    """
    try:
        import arcpy.sa as sa

        # check out license
        arcpy.CheckOutExtension('Spatial')
        fl_rast = sa.Float(arcpy.Raster(raster) / float(10**int(decimals)))
        try:
            fl_rast.save(out_raster)
        except:
            # having random issues with Esri GRID format, change to tiff
            #   if grid file is created
            if not arcpy.Exists(out_raster):
                out_raster = out_raster.split('.')[0] + '.tif'
                fl_rast.save(out_raster)
        try:
            arcpy.CalculateStatistics_management(out_raster)
            arcpy.BuildPyramids_management(out_raster)
        except:
            pass

        msg('Created: %s' %out_raster)
        arcpy.CheckInExtension('Spatial')
        return out_raster
    except ImportError:
        return 'Module arcpy.sa not found.'


def fill_no_data(in_raster, out_raster, w=5, h=5):
    """Fill "NoData" cells with mean values from focal statistics.

    Use a larger neighborhood for raster with large areas of no data cells.

    *** Requires spatial analyst extension ***

    Required:
    in_raster -- input raster
    out_raster -- output raster

    Optional:
    w -- search radius width for focal stats (rectangle)
    h -- search radius height for focal stats (rectangle)

    Example:
    >>> fill_no_data(r'C:\Temp\ndvi', r'C:\Temp\ndvi_filled', 10, 10)
    """
    try:
        import arcpy.sa as sa
        # Make Copy of Raster
        _dir, name = os.path.split(arcpy.Describe(in_raster).catalogPath)
        temp = os.path.join(_dir, 'rast_copyxxx')
        if arcpy.Exists(temp):
            arcpy.Delete_management(temp)
        arcpy.CopyRaster_management(in_raster, temp)

        # Fill NoData
        arcpy.CheckOutExtension('Spatial')
        filled = sa.Con(sa.IsNull(temp),sa.FocalStatistics(temp,sa.NbrRectangle(w,h),'MEAN'),temp)
        filled.save(out_raster)
        arcpy.BuildPyramids_management(out_raster)
        arcpy.CheckInExtension('Spatial')

        # Delete original and replace
        if arcpy.Exists(temp):
            arcpy.Delete_management(temp)
        msg('Filled NoData Cells in: %s' %out_raster)
        return out_raster
    except ImportError:
        return 'Module arcpy.sa not found.'


def meters_to_feet(in_dem, out_raster, factor=3.28084):
    """Convert DEM Z units by a factor, default factor converts m -> ft.
    *** Requires spatial analyst extension ***

    Required:
    in_dem -- input dem
    out_raster -- new raster with z values as feet
    factor -- number by which the input DEM is multiplied,
        default is 3.28084 to convert metres to feet.

    Example:
    >>> meters_to_feet(r'C:\Temp\dem_m', r'C:\Temp\dem_ft')
    """
    try:
        import arcpy.sa as sa
        arcpy.CheckOutExtension('Spatial')
        out = sa.Float(sa.Times(arcpy.Raster(in_dem), factor))
        try:
            out.save(out_raster)
        except:
            # having random issues with esri GRID format
            #  will try to create as tiff if it fails
            if not arcpy.Exists(out_raster):
                out_raster = out_raster.split('.')[0] + '.tif'
                out.save(out_raster)
        try:
            arcpy.CalculateStatistics_management(out_raster)
            arcpy.BuildPyramids_management(out_raster)
        except:
            pass
        arcpy.AddMessage('Created: %s' %out_raster)
        arcpy.CheckInExtension('Spatial')
        return out_raster
    except ImportError:
        return 'Module arcpy.sa not found'


def currentMxd():
    """Return handle to the CURRENT map document.
    ***Can be used only in an ArcMap session***
    """
    return arcpy.mapping.MapDocument("CURRENT")


def fixArgs(arg, arg_type=list):
    """Fixe arguments from a script tool.

    For example, when using a script tool with a multivalue parameter,
    it comes in as "val_a;val_b;val_c".  This function can automatically
    fix arguments based on the arg_type.
    Another example is the boolean type returned from a script tool -
    instead of True and False, it is returned as "true" and "false".

    Required:
    arg --  argument from script tool (arcpy.GetParameterAsText() or sys.argv[1]) (str)
    arg_type -- type to convert argument from script tool parameter. Default is list.

    Example:
    >>> # example of list returned from script tool multiparameter argument
    >>> arg = "val_a;val_b;val_c"
    >>> fixArgs(arg, list)
    ['val_a', 'val_b', 'val_c']
    """
    if arg_type == list:
        if isinstance(arg, str):
            # need to replace extra quotes for paths with spaces
            # or anything else that has a space in it
            return map(lambda a: a.replace("';'",";"), arg.split(';'))
        else:
            return list(arg)
    if arg_type == float:
        if arg != '#':
            return float(arg)
        else:
            return ''
    if arg_type == int:
        return int(arg)
    if arg_type == bool:
        if str(arg).lower() == 'true' or arg == 1:
            return True
        else:
            return False
    if arg_type == str:
        if arg in [None, '', '#']:
            return ''
    return arg


def copy_schema(template, new, sr=''):
    """Copy the schema (field definition) of a feature class or a table.

    Required:
    template -- template table or fc
    new -- new output fc or table

    Optional:
    sr -- spatial reference (only applies if fc) If no sr
          is defined, it will default to sr of template.

    Example:
    >>> copy_schema(r'C:\Temp\soils_city.shp', r'C:\Temp\soils_county.shp')
    """
    path, name = os.path.split(new)
    desc = arcpy.Describe(template)
    ftype = desc.dataType
    if 'table' in ftype.lower():
        arcpy.CreateTable_management(path, name, template)
    else:
        stype = desc.shapeType.upper()
        sm = 'SAME_AS_TEMPLATE'
        if not sr:
            sr = desc.spatialReference
        arcpy.CreateFeatureclass_management(path, name, stype, template, sm, sm, sr)
    return new


def make_poly_from_extent(ext, sr):
    """Make an arcpy polygon object from an input extent object.,Returns
    a polygon geometry object.

    Required:
    ext -- extent object
    sr -- spatial reference

    Example
    >>> ext = arcpy.Describe(fc).extent
    >>> sr = 4326  #WKID for WGS 84
    >>> poly = make_poly_from_extent(ext, sr)
    >>> arcpy.CopyFeatures_management(poly, r'C:\Temp\Project_boundary.shp')
    """
    array = arcpy.Array()
    array.add(ext.lowerLeft)
    array.add(ext.lowerRight)
    array.add(ext.upperRight)
    array.add(ext.upperLeft)
    array.add(ext.lowerLeft)
    return arcpy.Polygon(array, sr)


def list_all_fcs(gdb, wild = '*', ftype='All', rel=False):
    """Return a list of all feature classes in a geodatabase.

    if rel is True, only relative paths will be returned.  If
    false, the full path to each feature classs is returned
    Relative path Example:

    >>> # Return relative paths for fc
    >>> gdb = r'C:\TEMP\test.gdb'
    >>> for fc in getFCPaths(gdb, rel=True):
    >>> print fc

    Utilities\Storm_Mh
    Utilities\Storm_Cb
    Transportation\Roads


    Required:
    gdb -- geodatabase containing feature classes to be listed
    wild -- wildcard for feature classes. Default is "*"
    ftype -- feature class type. Default is 'All'
             Valid values:

             Annotation - Only annotation feature classes are returned.
             Arc - Only arc (or line) feature classes are returned.
             Dimension - Only dimension feature classes are returned.
             Edge - Only edge feature classes are returned.
             Junction - Only junction feature classes are returned.
             Label - Only label feature classes are returned.
             Line - Only line (or arc) feature classes are returned.
             Multipatch - Only multipatch feature classes are returned.
             Node - Only node feature classes are returned.
             Point - Only point feature classes are returned.
             Polygon - Only polygon feature classes are returned.
             Polyline - Only line (or arc) feature classes are returned.
             Region - Only region feature classes are returned.
             Route - Only route feature classes are returned.
             Tic - Only tic feature classes are returned.
             All - All datasets in the workspace. This is the default value.

    Optional:
    rel -- option to have relative paths. Default is false;
           will include full paths unless rel is set to True
    """
    # feature type (add all in case '' is returned
    # from script tool
    if not ftype:
        ftype = 'All'
    arcpy.env.workspace = gdb

    # loop through feature classes
    feats = []

    # Add top level fc's (not in feature data sets)
    feats += arcpy.ListFeatureClasses(wild, ftype)

    # loop through feature datasets
    for fd in arcpy.ListDatasets('*', 'Feature'):
        arcpy.env.workspace = fdws = os.path.join(gdb, fd)
        feats += [os.path.join(fd, fc) for fc in
                  arcpy.ListFeatureClasses(wild, ftype)]

    # return list of features, relative pathed or full pathed
    if rel:
        return sorted(feats)
    else:
        return sorted([os.path.join(gdb, ft) for ft in feats])


def field_list(in_fc, filterer=[], oid=True, shape=True, objects=False):
    """Return a list of fields or a list of field objects on input feature class.

    This function will handle list comprehensions to either return field names
    or field objects. Object ID and Geometry fields are omitted by default.

    Required:
    in_fc -- input feature class, feature layer,  table, or table view

    Optional:
    filterer -- Default is empty list.  If a list is passed in, it will list
                all fields
    oid -- Default is True.  If set to True, will list all fields
           excluding Geometry and other Geometry. (bool)
    shape -- Default is True.  If true will return the Geometry field (bool)
    object -- Default is False.  If set to true, will return field objects
              instead of field names allowing for access to field properties such as
              field.name, field.type, field.length, etc. (bool)

    Example:
    >>> field_list(r'C:\Temp\Counties.shp', ['STATE_FIPS', 'COUNTY_CODE'], objects=True)
    """

    # add exclude types and exclude fields
    ex_type = []
    if not oid:
        ex_type.append('OID')
    if not shape:
        ex_type.append('Geometry')
    exclude = map(lambda x: x.lower(), filterer)

    # return either field names or field objects
    if objects:
        return [f for f in arcpy.ListFields(in_fc)
                      if f.type not in ex_type
                      and f.name.lower() not in exclude]
    else:
        return [f.name.encode('utf-8') for f in arcpy.ListFields(in_fc)
                      if f.type not in ex_type
                      and f.name.lower() not in exclude]


def get_field_type(in_field, fc=''):
    """Converts esri field type returned from list fields or describe fields
    to format for adding fields to tables.

    Required:
    in_field -- field name to find field type. If no feature class
        is specified, the in_field paramter should be a describe of
        a field.type

    Optional:
    fc -- feature class or table.  If no feature class is specified,
        the in_field paramter should be a describe of a field.type

    Example
    >>> # field type of 'String' needs to be 'TEXT' to be added to table
    >>> # This is a text type field
    >>> # now get esri field type
    >>> print getFieldType(table, 'PARCEL_ID') #esri field.type return is 'String', we want 'TEXT'
    TEXT
    """
    if fc:
        field = [f.type for f in arcpy.ListFields(fc) if f.name == in_field][0]
    else:
        field = in_field
    if field in lut_field_types:
        return lut_field_types[field]
    else:
        return None


def match_field(table_or_list, pat, multi=False):
    """Return a list of field objects where name matches the specified pattern.

    Required:
    table_or_list -- input table or feature class or list of fields
    pat -- pattern to match to field

    Optional:
    multi: if True, will return a list of all matches,
           otherwise returns the first match

    Example:
    >>> match_field(r'C:\Temp\Counties.shp', 'county_*', True)
    ['COUNTY_CODE', 'COUNTY_FIPS']
    """

    import fnmatch

    if isinstance(table_or_list, list):
        fields = table_or_list
    else:
        fields = [f.name for f in arcpy.ListFields(table_or_list)]
    all_mats = []
    for f in fields:
        if fnmatch.fnmatch(f, pat):
            if not multi:
                return f
            else:
                all_mats.append(f)
    return all_mats

def add_fields_from_table(in_tab, template, add_fields=[]):
    """Add fields (schema only) from one table to another

    Required:
    in_tab -- input table
    template -- template table containing fields to add to in_tab
    add_fields -- fields from template table to add to input table (list)

    Example:
    >>> add_fields_from_table(parcels, permits, ['Permit_Num', 'Permit_Date'])
    """

    # fix args if args from script tool
    if isinstance(add_fields, str):
        add_fields = add_fields.split(';')

    # grab field types
    f_dict = dict((f.name, [get_field_type(f.type), f.length, f.aliasName]) for f in arcpy.ListFields(template))

    # Add fields
    for field in add_fields:
        if field in f_dict:
            f_ob = f_dict[field]
            arcpy.AddField_management(in_tab, field, f_ob[0], field_length=f_ob[1], field_alias=f_ob[2])
            msg('Added field: {0}'.format(field))
    return


def create_field_name(fc, new_field):
    """Return a valid field name that does not exist in fc and
    that is based on new_field.

    Required:
    fc -- feature class, feature layer, table, or table view
    new_field -- new field name, will be altered if field already exists

    Example:
    >>> fc = 'c:\\testing.gdb\\ne_110m_admin_0_countries'
    >>> create_field_name(fc, 'NEWCOL') # NEWCOL
    >>> create_field_name(fc, 'Shape') # Shape_1
    """

    # if fc is a table view or a feature layer, some fields may be hidden;
    # grab the data source to make sure all columns are examined
    fc = arcpy.Describe(fc).catalogPath
    new_field = arcpy.ValidateFieldName(new_field, os.path.dirname(fc))

    # maximum length of the new field name
    maxlen = 64
    dtype = arcpy.Describe(fc).dataType
    if dtype.lower() in ('dbasetable', 'shapefile'):
        maxlen = 10

    # field list
    fields = [f.name.lower() for f in arcpy.ListFields(fc)]

    # see if field already exists
    if new_field.lower() in fields:
        count = 1
        while new_field.lower() in fields:

            if count > 1000:
                raise ArcapiError('Maximum number of iterations reached in uniqueFieldName.')

            if len(new_field) > maxlen:
                ind = maxlen - (1 + len(str(count)))
                new_field = '{0}_{1}'.format(new_field[:ind], count)
                count += 1
            else:
                new_field = '{0}_{1}'.format(new_field, count)
                count += 1

    return new_field


def join_using_dict(source_table, in_field, join_table, join_key, join_values=[]):
    """Join values from one table to another using dictionary.

    Add fields from one table to another by using a dictionary rather
    than joining tables.  There must be a field with common values between the
    two tables to enable attribute matching.  Values from "join_key" field in
    "join_table" should be unique.  This function can be faster than a standard
    Join Tool for tables with only couple of hundreds or thousands of records.
    This function alters the input source_table.
    Returns path to the altered source table.

    source_table -- table to add fields
    in_field -- join field with common values from join_table
    join_table -- table containing fields to add
    join_key -- common field to match values to source_table
    join_values -- fields to add from join_table to source_table

    Example:
    >>> parcels = r'C:\Temp\Parcels.gdb\Parcels'
    >>> permits = r'C:\Temp\Parcels.gdb\Permits'
    >>> add_flds = ['PERMIT_NUM', 'PERMIT_DATE', 'NOTE']
    >>> join_using_dict(parcels, 'PIN', permits', 'PARCEL_ID', add_flds)
    """

    # test version for cursor type (data access module available @ 10.1 +)
    ver = arcpy.GetInstallInfo()['Version']
    dataAccess = False
    if ver != '10.0':
        dataAccess = True


    # Get Catalog path (for feature layers and table views)
    cat_path = arcpy.Describe(source_table).catalogPath

    # Find out if source table is NULLABLE
    if not os.path.splitext(cat_path)[1] in ['.dbf','.shp']:
        nullable = 'NULLABLE'
    else:
        nullable = 'NON_NULLABLE'

    # Add fields to be copied
    update_fields = []
    join_list = arcpy.ListFields(join_table)
    for field in join_list:
        ftype = field.type
        name = field.name
        length = field.length
        pres = field.precision
        scale = field.scale
        alias = field.aliasName
        domain = field.domain
        for fldb in join_values:
            if fldb == name:
                name = create_field_name(source_table, fldb)
                arcpy.AddField_management(source_table,name,ftype,pres,scale,length,alias,nullable,'',domain)
                msg("Added '%s' field to \"%s\"" %(name, os.path.basename(source_table)))
                update_fields.insert(join_values.index(fldb), name.encode('utf-8'))

    # update new fields
    path_dict = {}
    if dataAccess:

        # Create Dictionary
        join_values.insert(0, join_key)
        with arcpy.da.SearchCursor(join_table, join_values) as srows:
            for srow in srows:
                path_dict[srow[0]] = tuple(srow[1:])

        # Update Cursor
        update_index = list(range(len(update_fields)))
        row_index = list(x+1 for x in update_index)
        update_fields.insert(0, in_field)
        with arcpy.da.UpdateCursor(source_table, update_fields) as urows:
            for row in urows:
                if row[0] in path_dict:
                    try:
                        allVals =[path_dict[row[0]][i] for i in update_index]
                        for r,v in zip(row_index, allVals):
                            row[r] = v
                        urows.updateRow(row)
                    except: pass

    else:
        # version 10.0
        rows = arcpy.SearchCursor(join_table)
        for row in rows:
            path_dict[row.getValue(join_key)] = tuple(row.getValue(v) for v in join_values)
        del rows

        # Update Cursor
        rows = arcpy.UpdateCursor(source_table)
        for row in rows:
            theVal = row.getValue(in_field)
            if theVal in path_dict:
                try:
                    for i in range(len(update_fields)):
                        row.setValue(update_fields[i],path_dict[theVal][i])
                    rows.updateRow(row)
                except: pass
        del rows
    msg('Fields in "%s" updated successfully' %(os.path.basename(source_table)))
    return source_table


def concatenate(vals=[], delimiter='', number_only=False):
    """Concatenate a list of values using a specified delimiter.

    Required:
    vals -- list of values to concatenate

    Optional:
    delimiter -- separator for new concatenated string. Default is '' (no delimiter)
    number_only -- if True, only numbers in list will be used. Default is False (bool)
    """
    if number_only:
        return delimiter.join(''.join(str(i) for i in v if str(v).isdigit()) for v in vals)
    else:
        return delimiter.join(map(str, vals))


def concatenate_fields(table, new_field, length, fields=[], delimiter='', number_only=False):
    """Create a new field in a table and concatenate user defined fields.

    This can be used in situations such as creating a Section-Township-Range
    field from 3 different fields.
    Returns the field name that was added.

    Required:
    table -- Input table
    new_field -- new field name
    length -- field length
    fields -- list of fields to concatenate

    Optional:
    delimiter -- join value for concatenated fields
        (example: '-' , all fields will be delimited by dash)
    number_only -- if True, only numeric values from a text field are extracted.
        Default is False.

    Example:
    >>> sec = r'C:\Temp\Sections.shp'
    >>> concatenate_fields(sec, 'SEC_TWP_RNG', 15, ['SECTION', 'TOWNSHIP', 'RANGE'], '-')
    """

    # Add field
    new_field = create_field_name(table, new_field)
    arcpy.AddField_management(table, new_field, 'TEXT', field_length=length)

    # Concatenate fields
    if arcpy.GetInstallInfo()['Version'] != '10.0':
        # da cursor
        with arcpy.da.UpdateCursor(table, fields + [new_field]) as rows:
            for r in rows:
                r[-1] = concatenate(r[:-1], delimiter, number_only)
                rows.updateRow(r)

    else:
        # 10.0 cursor
        rows = arcpy.UpdateCursor(table)
        for r in rows:
            r.setValue(new_field, concatenate([r.getValue(f) for f in fields], delimiter, number_only))
            rows.updateRow(r)
        del r, rows
    return new_field


def list_data(top, **options):
    """Walk down a file structure and pick up all data sets (items).
    Returns a generator of full paths to the items.
    Uses arcpy.da.Walk to discover GIS data.

    Use the oneach parameter to do something with each item as it is discovered.

    Parameters:
        top -- full path to the root workspace to start from

    Optional keyword arguments:
        exclude -- Function that takes item as a parameter and returns True if
            the item should be skipped. Default is None, all items are listed.
        exclude_dir -- Function that takes the directory name as a parameter and
            returns True if the whole directory should be skipped.
            Default is None, all directories are listed.
        oneach -- Function that takes the item as a parameter.
            Default is None and does nothing
        onerror -- Function to handle errors, see arcpy.da.Walk help
        datatypes -- list of all data types to discover, see arcpy.da.Walk help
        type -- Feature and raster data types to discover, see arcpy.da.Walk help
            Feature: Multipatch, Multipoint, Point, Polygon, Polyline
            Raster: BIL, BIP, BMP, BSQ, DAT, GIF, GRID, IMG, JP2, JPG, PNG, TIF
        skippers -- iterable of strings, item is skipped if it contains a skipper
            Skippers are not case sensitive

    Example:
    >>> list_data(r'c:\temp')
    >>> skippers = (".txt", ".xls", ".ttf")
    >>> exclude = lambda a: "_expired2013" in a
    >>> list_data(r'c:\temp', exclude=exclude, skippers=skippers)
    """

    exclude = options.get('exclude', None)
    exclude_dir = options.get('exclude_dir', None)
    oneach = options.get('oneach', None)
    onerror = options.get('onerror', None)
    datatypes = options.get('datatypes', None)
    types = options.get('type', None)
    skippers = options.get('skippers', None)

    if skippers is not None:
        skippers = [str(sk).lower() for sk in skippers]

    for dirpath, dirnames, filenames in arcpy.da.Walk(top, topdown=True, onerror=onerror, followlinks=False, datatype=datatypes, type=types):

        if exclude_dir is not None:
            dirnames = [di for di in dirnames if not exclude_dir(di)]

        for filename in filenames:
            item = os.path.join(dirpath, filename)
            if exclude is not None:
                 # skip items for which exclude is True
                if exclude(item):
                    continue
            if skippers is not None:
                 # skip items that contain any skipper values
                if any([item.lower().find(sk) > -1 for sk in skippers]):
                    continue
            if oneach is not None:
                # handle item if 'oneach' handler provided
                oneach(item)
            yield item


def create_pie_chart(fig, table, case_field, data_field='', fig_title='', x=8.5, y=8.5, rounding=0):
    """Create a pie chart based on a case field and data field.

    If no data_field is specified, the pie chart slices reflect frequency
    of value in the case_field.

    WARNING: although this tool successfully creates a pie chart .png,
             it throws a C++ runtime error.  TODO: Need to investigate this.

    Required:
    fig -- output png file for pie chart
    table -- table for data to create pie chart
    case_field -- field that will be used to summarize values,
                  and also will provide the labels for the legend.

    Optional:
    data_field -- field with values for pie chart.  If no field is
                  specified, count of each value in case_field will be used
    fig_title -- title for the pie chart figure
    x --  size for y-axis side (in inches)
    y --  size for y-axis side (in inches)
    rounding -- rounding for pie chart legend labels.  Default is 0.

    Example:
    >>> wards = r'C:\Temp\Voting_Wards.shp'
    >>> figure = r'C:\Temp\Figures\Election_results.png'
    >>> create_pie_chart(figure, wards, 'CANDIDATE', 'NUM_VOTES', 'Election Results')
    """

    import pylab, numpy, re

    # make sure figure is .png or .jpg
    if not re.findall('.png', fig, flags=re.IGNORECASE):
        out_file += '.png'

    # rounding nested function, (rounding value of 0 from built in round does not return integer)
    def rnd(f, t, rounding):
        return round((f/float(t))*100, rounding) if rounding > 0 else int((f/float(t))*100)

    # Grab unique values
    with arcpy.da.SearchCursor(table, [case_field]) as rows:
        cases = sorted(list(set(r[0] for r in rows)))

    # if not data_field
    tmp_fld = 'cnt_xx_xx_'
    fields = [case_field, data_field]
    if not data_field:
        arcpy.AddField_management(table, tmp_fld, 'SHORT')
        with arcpy.da.UpdateCursor(table, [tmp_fld]) as rows:
            for r in rows:
                r[0] = 1
                rows.updateRow(r)
        fields = [case_field, tmp_fld]

    # vals for slices
    vals=[]

    # sum values
    sum_table = str(arcpy.Statistics_analysis(table, r'in_memory\sum_tab_xxx',
                                              [[fields[1], 'SUM']],
                                              fields[0]).getOutput(0))
    fields[1] = 'SUM_{0}'.format(fields[1])
    with arcpy.da.SearchCursor(sum_table, fields) as rows:
        for r in rows:
            vals.append([r[0],r[1]])

    # clean up tmp_fld if necessary
    if not data_field:
        if tmp_fld in [f.name for f in arcpy.ListFields(table)]:
            try:
                arcpy.DeleteField_management(table, tmp_fld)
            except:
                pass

    # Create Pie Charts
    the_fig = pylab.figure(figsize=(x, y))
    pylab.axes([0.1, 0.1, 0.8, 0.8])
    label = [v[0] for v in vals]
    fracs = [v[1] for v in vals]
    tot = [sum(fracs)] * len(fracs)
    if len(label) == len(fracs):
        cmap = pylab.plt.cm.prism
        color = cmap(numpy.linspace(0., 1., len(fracs)))
        pie_wedges = pylab.pie(fracs,colors=color,pctdistance=0.5, labeldistance=1.1)
        for wedge in pie_wedges[0]:
            wedge.set_edgecolor('white')
        pylab.legend(map(lambda x, f, t: '{0} ({1}, {2}%)'.format(x, f, rnd(f, t, rounding)),
                                                                  label, fracs, tot),
                                                                  loc=(0,0), prop={'size':8})
        pylab.title(fig_title)
        pylab.savefig(fig)
        msg('Created: %s' %fig)
    arcpy.Delete_management(sum_table)
    return fig


def combine_pdfs(out_pdf, pdf_path_or_list, wildcard=''):
    """Combine PDF documents using arcpy mapping  module

    Required:
    out_pdf -- output pdf document (.pdf)
    pdf_path_or_list -- list of pdf documents or folder
        path containing pdf documents.

    Optional:
    wildcard -- optional wildcard search (only applies
        when searching through paths)

    Example:
    >>> # test function with path
    >>> out_pdf = r'C:\Users\calebma\Desktop\test.pdf'
    >>> path = r'C:\Users\calebma\Desktop\pdfTest'
    >>> combine_pdfs(out_pdf, path)

    >>> # test function with list
    >>> out = r'C:\Users\calebma\Desktop\test2.pdf'
    >>> pdfs = [r'C:\Users\calebma\Desktop\pdfTest\Mailing_Labels5160.pdf',
                r'C:\Users\calebma\Desktop\pdfTest\Mailing_Taxpayer.pdf',
                r'C:\Users\calebma\Desktop\pdfTest\stfr.pdf']
    >>> combine_pdfs(out, pdfs)
    """

    import glob

    # Create new PDF document
    pdfDoc = arcpy.mapping.PDFDocumentCreate(out_pdf)

    # if list, use that to combine pdfs
    if isinstance(pdf_path_or_list, list):
        for pdf in pdf_path_or_list:
            pdfDoc.appendPages(pdf)
            msg('Added "%s" to "%s"' %(pdf, os.path.basename(out_pdf)))

    # search path to find pdfs
    elif isinstance(pdf_path_or_list, str):
        if os.path.exists(pdf_path_or_list):
            search = os.path.join(pdf_path_or_list,'{0}*.pdf'.format(wildcard))
            for pdf in sorted(glob.glob(search)):
                pdfDoc.appendPages(os.path.join(pdf_path_or_list, pdf))
                msg('Added "%s" to "%s"' %(pdf, os.path.basename(out_pdf)))

    # Save and close pdf document
    pdfDoc.saveAndClose()
    del pdfDoc
    msg('Created: %s' %out_pdf)
    return out_pdf


def request_http(url, data=None, data_type='text', headers={}):
    """Return result of an HTTP Request.

    Only GET and POST methods are supported. To issue a GET request, parameters
    must be encoded as part of the url and data must be None. To issue a POST
    request, parameters must be supplied as a dictionary for parameter data and
    the url must not include parameters.

    Parameters:
        url -- URL to issue the request to
    Optional:
        data -- dictionary of data to send
        data_type -- text(default)|xml|json|jsonp|pjson
            data is always obtained as text, but the this function
            can convert the text depending on the data_type parameter:
                text -- return the raw text as it is
                xml -- parse the text with xml.etree.ElementTree and return
                json -- parse the text with json.loads(text) and return
                jsonp,pjson -- parse the text with json.loads(text) and return
                    also, add parameter callback to the request
        header -- dictionary of headers to include in the request
    Example:
    >>> request('http://google.com')
    >>> u = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services'
    >>> request(u,{'f':'json'}, 'json')
    >>> request('http://epsg.io/4326.xml', None, 'xml')
    """

    result = ''
    callback = 'callmeback' # may not be used

    # prepare data
    data_type = str(data_type).lower()
    if data_type in ('jsonp', 'pjson'):
        if data is None:
            data = {}
        data['callback'] = callback

    if data is not None:
         data = urllib.urlencode(data)

    # make the request
    rq = urllib2.Request(url, data, headers)
    re = urllib2.urlopen(rq)
    rs = re.read()

    # handle result
    if data_type in ('json', 'jsonp', 'pjson'):
        rs = rs.strip()

        # strip callback function if present
        if rs.startswith(callback + '('):
            rs = rs.lstrip(callback + '(')
            rs = rs[:rs.rfind(')')]

        result = json.loads(rs)
    elif data_type == 'xml':
        from xml.etree import ElementTree as ET
        rs = rs.strip()
        result = ET.fromstring(rs)
    elif data_type == 'text':
        result = rs
    else:
        raise Exception('Unsupported data_type %s ' % data_type)

    return result


def request_https(url, data=None, data_type="text", headers={}):
    """Return result of an HTTPS Request.
    Uses urllib.HTTPSConnection to issue the request.

    Only GET and POST methods are supported. To issue a GET request, parameters
    must be encoded as part of the url and data must be None. To issue a POST
    request, parameters must be supplied as a dictionary for parameter data and
    the url must not include parameters.

    Parameters:
        url -- URL to issue the request to
    Optional:
        data -- dictionary of data to send
        data_type -- text(default)|xml|json|jsonp|pjson
            data is always obtained as text, but the this function
            can convert the text depending on the data_type parameter:
                text -- return the raw text as it is
                xml -- parse the text with xml.etree.ElementTree and return
                json -- parse the text with json.loads(text) and return
                jsonp,pjson -- parse the text with json.loads(text) and return
                    also, add parameter callback to the request
        header -- dictionary of headers to include in the request
    Example:
    >>> request_https('https://gitgub.com')
    >>> u = 'https://sampleserver3.arcgisonline.com/ArcGIS/rest/services'
    >>> request_https(u,{'f':'json'}, 'json')
    """
    url = str(url)
    callback = '' # may not be used

    # add the https protocol if not already specified
    if not url.lower().startswith("https://"):
        url = "https://" + url

    urlparsed = urlparse.urlparse(url)
    hostname = urlparsed.hostname
    path = url[8 + len(hostname):] # get path as url without https and host name

    # connect to the host and issue the request
    with closing(httplib.HTTPSConnection(hostname)) as cns:

        if data is None:

            if data_type == 'jsonp' or data_type == 'pjson':
                # TODO: Make sure callback parameter is included
                pass

            # use GET request, all parameters must be encoded as part of the url
            cns.request("GET", path, None, headers)

        else:

            if data_type == 'jsonp' or data_type == 'pjson':
                raise Exception("data_type 'jsonp' not allowed for POST method!")

            # use POST request, data must be a dictionary and not part of the url
            d = urllib.urlencode(data)
            cns.request("POST", path, d, headers)

        r = cns.getresponse()
        s = r.read()
        cns.close()

    # convert to required format
    result = None
    if data_type is None or data_type == 'text':
        result = s
    elif data_type == 'json':
        result = json.loads(s)
    elif data_type == 'jsonp' or data_type == 'pjson':
        result = json.loads(s.lstrip(callback + "(").rstrip(")"))
    elif data_type == 'xml':
        from xml.etree import ElementTree as ET
        rs = rs.strip()
        result = ET.fromstring(rs)

    return result


def request(url, data=None, data_type='text', headers={}):
    """Return result of an HTTP or HTTPS Request.

    Uses urllib2.Request to issue HTTP request and the urllib.HTTPSConnection
    to issue https requests.
    Only GET and POST methods are supported. To issue a GET request, parameters
    must be encoded as part of the url and data must be None. To issue a POST
    request, parameters must be supplied as a dictionary for parameter data and
    the url must not include parameters.

    ***Security warning***
    If url does not contain the protocol (http:// or https://) but just //,
    this function will first try https request, but if that fails, http request
    will be issued. For security reasons, protocol should be always specified,
    otherwise sensitive data inteded for encrypted connection (https) may be
    send to an unencripted connection. For example, consider you use the // to
    request a https url and you need to include secret token in the header.
    If the request fails, this function will attept to issue another request
    over http and will include the secret token with the http request, which can
    then be intercepted by malicious Internet users!

    Parameters:
        url -- URL to issue the request to
    Optional:
        data -- dictionary of data to send
        data_type -- text(default)|xml|json|jsonp|pjson
            data is always obtained as text, but the this function
            can convert the text depending on the data_type parameter:
                text -- return the raw text as it is
                xml -- parse the text with xml.etree.ElementTree and return
                json -- parse the text with json.loads(text) and return
                jsonp,pjson -- parse the text with json.loads(text) and return
                    also, add parameter callback to the request
        headers -- dictionary of headers to include in the request
    Example:
    >>> request('http://google.com')
    >>> u = 'http://sampleserver3.arcgisonline.com/ArcGIS/rest/services'
    >>> request(u,{'f':'json'}, 'json')
    >>> request('http://epsg.io/4326.xml', None, 'xml')
    >>> request('https://gitgub.com')
    """
    url = str(url)
    urll = url.lower()
    if urll.startswith("http://"):
        result = request_http(url, data, data_type, headers)
    elif urll.startswith("https://"):
        result = request_https(url, data, data_type, headers)
    elif url.startswith("//"):
        # try https first, then http
        try:
            result = request_https("https://" + url, data, data_type, headers)
        except:
            result = request_http("http://"+ url, data, data_type, headers)
    else:
        raise Exception("Protocol can only be http or https!")

    return result


def epsg(epsgcode, form='esriwkt'):
    """Get spatial reference system by EPSG code as string.
    Queries the http://epsg.io website.
    epsgcode -- European Petrol Survey Group code (http://www.epsg.org/)
    form -- Format to return:
        html : HTML
        wkt : Well Known Text
        esriwkt : Esri Well Known Text
        gml : GML
        xml : XML
        proj4 : Proj4
        js : proj4js
        geoserver : GeoServer
        map : MAPfile
        mapserverpython : MapServer - Python
        mapnik : Mapnik
        sql : PostGIS
    Example:
    >>> srs_str_by_epsg(27700, 'esriwkt')
    """
    srsstr = request('http://epsg.io/%s.%s' % (epsgcode, str(form).lower()))
    return srsstr


def arctype_to_ptype(tp):
    """Convert ArcGIS field type string to Python type.
      tp -- ArcGIS type as string like SHORT|LONG|TEXT|DOUBLE|FLOAT...

    Returns string for GUID, RASTER, BLOB, or other exotic types.

    Example:
    >>> arctype_to_ptype("SHORT") # returns int
    >>> arctype_to_ptype("long") # returns int
    >>> arctype_to_ptype("SmallInteger") # returns int
    >>> arctype_to_ptype("DATE") # returns datetime.datetime
    """
    tp = str(tp).upper().strip()
    o = str
    if tp == "TEXT" or tp == "STRING":
        o = str
    elif tp == "SHORT" or tp == "SMALLINTEGER":
        o = int
    elif tp == "LONG" or tp == "INTEGER":
        o = int
    elif tp == "DATE" or tp == "DATETIME":
        o = datetime.datetime
    elif tp == "FLOAT" or tp == "SINGLE":
        o = float
    elif tp == "DOUBLE":
        o = float
    else:
        o = str
    return o

def project_coordinates(xys, in_sr, out_sr, datum_transformation=None):
    """Project list of coordinate pairs (or triplets).
        xys -- list of coordinate pairs or triplets to project one by one
        in_sr -- input spatial reference, wkid, prj file, etc.
        out_sr -- output spatial reference, wkid, prj file, etc.
        datum_transformation=None -- datum transformation to use
            if in_sr and out_sr are defined on different datums,
            defining appropriate datum_transformation is necessary
            in order to obtain correct results!
            (hint: use arcpy.ListTransformations to list valid transformations)

    Example:
    >>> dtt = 'TM65_To_WGS_1984_2 + OSGB_1936_To_WGS_1984_NGA_7PAR'
    >>> coordinates = [(240600.0, 375800.0), (245900.0, 372200.0)]
    >>> project_coordinates(coordinates, 29902, 27700, dtt)
    """

    if not type(in_sr) is arcpy.SpatialReference:
        in_sr = arcpy.SpatialReference(in_sr)
    if not type(out_sr) is arcpy.SpatialReference:
        out_sr = arcpy.SpatialReference(out_sr)

    xyspr = []
    for xy in xys:
        pt = arcpy.Point(*xy)
        hasz = True if pt.Z is not None else False
        ptgeo = arcpy.PointGeometry(pt, in_sr)
        ptgeopr = ptgeo.projectAs(out_sr, datum_transformation)
        ptpr = ptgeopr.firstPoint
        if hasz:
            xypr = (ptpr.X, ptpr.Y, ptpr.Z)
        else:
            xypr = (ptpr.X, ptpr.Y)
        xyspr.append(xypr)

    return xyspr


class ArcapiError(Exception):
    """A type of exception raised from arcapi module"""
    pass


"""
Aliases
=======
Modified to allow computers without arcpy import arcapi.
That is why instead of just:
search = arcpy.da.SearchCursor
we need:
searcher = getattr(getattr(arcpy, "da", None), "SearchCursor", None)
"""
searcher = getattr(getattr(arcpy, "da", None), "SearchCursor", None)
updater = getattr(getattr(arcpy, "da", None), "UpdateCursor", None)
inserter = getattr(getattr(arcpy, "da", None), "InsertCursor", None)
add_col = getattr(getattr(arcpy, "management", None), "AddField", None)
descr = getattr(arcpy, "Describe", None)
flyr = getattr(getattr(arcpy, "management", None), "MakeFeatureLayer", None)
rlyr = getattr(getattr(arcpy, "management", None), "MakeRasterLayer", None)
tviw = getattr(getattr(arcpy, "management", None), "MakeTableView", None)
tos = to_scratch
wsps = swsp
osj = os.path.join
bname = os.path.basename
dname = os.path.dirname
srs = getattr(arcpy, "SpatialReference", None)


lut_field_types = {
    'Date':'DATE',
    'String':'TEXT',
    'Single':'FLOAT',
    'Double':'DOUBLE',
    'SmallInteger':'SHORT',
    'Integer':'LONG',
    'GUID':'GUID',
    'Raster':'RASTER'
}


def main():
    pass

if __name__ == '__main__':
    main()