Skip to content

Commit

Permalink
Merge pull request #40 from CDAT/fix_percentile_bug
Browse files Browse the repository at this point in the history 
Fix percentile bug
  • Loading branch information
@muryanto1
muryanto1 committed Jul 11, 2020
2 parents 8efedd2 + 9af4592 commit 696cf33
Show file tree
Hide file tree
Showing 6 changed files with 278 additions and 174 deletions.
49 changes: 7 additions & 42 deletions .circleci/config.yml
View file
Original file line number Diff line number Diff line change
@@ -1,25 +1,5 @@
version: 2.1

parameters:
pkg_name:
type: string
default: "genutil"
repo_name:
type: string
default: "genutil"
last_stable:
type: string
default: "8.2"
user:
type: string
default: "cdat"
label:
type: string
default: "nightly"
env_name:
type: string
default: "test_genutil"

aliases:
- &setup_env
name: setup_env
Expand All @@ -31,28 +11,24 @@ aliases:
export PROJECT_DIR=/home/circleci/project/workdir/linux
fi
echo "export WORKDIR=$PROJECT_DIR/$PY_VER" >> $BASH_ENV
cat $BASH_ENV
source $BASH_ENV
mkdir -p $WORKDIR
- &setup_miniconda
name: setup_miniconda
command: |
source $BASH_ENV
mkdir -p $WORKDIR
git clone https://github.com/CDAT/cdat.git $WORKDIR/cdat
# install_miniconda.py installs miniconda3 under $WORKDIR/miniconda
python $WORKDIR/cdat/scripts/install_miniconda.py -w $WORKDIR -p 'py3'
- &conda_rerender
name: conda_rerender
command: |
source $BASH_ENV
source $WORKDIR/miniconda/etc/profile.d/conda.sh
conda activate base
echo "make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH"
make conda-rerender conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR last_stable=$LAST_STABLE branch=$CIRCLE_BRANCH
make conda-rerender workdir=$WORKDIR branch=$CIRCLE_BRANCH
- &conda_build
name: conda_build
Expand All @@ -62,7 +38,7 @@ aliases:
conda activate base
os=`uname`
artifacts_dir="artifacts/artifacts.${os}.py_${PY_VER}"
make conda-build conda=$WORKDIR/miniconda/bin/conda workdir=$WORKDIR artifact_dir=$PWD/$artifacts_dir build_version=$PY_VER
make conda-build workdir=$WORKDIR artifact_dir=$artifacts_dir build_version=$PY_VER
- &setup_run_tests
name: setup_run_tests
Expand All @@ -72,19 +48,15 @@ aliases:
conda activate base
export CONDA_PY_VER="python=$PY_VER"
export LIBNETCDF_VER="libnetcdf=*=${LIBNETCDF}_*"
echo "make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs=\"$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS\""
make setup-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
make conda-dump-env conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
make setup-tests workdir=$WORKDIR extra_pkgs="$CONDA_PY_VER $LIBNETCDF_VER $COVERAGE_PKGS"
make conda-dump-env workdir=$WORKDIR artifact_dir=$PWD/spec_artifacts conda_env_filename=$CIRCLE_JOB
- &run_tests
name: run_tests
command: |
source $BASH_ENV
source $WORKDIR/miniconda/etc/profile.d/conda.sh
conda activate $ENV_NAME
make run-tests conda=$WORKDIR/miniconda/bin/conda conda_env=$ENV_NAME workdir=$WORKDIR
make run-tests workdir=$WORKDIR
no_output_timeout: 10m

- &conda_upload
Expand All @@ -93,8 +65,8 @@ aliases:
source $BASH_ENV
source $WORKDIR/miniconda/etc/profile.d/conda.sh
conda activate base
UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN user=$USER label=$LABEL"
make conda-upload $UPLOAD_OPTIONS conda=$WORKDIR/miniconda/bin/conda artifact_dir="$PWD/artifacts/*/"
UPLOAD_OPTIONS="conda_upload_token=$CONDA_UPLOAD_TOKEN"
make conda-upload workdir=$WORKDIR $UPLOAD_OPTIONS artifact_dir="artifacts/*/"
executors:
linux:
Expand All @@ -113,9 +85,6 @@ jobs:
type: string
executor: << parameters.os >>
environment:
PKG_NAME: << pipeline.parameters.pkg_name >>
REPO_NAME: << pipeline.parameters.repo_name >>
LAST_STABLE: << pipeline.parameters.last_stable >>
PY_VER: << parameters.py_ver >>
steps:
- checkout
Expand All @@ -141,8 +110,6 @@ jobs:
type: string
executor: << parameters.os >>
environment:
PKG_NAME: << pipeline.parameters.pkg_name >>
ENV_NAME: << pipeline.parameters.env_name >>
PY_VER: << parameters.py_ver >>
LIBNETCDF: << parameters.libnetcdf >>
steps:
Expand All @@ -164,8 +131,6 @@ jobs:
machine:
image: circleci/classic:latest
environment:
USER: << pipeline.parameters.user >>
LABEL: << pipeline.parameters.label >>
PY_VER: "3.7"
steps:
- checkout
Expand Down
8 changes: 5 additions & 3 deletions Lib/statistics.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1668,6 +1668,10 @@ def geometricmean(x, axis=0, max_pct_missing=100.):


def _percentiles(out, percent):
# change 1d to 2d array
if out.ndim == 1:
out = out.reshape((1,)+out.shape)

if cdms2.isVariable(out):
out = MV2.sort(out, axis=0).asma()
ns = MV2.count(out, axis=0).asma()
Expand Down Expand Up @@ -1696,10 +1700,8 @@ def _percentiles(out, percent):
pass
Aii = numpy.where(numpy.equal(ns, 1), 100., tmp)
ii = numpy.where(numpy.equal(ii, ns), ns - 1, ii)
if numpy.rank(ii) > 0:
if numpy.ndim(ii) > 0:
ii = ii.astype(numpy.int)
# tmp = (p-Ai)/(Aii-Ai)*array_indexing.extract(out,ii) + \
# (Aii-p)/(Aii-Ai)*array_indexing.extract(out,i)

tmp = (p - Ai) / (Aii - Ai) * arrayindexing.get(out, ii) + \
(Aii - p) / (Aii - Ai) * arrayindexing.get(out, i)
Expand Down
78 changes: 44 additions & 34 deletions Lib/stats_checker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,15 @@
from .averager import __check_weightoptions


class StatisticsError (Exception):
class StatisticsError(Exception):
def __init__(self, args=None):
"""Create an exception"""
self.args = args

def __str__(self):
"""Calculate the string representation"""
return str(self.args)

__repr__ = __str__


Expand All @@ -32,10 +33,10 @@ def __makeweights(x, w, axes):
if not numpy.ma.isarray(w):
# Ok Krishna returned a list of 1D arrays.... Let's put it together
axs = x.getAxisList()
axes = cdms2.order2index(axs, axes)[:len(cdms2.orderparse(axes))]
axes = cdms2.order2index(axs, axes)[: len(cdms2.orderparse(axes))]
endax = []
for i in range(len(axes)):
if w[i] == 'unweighted':
if w[i] == "unweighted":
w[i] = numpy.ma.ones(len(axs[axes[i]]), dtype=x.dtype.char)
if i == 0:
wo = w[i]
Expand All @@ -45,8 +46,8 @@ def __makeweights(x, w, axes):
endax.append(axs[axes[i]])
w = cdms2.MV2.array(wo)
w.setAxisList(endax)
# else:
# w.setAxisList(x.getAxisList())
# else:
# w.setAxisList(x.getAxisList())
return w


Expand All @@ -64,29 +65,31 @@ def __checker(x, y, w, axes, smally=0):
x = numpy.ma.array(x, copy=0)
if not numpy.ma.isarray(y) and y is not None:
y = numpy.ma.array(y, copy=0)
if not numpy.ma.isarray(
w) and w is not None and not isinstance(w, type('')):
if not isinstance(w[0], type('')):
if not numpy.ma.isarray(w) and w is not None and not isinstance(w, type("")):
if not isinstance(w[0], type("")):
w = numpy.ma.array(w, copy=0)
else:
if not xismv:
raise StatisticsError(
'Error if weights are a list then x must be an MV2 !!!')
"Error if weights are a list then x must be an MV2 !!!"
)
w = __makeweights(x, w, axes)
wismv = 1
elif w is not None:
if not xismv:
raise StatisticsError(
'Error if weights are a list then x must be an MV2 !!!')
"Error if weights are a list then x must be an MV2 !!!"
)
w = __makeweights(x, w, axes)
wismv = 1

if xismv * yismv * wismv != 1:
# We didn't pass all MV2s shapes have to match (unless None)
if smally == 0:
if x.shape != numpy.ma.shape(y) and y is not None:
raise StatisticsError('Error x and y shape do not match !' +
str(x.shape) + ',' + str(numpy.ma.shape(y)))
raise StatisticsError(
"Error x and y shape do not match !" + str(x.shape) + "," + str(numpy.ma.shape(y))
)
else:
shy = list(y.shape)
shy2 = y.shape
Expand All @@ -96,7 +99,9 @@ def __checker(x, y, w, axes, smally=0):
for i in axes:
myaxes.append(eval(i))
elif isinstance(axes, int):
myaxes = [axes, ]
myaxes = [
axes,
]
else:
myaxes = list(axes)
for anaxis in myaxes[::-1]:
Expand All @@ -109,25 +114,27 @@ def __checker(x, y, w, axes, smally=0):
sh[i] = myaxes[i]
y = numpy.ma.transpose(y, sh)
if x.shape != numpy.ma.shape(y) and y is not None:
raise StatisticsError('Error x and y shape do not match (y shouldbe 1D less than x) !' +
str(x.shape) + ',' + str(shy2) + ' Remember y must be 1D less than x')
err_msg = "Error x and y shape do not match (y shouldbe 1D less than x) !"
raise StatisticsError(
err_msg + str(x.shape) + "," + str(shy2) + " Remember y must be 1D less than x"
)
if x.shape != numpy.ma.shape(w) and w is not None:
raise StatisticsError('Error x and weights shape do not match !' +
str(x.shape) + ',' + str(numpy.ma.shape(w)) +
' ATTENTION if you are trynig to pass a list of 1D arrays' +
'for each dim, then x must be an MV2 !!!')
msg1 = "Error x and weights shape do not match !"
msg2 = " ATTENTION if you are trying to pass a list of 1D arrays for each dim, then x must be an MV2 !!!"
raise StatisticsError(msg1 + str(x.shape) + "," + str(numpy.ma.shape(w)) + msg2)
if not isinstance(axes, type([])):
axes = cdms2.orderparse(str(axes))
for i in axes:
if len(x.shape) < i:
raise StatisticsError('Error you have ' + str(len(x.shape)) +
' dimensions and try to work on dim:' + str(i))
err_msg = "Error you have " + str(len(x.shape)) + " dimensions and try to work on dim:" + str(i)
raise StatisticsError(err_msg)
else:
if y is not None:
x, y = grower(x, y)
if x.shape != y.shape:
raise StatisticsError(
'Error x and y have different shapes' + str(x.shape) + ', ' + str(y.shape))
"Error x and y have different shapes" + str(x.shape) + ", " + str(y.shape)
)
ax = x.getAxisList()
xorder = x.getOrder(ids=1)
# Now grows w
Expand All @@ -136,11 +143,13 @@ def __checker(x, y, w, axes, smally=0):
for o in worder:
if o not in xorder:
raise StatisticsError(
'Error weights have a dimension that is neither in x or y:' + o)
"Error weights have a dimension that is neither in x or y:" + o
)
x, w = grower(x, w)
if x.shape != w.shape:
raise StatisticsError(
'Error x and weights have different shapes' + str(x.shape) + ', ' + str(w.shape))
"Error x and weights have different shapes" + str(x.shape) + ", " + str(w.shape)
)
# Last thing convert the axes input to numbers
if isinstance(axes, type(1)):
axes = str(axes)
Expand All @@ -149,24 +158,21 @@ def __checker(x, y, w, axes, smally=0):
naxes = len(axesparse)
for i in range(naxes):
o = axesparse[i]
if isinstance(o, type('')):
if isinstance(o, type("")):
for j in range(len(xorder)):
if xorder[j] == o:
axesparse[i] = j
# Well it must be a name for x y t....
if isinstance(axesparse[i], type('')):
if isinstance(axesparse[i], type("")):
for j in range(len(x.shape)):
if o[1:-1] == x.getAxis(j).id:
axesparse[i] = j
# Everything failed the axis id must be not existing in the
# slab...
if isinstance(axesparse[i], type('')):
if isinstance(axesparse[i], type("")):
raise StatisticsError(
'Error axis id :' +
o +
' not found in first slab: ' +
x.getOrder(
ids=1))
"Error axis id :" + o + " not found in first slab: " + x.getOrder(ids=1)
)
axes = axesparse
# Now we have array those shape match, and a nice list of axes let's keep
# going
Expand All @@ -176,8 +182,12 @@ def __checker(x, y, w, axes, smally=0):
xorder = list(range(len(x.shape)))
forder = []
for i in range(naxes):
forder.append(axes[i])
n0 = n0 * xsh[axes[i]]
a = axes[i]
forder.append(a)
try:
n0 = n0 * xsh[a]
except IndexError:
raise Exception("Axis {} is out of bounds for dimension {}".format(a, len(xsh)))
fsh = [n0]
ax2 = []
for i in range(len(x.shape)):
Expand Down
Loading

0 comments on commit 696cf33

Please sign in to comment.