Merge pull request #165 from bashtage/rel-4.9

RLS: Release 4.9 commit

.lgtm.yml

@@ -0,0 +1,6 @@
+extraction:
+  python:
+    index:
+      exclude:
+        - versioneer.py
+        - linearmodels/_version.py

.travis.yml

@@ -24,7 +24,7 @@ env:
 matrix:
   fast_finish: true
   include:
-  - python: 3.6
+  - python: 3.5
     env:
     - PYTHON=3.5
     - NUMPY=1.12
@@ -42,15 +42,15 @@ matrix:
   - python: 3.6
     env:
     - PYTHON=3.6
-    - NUMPY=1.15
-    - SCIPY=1.1
+    - NUMPY=1.14
+    - SCIPY=1
     - PANDAS=0.22
     - XARRAY=0.10
     - DOCBUILD=true
     - STATAMODELS=0.9
   - python: 3.6
     env:
-    - PYTHON=3.6
+    - PYTHON=3.7
     - NUMPY=1.14
     - SCIPY=1.1
     - PANDAS=0.23

README.md

@@ -2,6 +2,8 @@
 
 [![Build Status](https://travis-ci.org/bashtage/linearmodels.svg?branch=master)](https://travis-ci.org/bashtage/linearmodels)
 [![codecov](https://codecov.io/gh/bashtage/linearmodels/branch/master/graph/badge.svg)](https://codecov.io/gh/bashtage/linearmodels)
+[![Codacy Badge](https://api.codacy.com/project/badge/Grade/c771bce50a164b6fa71c344b374f140d)](https://www.codacy.com/app/bashtage/linearmodels?utm_source=github.com&utm_medium=referral&utm_content=bashtage/linearmodels&utm_campaign=Badge_Grade)
+[![codebeat badge](https://codebeat.co/badges/aaae2fb4-72b5-4a66-97cd-77b93488f243)](https://codebeat.co/projects/github-com-bashtage-linearmodels-master)
 
 Linear (regression) models for Python.  Extends
 [statsmodels](http://www.statsmodels.org) with Panel regression,

README.rst

@@ -1,8 +1,7 @@
 Linear Models
 =============
 
-`Build Status <https://travis-ci.org/bashtage/linearmodels>`__
-`codecov <https://codecov.io/gh/bashtage/linearmodels>`__
+|Build Status| |codecov| |Codacy Badge| |codebeat badge|
 
 Linear (regression) models for Python. Extends
 `statsmodels <http://www.statsmodels.org>`__ with Panel regression,
@@ -161,3 +160,12 @@ Documentation
 -  nbformat
 -  ipython
 -  jupyter
+
+.. |Build Status| image:: https://travis-ci.org/bashtage/linearmodels.svg?branch=master
+   :target: https://travis-ci.org/bashtage/linearmodels
+.. |codecov| image:: https://codecov.io/gh/bashtage/linearmodels/branch/master/graph/badge.svg
+   :target: https://codecov.io/gh/bashtage/linearmodels
+.. |Codacy Badge| image:: https://api.codacy.com/project/badge/Grade/c771bce50a164b6fa71c344b374f140d
+   :target: https://www.codacy.com/app/bashtage/linearmodels?utm_source=github.com&utm_medium=referral&utm_content=bashtage/linearmodels&utm_campaign=Badge_Grade
+.. |codebeat badge| image:: https://codebeat.co/badges/aaae2fb4-72b5-4a66-97cd-77b93488f243
+   :target: https://codebeat.co/projects/github-com-bashtage-linearmodels-master

doc/source/changes.rst

@@ -1,8 +1,12 @@
 Change Log
 ----------
 
-Since 4.8
-=========
+Version 4.9
+===========
+* Changed the return type of Wooldridge's over identification test when
+  invalid to `InvalidTestStatistic`
+* Add typing information to IV models
+* Allow optimization parameters to be passed to `IVGMMCUE`
 * Removed internal use of pandas Panel
 * Improved performance in panel models when using `from_formula`
 * Switched to retaining index column names when original input index is named

doc/source/plan.rst

@@ -47,6 +47,10 @@ System Estimation
 =================
 * Seemingly Unrelated Regression (SUR) Estimator - :class:`linearmodels.system.model.SUR`
 
+  - Multivariate OLS is supported as a special case of SUR.  This method does
+    not perform well on large datasets and should be improved by a special
+    purpose implementation.
+
 Instrumental Variable Estimators
 ********************************
 * Three-stage Least Squares (3SLS) Estimator - :class:`linearmodels.system.model.IV3SLS`

linearmodels/compat/pandas.py

@@ -1,3 +1,27 @@
+from distutils.version import LooseVersion
+
+import pandas as pd
+
+PD_LT_023 = LooseVersion(pd.__version__) < LooseVersion('0.23')
+
+
+def concat(*args, **kwargs):
+    """
+    Shim around pandas concat that passes sort if allowed
+
+    See pandas.compat
+    """
+    if PD_LT_023 and 'sort' in kwargs:
+        kwargs = kwargs.copy()
+        del kwargs['sort']
+    elif not PD_LT_023:
+        if 'sort' not in kwargs:
+            kwargs = kwargs.copy()
+            kwargs['sort'] = False
+
+    return pd.concat(*args, **kwargs)
+
+
 try:
     from pandas.api.types import (is_numeric_dtype, is_categorical,
                                   is_string_dtype, is_categorical_dtype,
@@ -24,11 +48,6 @@ def is_string_like(obj):
                                     is_categorical, is_categorical_dtype,
                                     is_datetime64_any_dtype, is_string_like)
 
-try:
-    from pandas.testing import assert_frame_equal, assert_series_equal
-except ImportError:
-    from pandas.util.testing import assert_frame_equal, assert_series_equal
-
 __all__ = ['is_string_dtype', 'is_numeric_dtype', 'is_categorical',
            'is_string_like', 'is_categorical_dtype', 'is_datetime64_any_dtype',
-           'assert_frame_equal', 'assert_series_equal']
+           'concat']

linearmodels/iv/data.py

@@ -8,7 +8,7 @@
 
 from linearmodels.compat.pandas import (is_categorical, is_categorical_dtype,
                                         is_numeric_dtype, is_string_dtype,
-                                        is_string_like)
+                                        is_string_like, concat)
 
 dim_err = '{0} has too many dims.  Maximum is 2, actual is {1}'
 type_err = 'Only ndarrays, DataArrays and Series and DataFrames are supported'
@@ -25,7 +25,7 @@ def convert_columns(s, drop_first):
 def expand_categoricals(x, drop_first):
     if x.shape[1] == 0:
         return x
-    return pd.concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
+    return concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
 
 
 class IVData(object):

linearmodels/panel/data.py

@@ -2,14 +2,13 @@
 
 import numpy as np
 import pandas as pd
-from numpy import ndarray
 from pandas import DataFrame, Panel, Series
 
 from linearmodels.compat.numpy import lstsq
 from linearmodels.compat.pandas import (is_categorical,
                                         is_datetime64_any_dtype,
                                         is_numeric_dtype, is_string_dtype,
-                                        is_string_like)
+                                        is_string_like, concat)
 from linearmodels.utility import ensure_unique_column, panel_to_frame
 
 __all__ = ['PanelData']
@@ -90,7 +89,7 @@ def convert_columns(s, drop_first):
 
 
 def expand_categoricals(x, drop_first):
-    return pd.concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
+    return concat([convert_columns(x[c], drop_first) for c in x.columns], axis=1)
 
 
 class PanelData(object):
@@ -153,7 +152,7 @@ def __init__(self, x, var_name='x', convert_dummies=True, drop_first=True, copy=
             x = x.dataframe
         self._original = x
 
-        if not isinstance(x, (Series, DataFrame, Panel, ndarray)):
+        if not isinstance(x, (Series, DataFrame, Panel, np.ndarray)):
             try:
                 from xarray import DataArray
                 if isinstance(x, DataArray):
@@ -191,7 +190,7 @@ def __init__(self, x, var_name='x', convert_dummies=True, drop_first=True, copy=
                     self._frame = DataFrame({var_name: x.T.stack(dropna=False)})
             else:
                 self._frame = x.swapaxes(1, 2).to_frame(filter_observations=False)
-        elif isinstance(x, ndarray):
+        elif isinstance(x, np.ndarray):
             if x.ndim not in (2, 3):
                 raise ValueError('2 or 3-d array required for numpy input')
             if x.ndim == 2:

linearmodels/panel/results.py

@@ -1,7 +1,6 @@
 import datetime as dt
 
 import numpy as np
-from numpy import diag, sqrt
 from pandas import DataFrame, Series, concat
 from scipy import stats
 from statsmodels.iolib.summary import SimpleTable, fmt_2cols, fmt_params
@@ -67,7 +66,7 @@ def cov(self):
     @property
     def std_errors(self):
         """Estimated parameter standard errors"""
-        return Series(sqrt(diag(self.cov)), self._var_names, name='std_error')
+        return Series(np.sqrt(np.diag(self.cov)), self._var_names, name='std_error')
 
     @property
     def tstats(self):

linearmodels/system/results.py

@@ -1,7 +1,6 @@
 import datetime as dt
 
 import numpy as np
-from numpy import diag, sqrt
 from pandas import DataFrame, Series, concat
 from scipy import stats
 from statsmodels.iolib.summary import SimpleTable, fmt_2cols
@@ -76,7 +75,7 @@ def params(self):
     @property
     def std_errors(self):
         """Estimated parameter standard errors"""
-        std_errors = sqrt(diag(self.cov))
+        std_errors = np.sqrt(np.diag(self.cov))
         return Series(std_errors, index=self._param_names, name='stderr')
 
     @property

linearmodels/tests/asset_pricing/test_formulas.py

@@ -1,11 +1,11 @@
 import numpy as np
-import pandas as pd
 import pytest
+from pandas.testing import assert_frame_equal
 
 from linearmodels.asset_pricing.model import (LinearFactorModel,
                                               LinearFactorModelGMM,
                                               TradedFactorModel)
-from linearmodels.compat.pandas import assert_frame_equal
+from linearmodels.compat.pandas import concat
 from linearmodels.tests.asset_pricing._utility import generate_data
 
 FORMULA_FACTORS = 'factor_1 + factor_2 + factor_3'
@@ -29,7 +29,7 @@ def non_traded_model(request):
 def data(request):
     premia = np.array([.1, .1, .1])
     out = generate_data(nportfolio=10, output='pandas', alpha=True, premia=premia)
-    out['joined'] = pd.concat([out.factors, out.portfolios], 1)
+    out['joined'] = concat([out.factors, out.portfolios], 1)
     return out
 
 

linearmodels/tests/iv/test_data.py

@@ -12,7 +12,7 @@
 except ImportError:
     MISSING_XARRAY = True
 
-from linearmodels.compat.pandas import assert_frame_equal, assert_series_equal
+from pandas.testing import assert_frame_equal, assert_series_equal
 from linearmodels.iv.data import IVData
 
 

linearmodels/tests/iv/test_formulas.py

@@ -1,9 +1,10 @@
 import numpy as np
-import pandas as pd
 import pytest
 from numpy.testing import assert_allclose, assert_equal
+from pandas import DataFrame, Categorical
+from pandas.testing import assert_frame_equal
 
-from linearmodels.compat.pandas import assert_frame_equal
+from linearmodels.compat.pandas import concat
 from linearmodels.formula import iv_2sls, iv_gmm, iv_gmm_cue, iv_liml
 from linearmodels.iv import IV2SLS, IVGMM, IVGMMCUE, IVLIML
 
@@ -49,7 +50,7 @@ def data():
     y = x @ params + e
     cols = ['y'] + ['x' + str(i) for i in range(1, 6)]
     cols += ['z' + str(i) for i in range(1, 4)]
-    data = pd.DataFrame(np.c_[y, x, z], columns=cols)
+    data = DataFrame(np.c_[y, x, z], columns=cols)
     data['Intercept'] = 1.0
     data['weights'] = np.random.chisquare(10, size=data.shape[0]) / 10
     return data
@@ -164,8 +165,8 @@ def test_categorical(model_and_func):
     y = np.random.randn(1000)
     x1 = np.random.randn(1000)
     d = np.random.randint(0, 4, 1000)
-    d = pd.Categorical(d)
-    data = pd.DataFrame({'y': y, 'x1': x1, 'd': d})
+    d = Categorical(d)
+    data = DataFrame({'y': y, 'x1': x1, 'd': d})
     data['Intercept'] = 1.0
     model, func = model_and_func
     mod = model.from_formula(formula, data)
@@ -199,7 +200,7 @@ def test_formula_function(data, model_and_func):
     dep = data.y
     exog = [data[['Intercept']], sigmoid(data[['x3']]), data[['x4']],
             np.exp(data[['x5']])]
-    exog = pd.concat(exog, 1)
+    exog = concat(exog, 1)
     endog = data[['x1', 'x2']]
     instr = data[['z1', 'z2', 'z3']]
     mod = model(dep, exog, endog, instr)
@@ -220,7 +221,7 @@ def test_predict_formula_function(data, model_and_func):
 
     exog = [data[['Intercept']], sigmoid(data[['x3']]), data[['x4']],
             np.exp(data[['x5']])]
-    exog = pd.concat(exog, 1)
+    exog = concat(exog, 1)
     endog = data[['x1', 'x2']]
     pred = res.predict(exog, endog)
     pred2 = res.predict(data=data)

linearmodels/tests/iv/test_results.py

@@ -1,8 +1,8 @@
-import pandas as pd
 import pytest
 from numpy.testing import assert_allclose
+from pandas import DataFrame
+from pandas.testing import assert_series_equal
 
-from linearmodels.compat.pandas import assert_series_equal
 from linearmodels.iv.data import IVData
 from linearmodels.iv.model import IV2SLS, IVGMM, IVGMMCUE, IVLIML
 from linearmodels.tests.iv._utility import generate_data
@@ -65,12 +65,12 @@ def test_fitted_predict(data, model):
     assert_series_equal(res.idiosyncratic, res.resids)
     y = mod.dependent.pandas
     expected = y.values - res.resids.values[:, None]
-    expected = pd.DataFrame(expected, y.index, ['fitted_values'])
+    expected = DataFrame(expected, y.index, ['fitted_values'])
     assert_frame_similar(expected, res.fitted_values)
     assert_allclose(expected, res.fitted_values)
     pred = res.predict()
     nobs = res.resids.shape[0]
-    assert isinstance(pred, pd.DataFrame)
+    assert isinstance(pred, DataFrame)
     assert pred.shape == (nobs, 1)
     pred = res.predict(idiosyncratic=True, missing=True)
     nobs = IVData(data.dep).pandas.shape[0]