181 alternative to q q plot to evaluate regression results

cobra/evaluation/pigs_tables.py

@@ -4,7 +4,7 @@
 import seaborn as sns
 import numpy as np
 from matplotlib.ticker import FuncFormatter
-
+from typing import Tuple
 import cobra.utils as utils
 
 def generate_pig_tables(basetable: pd.DataFrame,
@@ -107,7 +107,7 @@ def plot_incidence(pig_tables: pd.DataFrame,
                    variable: str,
                    model_type: str,
                    column_order: list=None,
-                   dim: tuple=(12, 8)):
+                   dim: tuple=(12, 8)) -> Tuple[plt.Figure, plt.Axes]:
     """Plots a Predictor Insights Graph (PIG), a graph in which the mean
     target value is plotted for a number of bins constructed from a predictor
     variable. When the target is a binary classification target,
@@ -130,6 +130,13 @@ def plot_incidence(pig_tables: pd.DataFrame,
         on the PIG.
     dim: tuple, default=(12, 8)
         Optional tuple to configure the width and length of the plot.
+
+    Retruns
+    -------
+    fig : plt.Figure
+        figure object contining the PIG
+    ax : plt.Axes 
+        axes object linked to the figure
     """
     if model_type not in ["classification", "regression"]:
         raise ValueError("An unexpected value was set for the model_type "
@@ -154,7 +161,7 @@ def plot_incidence(pig_tables: pd.DataFrame,
         df_plot.sort_values(by=['avg_target'], ascending=False, inplace=True)
         df_plot.reset_index(inplace=True)
 
-    with plt.style.context("seaborn-ticks"):
+    with sns.axes_style("ticks"):
         fig, ax = plt.subplots(figsize=dim)
 
         # --------------------------
@@ -257,5 +264,6 @@ def plot_incidence(pig_tables: pd.DataFrame,
         plt.tight_layout()
         plt.margins(0.01)
 
-        # Show
         plt.show()
+        plt.close()
+        return fig, ax

cobra/evaluation/plotting_utils.py

@@ -5,10 +5,11 @@
 
 import matplotlib.pyplot as plt
 import seaborn as sns
+from typing import Tuple
 
 def plot_univariate_predictor_quality(df_metric: pd.DataFrame,
                                       dim: tuple=(12, 8),
-                                      path: str=None):
+                                      path: str=None) -> Tuple[plt.Figure, plt.Axes]:
     """Plot univariate quality of the predictors.
 
     Parameters
@@ -21,6 +22,13 @@ def plot_univariate_predictor_quality(df_metric: pd.DataFrame,
         Width and length of the plot.
     path : str, optional
         Path to store the figure.
+
+    Retruns
+    -------
+    fig : plt.Figure
+        figure object containing the univariate predictor quality
+    ax : plt.Axes 
+        axes object linked to the figure
     """
 
     if "AUC selection" in df_metric.columns:
@@ -39,7 +47,7 @@ def plot_univariate_predictor_quality(df_metric: pd.DataFrame,
                  value_name=metric)
 
     # plot data
-    with plt.style.context("seaborn-ticks"):
+    with sns.axes_style("ticks"):
         fig, ax = plt.subplots(figsize=dim)
 
         ax = sns.barplot(x=metric, y="predictor", hue="split", data=df)
@@ -59,6 +67,8 @@ def plot_univariate_predictor_quality(df_metric: pd.DataFrame,
         plt.gca().legend().set_title("")
 
         plt.show()
+        plt.close()
+        return fig, ax
 
 def plot_correlation_matrix(df_corr: pd.DataFrame,
                             dim: tuple=(12, 8),
@@ -73,6 +83,13 @@ def plot_correlation_matrix(df_corr: pd.DataFrame,
         Width and length of the plot.
     path : str, optional
         Path to store the figure.
+
+    Retruns
+    -------
+    fig : plt.Figure
+        figure object containing the correlation matrix
+    ax : plt.Axes 
+        axes object linked to the figure
     """
     fig, ax = plt.subplots(figsize=dim)
     ax = sns.heatmap(df_corr, cmap="Blues")
@@ -81,15 +98,16 @@ def plot_correlation_matrix(df_corr: pd.DataFrame,
     if path is not None:
         plt.savefig(path, format="png", dpi=300, bbox_inches="tight")
 
-    plt.show()
+    plt.close()
+    return fig, ax
 
 def plot_performance_curves(model_performance: pd.DataFrame,
                             dim: tuple=(12, 8),
                             path: str=None,
                             colors: dict={"train": "#0099bf",
                                           "selection": "#ff9500",
                                           "validation": "#8064a2"},
-                            metric_name: str=None):
+                            metric_name: str=None) -> Tuple[plt.Figure, plt.Axes]:
     """Plot performance curves generated by the forward feature selection
     for the train-selection-validation sets.
 
@@ -108,6 +126,13 @@ def plot_performance_curves(model_performance: pd.DataFrame,
         Name to indicate the metric used in model_performance.
         Defaults to RMSE in case of regression and AUC in case of
         classification.
+
+    Retruns
+    -------
+    fig : plt.Figure
+        figure object that contains the performance curves
+    ax : plt.Axes 
+        axes object linked to the figure
     """
 
     model_type = model_performance["model_type"][0]
@@ -122,7 +147,7 @@ def plot_performance_curves(model_performance: pd.DataFrame,
                               max(model_performance['selection_performance']),
                               max(model_performance['validation_performance'])), 1)
 
-    with plt.style.context("seaborn-whitegrid"):
+    with sns.axes_style("whitegrid"):
 
         fig, ax = plt.subplots(figsize=dim)
 
@@ -160,11 +185,13 @@ def plot_performance_curves(model_performance: pd.DataFrame,
             plt.savefig(path, format="png", dpi=300, bbox_inches="tight")
 
         plt.show()
+        plt.close()
+        return fig, ax
 
 def plot_variable_importance(df_variable_importance: pd.DataFrame,
                              title: str=None,
                              dim: tuple=(12, 8),
-                             path: str=None):
+                             path: str=None) -> Tuple[plt.Figure, plt.Axes]:
     """Plot variable importance of a given model.
 
     Parameters
@@ -177,8 +204,15 @@ def plot_variable_importance(df_variable_importance: pd.DataFrame,
         Width and length of the plot.
     path : str, optional
         Path to store the figure.
+
+    Retruns
+    -------
+    fig : plt.Figure
+        figure object containing the variable importance
+    ax : plt.Axes 
+        axes object linked to the figure
     """
-    with plt.style.context("seaborn-ticks"):
+    with sns.axes_style("ticks"):
         fig, ax = plt.subplots(figsize=dim)
         ax = sns.barplot(x="importance", y="predictor",
                          data=df_variable_importance,
@@ -200,3 +234,5 @@ def plot_variable_importance(df_variable_importance: pd.DataFrame,
             plt.savefig(path, format="png", dpi=300, bbox_inches="tight")
 
         plt.show()
+        plt.close()
+        return fig, ax

tests/conftest.py

@@ -0,0 +1,14 @@
+import matplotlib.pyplot as plt
+
+def pytest_configure(config):
+    plt.ion()
+    pass # your code goes here
+
+def pytest_sessionfinish(session, exitstatus):
+    """
+    Called after whole test run finished, right before
+    returning the exit status to the system.
+    """
+    plt.close("all")
+
+# other config

tests/evaluation/test_evaluation.py

@@ -10,8 +10,8 @@ def mock_data():
     d = {'variable': ['education', 'education', 'education', 'education'],
          'label': ['1st-4th', '5th-6th', '7th-8th', '9th'],
          'pop_size': [0.002, 0.004, 0.009, 0.019],
-         'avg_incidence': [0.23, 0.23, 0.23, 0.23],
-         'incidence': [0.047, 0.0434, 0.054, 0.069]}
+         'global_avg_target': [0.23, 0.23, 0.23, 0.23],
+         'avg_target': [0.047, 0.0434, 0.054, 0.069]}
     return pd.DataFrame(d)
 
 def mock_preds(n, seed = 505):
@@ -22,6 +22,11 @@ def mock_preds(n, seed = 505):
 
     return y_true, y_pred
 
+
+
+
+
+
 class TestEvaluation:
 
     def test_plot_incidence_with_unsupported_model_type(self):
@@ -127,3 +132,54 @@ def test_fit_regression(self):
         for metric in ["R2", "MAE", "MSE", "RMSE"]:
             assert evaluator.scalar_metrics[metric] is not None
         assert evaluator.qq is not None
+
+class TestClassificationEvaluator:
+    y_true, y_pred = mock_preds(50)
+    y_true = (y_true > 0.5).astype(int)  # convert to 0-1 labels
+
+    evaluator = ClassificationEvaluator(n_bins=5)
+    evaluator.fit(y_true, y_pred)
+
+    def test_plot_roc_curve(self):
+        self.evaluator.plot_roc_curve()
+    def test_plot_confusion_matrix(self):
+        self.evaluator.plot_confusion_matrix()
+    def test_plot_cumulative_response_curve(self):
+        self.evaluator.plot_cumulative_response_curve()
+    def test_plot_lift_curve(self):
+        self.evaluator.plot_lift_curve()
+    def test_plot_cumulative_gains(self):
+        self.evaluator.plot_cumulative_gains()
+
+class TestRegressionEvaluator:
+    y_true, y_pred = mock_preds(50)
+    evaluator = RegressionEvaluator()
+    evaluator.fit(y_true, y_pred)
+    def test_plot_predictions(self):
+        self.evaluator.plot_predictions()
+    def test_plot_qq(self):
+        self.evaluator.plot_qq()
+
+
+
+    def test_compute_residuals_by_bin_uniform(self, example_evaluator):
+        # Test the function with "uniform" binning strategy
+        result_df = example_evaluator._compute_residuals_by_bin(nbins=10, binning_strat="uniform")
+        assert isinstance(result_df, pd.DataFrame)
+        assert len(result_df) == 10  # Check the number of bins
+
+    def test_compute_residuals_by_bin_quantile(self, example_evaluator):
+        # Test the function with "quantile" binning strategy
+        result_df = example_evaluator._compute_residuals_by_bin(nbins=10, binning_strat="quantile")
+        assert isinstance(result_df, pd.DataFrame)
+        assert len(result_df) == 10  # Check the number of bins
+
+
+
+@pytest.fixture
+def example_evaluator():
+    evaluator = RegressionEvaluator()
+    np.random.seed(42)
+    evaluator.y_true = np.random.rand(100)
+    evaluator.y_pred = np.random.rand(100)
+    return evaluator

tests/evaluation/test_pig_tables.py

@@ -0,0 +1,19 @@
+import pytest
+import pandas as pd
+
+from cobra.evaluation import plot_incidence
+
+def mock_data():
+    d = {'variable': ['education', 'education', 'education', 'education'],
+         'label': ['1st-4th', '5th-6th', '7th-8th', '9th'],
+         'pop_size': [0.002, 0.004, 0.009, 0.019],
+         'global_avg_target': [0.23, 0.23, 0.23, 0.23],
+         'avg_target': [0.047, 0.0434, 0.054, 0.069]}
+    return pd.DataFrame(d)
+
+
+def test_plot_incidence():
+    plot_incidence(pig_tables=mock_data(),
+                   variable="education",
+                   model_type="regression",)
+

tests/evaluation/test_plotting_utils.py

@@ -0,0 +1,57 @@
+from pandas import DataFrame
+from cobra.evaluation import (plot_univariate_predictor_quality, 
+                              plot_correlation_matrix,
+                              plot_performance_curves,
+                              plot_variable_importance)
+
+def mock_df_rmse() -> DataFrame:
+    return DataFrame(
+        {'predictor': {0: 'weight', 1: 'displacement', 2: 'horsepower',
+                       3: 'cylinders',  4: 'origin',  5: 'model_year',
+                       6: 'name',  7: 'acceleration'},
+        'RMSE train': {0: 4.225088318760745, 1: 4.403878881676005,
+                       2: 4.3343326307873875, 3: 4.901531871261906,
+                       4: 6.6435969708016955, 5: 6.318271823003904,
+                       6: 1.4537996193882199, 7: 6.631180878197439},
+        'RMSE selection': {0: 4.006855931973032, 1: 4.146696570151399,
+                           2: 4.321365764687869, 3: 4.466259266291863,
+                           4: 5.833138420191894, 5: 5.979795941821068,
+                           6: 6.99641113758452, 7: 7.449190759856361},
+        'preselection': {0: True, 1: True, 2: True, 3: True, 4: True,
+                         5: True, 6: True, 7: True}}
+        )
+
+
+def mock_df_corr() -> DataFrame:
+    return DataFrame({
+        'cylinders': {'cylinders': 1.0, 'weight': 0.8767772796304492, 'horsepower': 0.8124872187173973},
+        'weight': {'cylinders': 0.8767772796304492, 'weight': 1.0, 'horsepower': 0.8786843186591881},
+        'horsepower': {'cylinders': 0.8124872187173973, 'weight': 0.8786843186591881, 'horsepower': 1.0}})
+
+def mock_performances() -> DataFrame:
+    return DataFrame({
+        'predictors': {0: ['weight_enc'], 1: ['weight_enc', 'horsepower_enc'], 2: ['horsepower_enc', 'weight_enc', 'cylinders_enc']},
+        'last_added_predictor': {0: 'weight_enc', 1: 'horsepower_enc', 2: 'cylinders_enc'},
+        'train_performance': {0: 4.225088318760745, 1: 3.92118718828259, 2: 3.8929681840552495},
+        'selection_performance': {0: 4.006855931973032, 1: 3.630079770314085, 2: 3.531305702221386},
+        'validation_performance': {0: 4.348180862267973, 1: 4.089638309577036, 2: 3.9989641017455995},
+        'model_type': {0: 'regression', 1: 'regression', 2: 'regression'}
+        })
+
+def mock_variable_importance() -> DataFrame:
+    return DataFrame({
+        'predictor': {0: 'weight', 1: 'horsepower', 2: 'model_year', 3: 'origin'},
+        'importance': {0: 0.8921354566046729, 1: 0.864633073581914, 2: 0.694399044392948, 3: 0.6442243718390968}
+        })
+
+def test_plot_univariate_predictor_quality():
+    plot_univariate_predictor_quality(mock_df_rmse())
+
+def test_plot_correlation_matrix():
+    plot_correlation_matrix(mock_df_corr())
+
+def test_plot_performance_curves():
+    plot_performance_curves(mock_performances())
+
+def test_plot_variable_importance():
+    plot_variable_importance(mock_variable_importance())

tests/preprocessing/test_preprocessor.py

@@ -357,13 +357,13 @@ def test_mutable_train_data_fit_transform(self, mocker: MockerFixture):
                     },
                 ).astype(
                     {
-                        "a": np.float64(),
-                        "b": np.float64(),
-                        "d": np.float64(),
-                        "e": np.float64(),
-                        "category_1": pd.CategoricalDtype(),
-                        "category_2": pd.CategoricalDtype(),
-                        "category_3": pd.CategoricalDtype(),
+                        "a": "float64",
+                        "b": "float64",
+                        "d": "float64",
+                        "e": "float64",
+                        "category_1": "category",
+                        "category_2": "category",
+                        "category_3": "category",
                     }
                 ),
                 pd.DataFrame(
@@ -376,11 +376,11 @@ def test_mutable_train_data_fit_transform(self, mocker: MockerFixture):
                     }
                 ).astype(
                     {
-                        "a": np.float64(),
-                        "d": np.float64(),
-                        "e": np.float64(),
-                        "category_1": pd.CategoricalDtype(),
-                        "category_2": pd.CategoricalDtype(),
+                        "a": "float64",
+                        "d": "float64",
+                        "e": "float64",
+                        "category_1": "category",
+                        "category_2": "category",
                     }
                 ),
             ),