5 - Otros Modelos- Boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5Va2dZsnrWo1"
   },
   "source": [
    "### Modelado"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: statsmodels in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (0.14.0)\n",
      "Requirement already satisfied: numpy>=1.18 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from statsmodels) (1.24.3)\n",
      "Requirement already satisfied: scipy!=1.9.2,>=1.4 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from statsmodels) (1.10.1)\n",
      "Requirement already satisfied: pandas>=1.0 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from statsmodels) (1.5.3)\n",
      "Requirement already satisfied: patsy>=0.5.2 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from statsmodels) (0.5.3)\n",
      "Requirement already satisfied: packaging>=21.3 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from statsmodels) (23.0)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from pandas>=1.0->statsmodels) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from pandas>=1.0->statsmodels) (2022.7)\n",
      "Requirement already satisfied: six in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from patsy>=0.5.2->statsmodels) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install statsmodels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: lightgbm in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (4.3.0)\n",
      "Requirement already satisfied: numpy in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from lightgbm) (1.24.3)\n",
      "Requirement already satisfied: scipy in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from lightgbm) (1.10.1)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install lightgbm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: skfeature-chappers in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (1.1.0)\n",
      "Requirement already satisfied: scikit-learn in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from skfeature-chappers) (1.3.0)\n",
      "Requirement already satisfied: pandas in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from skfeature-chappers) (1.5.3)\n",
      "Requirement already satisfied: numpy in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from skfeature-chappers) (1.24.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.1 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from pandas->skfeature-chappers) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from pandas->skfeature-chappers) (2022.7)\n",
      "Requirement already satisfied: scipy>=1.5.0 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from scikit-learn->skfeature-chappers) (1.10.1)\n",
      "Requirement already satisfied: joblib>=1.1.1 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from scikit-learn->skfeature-chappers) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from scikit-learn->skfeature-chappers) (2.2.0)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\perelmuter-pc\\anaconda3\\lib\\site-packages (from python-dateutil>=2.8.1->pandas->skfeature-chappers) (1.16.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install skfeature-chappers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import plotly.express as px\n",
    "import plotly.graph_objects as go\n",
    "\n",
    "from sklearn.impute import KNNImputer\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.feature_selection import SelectKBest, f_regression \n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import BaggingRegressor\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n",
    "from sklearn.feature_selection import f_regression\n",
    "from sklearn.feature_selection import mutual_info_regression\n",
    "from sklearn.model_selection import KFold, cross_val_score, cross_val_predict\n",
    "import multiprocessing  \n",
    "\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.model_selection import cross_val_score, KFold\n",
    "from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, KFold\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "import numpy as np\n",
    "\n",
    "import statsmodels as sm\n",
    "import statsmodels.api as sm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "pd.set_option('display.max_rows', 120)\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = pd.read_csv('X_train.csv.gz')\n",
    "X_test = pd.read_csv('X_test.csv.gz')\n",
    "y_train = pd.read_csv('y_train.csv.gz')\n",
    "y_test = pd.read_csv('y_test.csv.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = y_train.drop(columns=['price'])\n",
    "y_test = y_test.drop(columns=['price'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "nTtMIGN8sBLv"
   },
   "outputs": [],
   "source": [
    "#selected_columns = ['accommodates','bedrooms', 'bathrooms_count' ,'number_of_reviews',\n",
    "                              #'availability_30', \n",
    "                              #'antiguedad', 'host_is_superhost_encoded','room_type_encoded', 'amenity_count', 'neighbourhood_cleansed_encoded', 'month']\n",
    "\n",
    "X_train_reg =  X_train.drop(['date', 'latitude', 'longitude', 'available', 'host_is_superhost', 'amenities', 'neighbourhood_cleansed', 'room_type'], axis=1)\n",
    "X_test_reg = X_test.drop(['date', 'latitude', 'longitude', 'available', 'host_is_superhost', 'amenities', 'neighbourhood_cleansed', 'room_type'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Gradient Boosting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "boosting_model = GradientBoostingRegressor(random_state=42)\n",
    "\n",
    "boosting_model.fit(X_train_rf, y_train.values.ravel())\n",
    "\n",
    "y_train_pred_boosting = boosting_model.predict(X_train_rf)\n",
    "\n",
    "y_test_pred_boosting = boosting_model.predict(X_test_rf)\n",
    "\n",
    "r2_train_boosting = r2_score(y_train, y_train_pred_boosting)\n",
    "mse_train_boosting = mean_squared_error(y_train, y_train_pred_boosting)\n",
    "\n",
    "r2_test_boosting = r2_score(y_test, y_test_pred_boosting)\n",
    "mse_test_boosting = mean_squared_error(y_test, y_test_pred_boosting)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gradient Boosting Metrics:\n",
      "Training Set - R-squared: 0.5206425399757273\n",
      "Training Set - Mean Squared Error: 0.4254067771219621\n",
      "Test Set - R-squared: 0.5212639044453273\n",
      "Test Set - Mean Squared Error: 0.4236505916240194\n"
     ]
    }
   ],
   "source": [
    "print('Gradient Boosting Metrics:')\n",
    "print('Training Set - R-squared:', r2_train_boosting)\n",
    "print('Training Set - Mean Squared Error:', mse_train_boosting)\n",
    "print('Test Set - R-squared:', r2_test_boosting)\n",
    "print('Test Set - Mean Squared Error:', mse_test_boosting)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost (Sin GridSearch) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_model = xgb.XGBRegressor(random_state=42)\n",
    "\n",
    "xgb_model.fit(X_train_rf, y_train.values.ravel())\n",
    "\n",
    "y_train_pred_xgb = xgb_model.predict(X_train_rf)\n",
    "\n",
    "y_test_pred_xgb = xgb_model.predict(X_test_rf)\n",
    "\n",
    "r2_train_xgb = r2_score(y_train, y_train_pred_xgb)\n",
    "mse_train_xgb = mean_squared_error(y_train, y_train_pred_xgb)\n",
    "\n",
    "r2_test_xgb = r2_score(y_test, y_test_pred_xgb)\n",
    "mse_test_xgb = mean_squared_error(y_test, y_test_pred_xgb)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "XGBoost Metrics:\n",
      "Training Set - R-squared: 0.7074762175432394\n",
      "Training Set - Mean Squared Error: 0.25960084050878285\n",
      "Test Set - R-squared: 0.7006104916257658\n",
      "Test Set - Mean Squared Error: 0.26494042025766495\n"
     ]
    }
   ],
   "source": [
    "print('XGBoost Metrics:')\n",
    "print('Training Set - R-squared:', r2_train_xgb)\n",
    "print('Training Set - Mean Squared Error:', mse_train_xgb)\n",
    "print('Test Set - R-squared:', r2_test_xgb)\n",
    "print('Test Set - Mean Squared Error:', mse_test_xgb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### XGBoost (Con GridSearch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best Hyperparameters: {'learning_rate': 0.1, 'max_depth': 7, 'min_child_weight': 1, 'n_estimators': 200}\n"
     ]
    }
   ],
   "source": [
    "xgb_model = xgb.XGBRegressor()\n",
    "\n",
    "param_grid = {\n",
    "    'max_depth': [3, 5, 7],  \n",
    "    'learning_rate': [0.1, 0.01],  \n",
    "    'n_estimators': [100, 200],  \n",
    "    'min_child_weight': [1, 3],  \n",
    "}\n",
    "\n",
    "grid_search = GridSearchCV(estimator=xgb_model, param_grid=param_grid, cv=5, scoring='neg_mean_squared_error', n_jobs=-1)\n",
    "grid_search.fit(X_train_rf, y_train)\n",
    "\n",
    "best_params = grid_search.best_params_\n",
    "print(\"Best Hyperparameters:\", best_params)\n",
    "\n",
    "best_estimator = grid_search.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set metrics:\n",
      "MAE: 0.36178843813937644\n",
      "MSE: 0.24324954261887066\n",
      "RMSE: 0.4932033481423972\n",
      "R-squared: 0.7259012099179176\n",
      "\n",
      "Test set metrics:\n",
      "MAE: 0.36436851581485924\n",
      "MSE: 0.24750896793365998\n",
      "RMSE: 0.497502731584119\n",
      "R-squared: 0.7203084823531047\n"
     ]
    }
   ],
   "source": [
    "y_train_pred = best_estimator.predict(X_train_rf)\n",
    "\n",
    "y_test_pred = best_estimator.predict(X_test_rf)\n",
    "\n",
    "train_mae = mean_absolute_error(y_train, y_train_pred)\n",
    "train_mse = mean_squared_error(y_train, y_train_pred)\n",
    "train_rmse = mean_squared_error(y_train, y_train_pred, squared=False)\n",
    "train_r2 = r2_score(y_train, y_train_pred)\n",
    "\n",
    "test_mae = mean_absolute_error(y_test, y_test_pred)\n",
    "test_mse = mean_squared_error(y_test, y_test_pred)\n",
    "test_rmse = mean_squared_error(y_test, y_test_pred, squared=False)\n",
    "test_r2 = r2_score(y_test, y_test_pred)\n",
    "\n",
    "print(\"Training set metrics:\")\n",
    "print(\"MAE:\", train_mae)\n",
    "print(\"MSE:\", train_mse)\n",
    "print(\"RMSE:\", train_rmse)\n",
    "print(\"R-squared:\", train_r2)\n",
    "print()\n",
    "print(\"Test set metrics:\")\n",
    "print(\"MAE:\", test_mae)\n",
    "print(\"MSE:\", test_mse)\n",
    "print(\"RMSE:\", test_rmse)\n",
    "print(\"R-squared:\", test_r2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Adaboost "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "adaboost_model = AdaBoostRegressor(random_state=42)\n",
    "\n",
    "adaboost_model.fit(X_train_rf, y_train.values.ravel())\n",
    "\n",
    "y_train_pred_adaboost = adaboost_model.predict(X_train_rf)\n",
    "\n",
    "y_test_pred_adaboost = adaboost_model.predict(X_test_rf)\n",
    "\n",
    "r2_train_adaboost = r2_score(y_train, y_train_pred_adaboost)\n",
    "mse_train_adaboost = mean_squared_error(y_train, y_train_pred_adaboost)\n",
    "\n",
    "r2_test_adaboost = r2_score(y_test, y_test_pred_adaboost)\n",
    "mse_test_adaboost = mean_squared_error(y_test, y_test_pred_adaboost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "AdaBoost Metrics:\n",
      "Training Set - R-squared: 0.3188146336095182\n",
      "Training Set - Mean Squared Error: 0.6045193733381024\n",
      "Test Set - R-squared: 0.3186852322952404\n",
      "Test Set - Mean Squared Error: 0.6029196609582566\n"
     ]
    }
   ],
   "source": [
    "print('AdaBoost Metrics:')\n",
    "print('Training Set - R-squared:', r2_train_adaboost)\n",
    "print('Training Set - Mean Squared Error:', mse_train_adaboost)\n",
    "print('Test Set - R-squared:', r2_test_adaboost)\n",
    "print('Test Set - Mean Squared Error:', mse_test_adaboost)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}