Feature test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import string\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import scipy\n",
    "import lightgbm as lgb\n",
    "pd.options.display.max_rows=200"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "SEED=411\n",
    "SPLIT_TEST = False\n",
    "target = 'deal_probability'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Basic Agg\\Naive Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## gp features\n",
    "\n",
    "train = pd.read_csv('data/train.csv', parse_dates=['activation_date'])\n",
    "train = train.sort_values(['activation_date']).reset_index(drop=True)\n",
    "test = pd.read_csv('data/test.csv', parse_dates=['activation_date'])\n",
    "y_train = train.deal_probability.values\n",
    "\n",
    "\n",
    "\n",
    "'''\n",
    "gp = pd.read_csv('aggregated_features.csv')\n",
    "train = train.merge(gp, on='user_id', how='left')\n",
    "test = test.merge(gp, on='user_id', how='left')\n",
    "   \n",
    "agg_cols = list(gp.columns)[1:]\n",
    "del gp; gc.collect()\n",
    "'''\n",
    "agg_cols =[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# impute missing values\\n# Impute image_top_1 with max count\\nenc = train.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0]).astype(np.float32).reset_index()\\nenc.columns = ['category_name' ,'image_top_1_impute']\\ntrain = pd.merge(train, enc, how='left', on='category_name')\\ntest = pd.merge(test, enc, how='left', on='category_name')\\ntrain['image_top_1'].fillna(train['image_top_1_impute'], inplace=True)\\ntest['image_top_1'].fillna(test['image_top_1_impute'], inplace=True)\\ntrain.drop('image_top_1_impute', axis=1, inplace=True)\\ntest.drop('image_top_1_impute', axis=1, inplace=True)\\ndel enc; gc.collect()\\n\\n# Impute agg features with med\\nfor f in agg_cols:\\n    enc = train.groupby('category_name')[f].agg('median').astype(np.float32).reset_index()\\n    enc.columns = ['category_name' ,f+'_impute']\\n    train = pd.merge(train, enc, how='left', on='category_name')\\n    test = pd.merge(test, enc, how='left', on='category_name')\\n    train[f].fillna(train[f+'_impute'], inplace=True)\\n    test[f].fillna(test[f+'_impute'], inplace=True)\\n    train.drop(f+'_impute', axis=1, inplace=True)\\n    test.drop(f+'_impute', axis=1, inplace=True)\\n    del enc; gc.collect()\\n    print(f, ' imputed')\\n\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# basic features\n",
    "import string\n",
    "\n",
    "count = lambda l1,l2: sum([1 for x in l1 if x in l2])\n",
    "for df in [train, test]:\n",
    "    df['description'].fillna('unknowndescription', inplace=True)\n",
    "    df['title'].fillna('unknowntitle', inplace=True)\n",
    "\n",
    "    #df['day_to_'] = pd.to_datetime(df['activation_date']).dt.day\n",
    "    df['weekday'] = pd.to_datetime(df['activation_date']).dt.weekday\n",
    "    \n",
    "    for col in ['description', 'title']:\n",
    "        df['num_words_' + col] = df[col].apply(lambda comment: len(comment.split()))\n",
    "        df['num_unique_words_' + col] = df[col].apply(lambda comment: len(set(w for w in comment.split())))\n",
    "\n",
    "    df['words_vs_unique_title'] = df['num_unique_words_title'] / df['num_words_title'] * 100\n",
    "    df['words_vs_unique_description'] = df['num_unique_words_description'] / df['num_words_description'] * 100\n",
    "    \n",
    "    df['city'] = df['city'] + ' ' + df['region']\n",
    "    df['num_desc_punct'] = df['description'].apply(lambda x: count(x, set(string.punctuation)))\n",
    "    \n",
    "            \n",
    "    for col in agg_cols+['price', 'image_top_1']:\n",
    "         df[col].fillna(-1, inplace=True)\n",
    "    \n",
    "    for col in ['param_1', 'param_2', 'param_3']:\n",
    "         df[col].fillna('unknown'+col, inplace=True)\n",
    "    \n",
    "    for col in ['price']:\n",
    "         df[col].fillna(-1, inplace=True)\n",
    "            \n",
    "    \n",
    "'''\n",
    "# impute missing values\n",
    "# Impute image_top_1 with max count\n",
    "enc = train.groupby('category_name')['image_top_1'].agg(lambda x:x.value_counts().index[0]).astype(np.float32).reset_index()\n",
    "enc.columns = ['category_name' ,'image_top_1_impute']\n",
    "train = pd.merge(train, enc, how='left', on='category_name')\n",
    "test = pd.merge(test, enc, how='left', on='category_name')\n",
    "train['image_top_1'].fillna(train['image_top_1_impute'], inplace=True)\n",
    "test['image_top_1'].fillna(test['image_top_1_impute'], inplace=True)\n",
    "train.drop('image_top_1_impute', axis=1, inplace=True)\n",
    "test.drop('image_top_1_impute', axis=1, inplace=True)\n",
    "del enc; gc.collect()\n",
    "\n",
    "# Impute agg features with med\n",
    "for f in agg_cols:\n",
    "    enc = train.groupby('category_name')[f].agg('median').astype(np.float32).reset_index()\n",
    "    enc.columns = ['category_name' ,f+'_impute']\n",
    "    train = pd.merge(train, enc, how='left', on='category_name')\n",
    "    test = pd.merge(test, enc, how='left', on='category_name')\n",
    "    train[f].fillna(train[f+'_impute'], inplace=True)\n",
    "    test[f].fillna(test[f+'_impute'], inplace=True)\n",
    "    train.drop(f+'_impute', axis=1, inplace=True)\n",
    "    test.drop(f+'_impute', axis=1, inplace=True)\n",
    "    del enc; gc.collect()\n",
    "    print(f, ' imputed')\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Other advanced features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merge_cols = []\n",
    "merge_cat = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "42"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# vgg 16 pca features\n",
    "merge = pd.read_pickle('vgg16_image_pca_feature.csv_pkl')\n",
    "merge_cols.extend(merge.columns.tolist()[1:]) # column 0 is image id\n",
    "\n",
    "train = train.merge(merge, on='image', how='left')\n",
    "test = test.merge(merge, on='image', how='left')\n",
    "\n",
    "del merge; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# other pretrained net predictions\n",
    "files = ['image_top_1_resnet_tsvd.csv_pkl',\n",
    "         'image_top_1_inception_tsvd.csv_pkl',\n",
    "         'image_top_1_xception_tsvd.csv_pkl']\n",
    "\n",
    "for fp in files:\n",
    "    merge = pd.read_pickle(fp)\n",
    "    merge_cols.extend(merge.columns.tolist()[1:]) # column 0 is image id\n",
    "\n",
    "    train = train.merge(merge, on='image', how='left')\n",
    "    test = test.merge(merge, on='image', how='left')\n",
    "\n",
    "    del merge; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# opencv features\n",
    "merge = pd.read_pickle('train_keypoints')\n",
    "merge_cols.extend(merge.columns.tolist())\n",
    "train = pd.concat([train, merge], axis=1)\n",
    "del merge; gc.collect()\n",
    "\n",
    "merge = pd.read_pickle('test_keypoints')\n",
    "test = pd.concat([test, merge], axis=1)\n",
    "del merge; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# region macros:\n",
    "merge = pd.read_csv('region_macro.csv')\n",
    "merge_cols.extend(merge.columns.tolist()[1:]) # column 0 is region\n",
    "\n",
    "train = train.merge(merge, on='region', how='left')\n",
    "test = test.merge(merge, on='region', how='left')\n",
    "\n",
    "del merge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>deal_probability</th>\n",
       "      <th>vgg16_image_pca_feature_1</th>\n",
       "      <th>vgg16_image_pca_feature_2</th>\n",
       "      <th>vgg16_image_pca_feature_3</th>\n",
       "      <th>vgg16_image_pca_feature_4</th>\n",
       "      <th>vgg16_image_pca_feature_5</th>\n",
       "      <th>vgg16_image_pca_feature_6</th>\n",
       "      <th>vgg16_image_pca_feature_7</th>\n",
       "      <th>vgg16_image_pca_feature_8</th>\n",
       "      <th>vgg16_image_pca_feature_9</th>\n",
       "      <th>...</th>\n",
       "      <th>image_top_1_inception_tsvd_1</th>\n",
       "      <th>image_top_1_inception_tsvd_2</th>\n",
       "      <th>image_top_1_xception_tsvd_0</th>\n",
       "      <th>image_top_1_xception_tsvd_1</th>\n",
       "      <th>image_top_1_xception_tsvd_2</th>\n",
       "      <th>corner</th>\n",
       "      <th>keypoint</th>\n",
       "      <th>unemployment_rate</th>\n",
       "      <th>GDP_PC_PPP</th>\n",
       "      <th>HDI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>deal_probability</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.198504</td>\n",
       "      <td>0.055239</td>\n",
       "      <td>-0.062594</td>\n",
       "      <td>-0.112870</td>\n",
       "      <td>0.024013</td>\n",
       "      <td>0.012015</td>\n",
       "      <td>0.027724</td>\n",
       "      <td>0.016041</td>\n",
       "      <td>-0.025976</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.034509</td>\n",
       "      <td>-0.040940</td>\n",
       "      <td>-0.018907</td>\n",
       "      <td>-0.032871</td>\n",
       "      <td>-0.063092</td>\n",
       "      <td>-0.025657</td>\n",
       "      <td>-0.018589</td>\n",
       "      <td>-0.007327</td>\n",
       "      <td>-0.006204</td>\n",
       "      <td>-0.007730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_1</th>\n",
       "      <td>0.198504</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.000674</td>\n",
       "      <td>-0.001731</td>\n",
       "      <td>0.003262</td>\n",
       "      <td>0.006726</td>\n",
       "      <td>-0.012170</td>\n",
       "      <td>-0.005115</td>\n",
       "      <td>-0.000616</td>\n",
       "      <td>0.000519</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016344</td>\n",
       "      <td>-0.155211</td>\n",
       "      <td>-0.024092</td>\n",
       "      <td>-0.015289</td>\n",
       "      <td>-0.207773</td>\n",
       "      <td>-0.101334</td>\n",
       "      <td>-0.164998</td>\n",
       "      <td>-0.035786</td>\n",
       "      <td>0.020745</td>\n",
       "      <td>0.027859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_2</th>\n",
       "      <td>0.055239</td>\n",
       "      <td>-0.000674</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004513</td>\n",
       "      <td>-0.002249</td>\n",
       "      <td>-0.004327</td>\n",
       "      <td>-0.008510</td>\n",
       "      <td>-0.003731</td>\n",
       "      <td>-0.007622</td>\n",
       "      <td>0.006898</td>\n",
       "      <td>...</td>\n",
       "      <td>0.174510</td>\n",
       "      <td>-0.054680</td>\n",
       "      <td>0.183707</td>\n",
       "      <td>0.177794</td>\n",
       "      <td>-0.056959</td>\n",
       "      <td>0.049915</td>\n",
       "      <td>0.143314</td>\n",
       "      <td>-0.004640</td>\n",
       "      <td>-0.016885</td>\n",
       "      <td>-0.025344</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_3</th>\n",
       "      <td>-0.062594</td>\n",
       "      <td>-0.001731</td>\n",
       "      <td>0.004513</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.004964</td>\n",
       "      <td>0.005331</td>\n",
       "      <td>0.006862</td>\n",
       "      <td>0.006180</td>\n",
       "      <td>0.001889</td>\n",
       "      <td>-0.003499</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029241</td>\n",
       "      <td>0.033030</td>\n",
       "      <td>-0.009767</td>\n",
       "      <td>-0.032573</td>\n",
       "      <td>0.055859</td>\n",
       "      <td>0.046569</td>\n",
       "      <td>0.067952</td>\n",
       "      <td>0.016858</td>\n",
       "      <td>0.007530</td>\n",
       "      <td>0.002575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_4</th>\n",
       "      <td>-0.112870</td>\n",
       "      <td>0.003262</td>\n",
       "      <td>-0.002249</td>\n",
       "      <td>-0.004964</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.002058</td>\n",
       "      <td>-0.005583</td>\n",
       "      <td>0.000574</td>\n",
       "      <td>-0.003321</td>\n",
       "      <td>0.001678</td>\n",
       "      <td>...</td>\n",
       "      <td>0.084966</td>\n",
       "      <td>-0.014753</td>\n",
       "      <td>0.084109</td>\n",
       "      <td>0.085674</td>\n",
       "      <td>-0.021150</td>\n",
       "      <td>-0.040514</td>\n",
       "      <td>-0.146763</td>\n",
       "      <td>0.004372</td>\n",
       "      <td>-0.006720</td>\n",
       "      <td>0.002337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_5</th>\n",
       "      <td>0.024013</td>\n",
       "      <td>0.006726</td>\n",
       "      <td>-0.004327</td>\n",
       "      <td>0.005331</td>\n",
       "      <td>-0.002058</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.005496</td>\n",
       "      <td>0.000706</td>\n",
       "      <td>-0.017257</td>\n",
       "      <td>0.005988</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.031581</td>\n",
       "      <td>-0.074676</td>\n",
       "      <td>-0.056995</td>\n",
       "      <td>-0.028253</td>\n",
       "      <td>-0.101987</td>\n",
       "      <td>0.161868</td>\n",
       "      <td>0.240648</td>\n",
       "      <td>0.005020</td>\n",
       "      <td>0.004308</td>\n",
       "      <td>0.006604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_6</th>\n",
       "      <td>0.012015</td>\n",
       "      <td>-0.012170</td>\n",
       "      <td>-0.008510</td>\n",
       "      <td>0.006862</td>\n",
       "      <td>-0.005583</td>\n",
       "      <td>-0.005496</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.005300</td>\n",
       "      <td>-0.005890</td>\n",
       "      <td>-0.000090</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.136386</td>\n",
       "      <td>0.164593</td>\n",
       "      <td>-0.164042</td>\n",
       "      <td>-0.142529</td>\n",
       "      <td>0.076480</td>\n",
       "      <td>-0.045213</td>\n",
       "      <td>-0.055713</td>\n",
       "      <td>-0.020014</td>\n",
       "      <td>0.004831</td>\n",
       "      <td>0.009853</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_7</th>\n",
       "      <td>0.027724</td>\n",
       "      <td>-0.005115</td>\n",
       "      <td>-0.003731</td>\n",
       "      <td>0.006180</td>\n",
       "      <td>0.000574</td>\n",
       "      <td>0.000706</td>\n",
       "      <td>-0.005300</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000284</td>\n",
       "      <td>0.001419</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.129130</td>\n",
       "      <td>-0.035673</td>\n",
       "      <td>-0.113163</td>\n",
       "      <td>-0.118766</td>\n",
       "      <td>-0.024631</td>\n",
       "      <td>0.266293</td>\n",
       "      <td>0.357230</td>\n",
       "      <td>-0.013813</td>\n",
       "      <td>-0.003976</td>\n",
       "      <td>-0.005617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_8</th>\n",
       "      <td>0.016041</td>\n",
       "      <td>-0.000616</td>\n",
       "      <td>-0.007622</td>\n",
       "      <td>0.001889</td>\n",
       "      <td>-0.003321</td>\n",
       "      <td>-0.017257</td>\n",
       "      <td>-0.005890</td>\n",
       "      <td>0.000284</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.008373</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007587</td>\n",
       "      <td>0.008872</td>\n",
       "      <td>-0.011020</td>\n",
       "      <td>0.001085</td>\n",
       "      <td>0.044610</td>\n",
       "      <td>-0.065216</td>\n",
       "      <td>-0.098503</td>\n",
       "      <td>0.007318</td>\n",
       "      <td>0.007302</td>\n",
       "      <td>0.005305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_9</th>\n",
       "      <td>-0.025976</td>\n",
       "      <td>0.000519</td>\n",
       "      <td>0.006898</td>\n",
       "      <td>-0.003499</td>\n",
       "      <td>0.001678</td>\n",
       "      <td>0.005988</td>\n",
       "      <td>-0.000090</td>\n",
       "      <td>0.001419</td>\n",
       "      <td>0.008373</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008726</td>\n",
       "      <td>-0.015901</td>\n",
       "      <td>0.064436</td>\n",
       "      <td>0.020258</td>\n",
       "      <td>-0.047626</td>\n",
       "      <td>-0.111781</td>\n",
       "      <td>-0.140270</td>\n",
       "      <td>-0.006359</td>\n",
       "      <td>0.016775</td>\n",
       "      <td>0.019899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_10</th>\n",
       "      <td>0.035435</td>\n",
       "      <td>-0.001191</td>\n",
       "      <td>-0.009936</td>\n",
       "      <td>0.004563</td>\n",
       "      <td>-0.002312</td>\n",
       "      <td>-0.003966</td>\n",
       "      <td>-0.001688</td>\n",
       "      <td>-0.000535</td>\n",
       "      <td>-0.007754</td>\n",
       "      <td>0.006219</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.016304</td>\n",
       "      <td>0.056849</td>\n",
       "      <td>-0.043940</td>\n",
       "      <td>-0.012988</td>\n",
       "      <td>-0.086506</td>\n",
       "      <td>-0.261298</td>\n",
       "      <td>-0.259910</td>\n",
       "      <td>-0.009784</td>\n",
       "      <td>-0.001185</td>\n",
       "      <td>-0.011437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_11</th>\n",
       "      <td>0.021842</td>\n",
       "      <td>-0.001155</td>\n",
       "      <td>0.004088</td>\n",
       "      <td>-0.004426</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>-0.001187</td>\n",
       "      <td>0.007404</td>\n",
       "      <td>-0.001852</td>\n",
       "      <td>0.003573</td>\n",
       "      <td>-0.000105</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.046724</td>\n",
       "      <td>0.058678</td>\n",
       "      <td>0.007624</td>\n",
       "      <td>-0.052340</td>\n",
       "      <td>0.041661</td>\n",
       "      <td>-0.070796</td>\n",
       "      <td>-0.036000</td>\n",
       "      <td>0.010515</td>\n",
       "      <td>0.006814</td>\n",
       "      <td>0.000376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_12</th>\n",
       "      <td>-0.049615</td>\n",
       "      <td>0.000359</td>\n",
       "      <td>-0.001052</td>\n",
       "      <td>0.003923</td>\n",
       "      <td>0.004820</td>\n",
       "      <td>-0.001410</td>\n",
       "      <td>-0.000007</td>\n",
       "      <td>-0.003814</td>\n",
       "      <td>-0.000090</td>\n",
       "      <td>0.001930</td>\n",
       "      <td>...</td>\n",
       "      <td>0.097254</td>\n",
       "      <td>0.012913</td>\n",
       "      <td>0.137586</td>\n",
       "      <td>0.087242</td>\n",
       "      <td>0.130352</td>\n",
       "      <td>0.108341</td>\n",
       "      <td>0.161320</td>\n",
       "      <td>-0.000516</td>\n",
       "      <td>-0.008541</td>\n",
       "      <td>-0.006505</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_13</th>\n",
       "      <td>-0.044503</td>\n",
       "      <td>-0.000432</td>\n",
       "      <td>0.001890</td>\n",
       "      <td>-0.001684</td>\n",
       "      <td>0.000601</td>\n",
       "      <td>-0.000452</td>\n",
       "      <td>0.002582</td>\n",
       "      <td>0.005405</td>\n",
       "      <td>-0.001975</td>\n",
       "      <td>0.001398</td>\n",
       "      <td>...</td>\n",
       "      <td>0.028737</td>\n",
       "      <td>0.055556</td>\n",
       "      <td>0.058481</td>\n",
       "      <td>0.034415</td>\n",
       "      <td>-0.026629</td>\n",
       "      <td>-0.000048</td>\n",
       "      <td>0.005194</td>\n",
       "      <td>0.002673</td>\n",
       "      <td>0.006895</td>\n",
       "      <td>0.007179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_14</th>\n",
       "      <td>0.002965</td>\n",
       "      <td>0.000750</td>\n",
       "      <td>0.003453</td>\n",
       "      <td>-0.000957</td>\n",
       "      <td>-0.001202</td>\n",
       "      <td>0.004679</td>\n",
       "      <td>-0.003405</td>\n",
       "      <td>0.001275</td>\n",
       "      <td>0.001356</td>\n",
       "      <td>0.001142</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.010189</td>\n",
       "      <td>-0.071888</td>\n",
       "      <td>0.000059</td>\n",
       "      <td>-0.015568</td>\n",
       "      <td>-0.076831</td>\n",
       "      <td>0.105507</td>\n",
       "      <td>0.078608</td>\n",
       "      <td>-0.001148</td>\n",
       "      <td>-0.001210</td>\n",
       "      <td>0.005084</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_15</th>\n",
       "      <td>0.001471</td>\n",
       "      <td>0.006521</td>\n",
       "      <td>0.001252</td>\n",
       "      <td>-0.001743</td>\n",
       "      <td>0.000710</td>\n",
       "      <td>-0.000552</td>\n",
       "      <td>-0.001403</td>\n",
       "      <td>0.001511</td>\n",
       "      <td>0.001299</td>\n",
       "      <td>0.000465</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.070619</td>\n",
       "      <td>-0.059111</td>\n",
       "      <td>-0.070181</td>\n",
       "      <td>-0.075463</td>\n",
       "      <td>0.042678</td>\n",
       "      <td>-0.051641</td>\n",
       "      <td>-0.067026</td>\n",
       "      <td>-0.007526</td>\n",
       "      <td>-0.000863</td>\n",
       "      <td>0.002631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_16</th>\n",
       "      <td>-0.001063</td>\n",
       "      <td>-0.000701</td>\n",
       "      <td>-0.000679</td>\n",
       "      <td>-0.002211</td>\n",
       "      <td>-0.000832</td>\n",
       "      <td>0.004305</td>\n",
       "      <td>0.004052</td>\n",
       "      <td>0.003322</td>\n",
       "      <td>0.002829</td>\n",
       "      <td>-0.002261</td>\n",
       "      <td>...</td>\n",
       "      <td>0.041170</td>\n",
       "      <td>0.013833</td>\n",
       "      <td>0.084856</td>\n",
       "      <td>0.040491</td>\n",
       "      <td>-0.089641</td>\n",
       "      <td>-0.040459</td>\n",
       "      <td>-0.034711</td>\n",
       "      <td>-0.005801</td>\n",
       "      <td>0.002328</td>\n",
       "      <td>0.008010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_17</th>\n",
       "      <td>-0.031690</td>\n",
       "      <td>-0.001435</td>\n",
       "      <td>0.001890</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>-0.004809</td>\n",
       "      <td>-0.003370</td>\n",
       "      <td>-0.001838</td>\n",
       "      <td>-0.000474</td>\n",
       "      <td>-0.001583</td>\n",
       "      <td>-0.002150</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.040690</td>\n",
       "      <td>0.031371</td>\n",
       "      <td>-0.017308</td>\n",
       "      <td>-0.035713</td>\n",
       "      <td>-0.068507</td>\n",
       "      <td>0.013181</td>\n",
       "      <td>0.038124</td>\n",
       "      <td>0.001464</td>\n",
       "      <td>-0.007527</td>\n",
       "      <td>-0.008833</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_18</th>\n",
       "      <td>-0.011732</td>\n",
       "      <td>-0.005494</td>\n",
       "      <td>-0.002783</td>\n",
       "      <td>0.005696</td>\n",
       "      <td>-0.001029</td>\n",
       "      <td>-0.007096</td>\n",
       "      <td>0.001449</td>\n",
       "      <td>-0.004951</td>\n",
       "      <td>-0.006836</td>\n",
       "      <td>0.004060</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009171</td>\n",
       "      <td>0.051357</td>\n",
       "      <td>-0.011322</td>\n",
       "      <td>0.011690</td>\n",
       "      <td>0.098845</td>\n",
       "      <td>-0.042387</td>\n",
       "      <td>-0.083247</td>\n",
       "      <td>0.004063</td>\n",
       "      <td>0.002658</td>\n",
       "      <td>0.002711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_19</th>\n",
       "      <td>0.025820</td>\n",
       "      <td>-0.001093</td>\n",
       "      <td>-0.000869</td>\n",
       "      <td>-0.000281</td>\n",
       "      <td>-0.000948</td>\n",
       "      <td>0.001089</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>-0.002601</td>\n",
       "      <td>-0.003443</td>\n",
       "      <td>0.002117</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.045438</td>\n",
       "      <td>0.016053</td>\n",
       "      <td>-0.096741</td>\n",
       "      <td>-0.054898</td>\n",
       "      <td>0.072088</td>\n",
       "      <td>0.042840</td>\n",
       "      <td>0.044546</td>\n",
       "      <td>0.002148</td>\n",
       "      <td>-0.005539</td>\n",
       "      <td>-0.009171</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_20</th>\n",
       "      <td>0.002769</td>\n",
       "      <td>-0.001511</td>\n",
       "      <td>0.000829</td>\n",
       "      <td>0.001670</td>\n",
       "      <td>-0.000072</td>\n",
       "      <td>-0.004755</td>\n",
       "      <td>-0.002766</td>\n",
       "      <td>-0.000995</td>\n",
       "      <td>-0.006825</td>\n",
       "      <td>0.005444</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.039731</td>\n",
       "      <td>-0.031203</td>\n",
       "      <td>-0.011680</td>\n",
       "      <td>-0.044321</td>\n",
       "      <td>-0.045357</td>\n",
       "      <td>0.070068</td>\n",
       "      <td>0.083423</td>\n",
       "      <td>0.001910</td>\n",
       "      <td>0.002989</td>\n",
       "      <td>0.001895</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_21</th>\n",
       "      <td>0.000634</td>\n",
       "      <td>-0.004088</td>\n",
       "      <td>-0.000372</td>\n",
       "      <td>0.003661</td>\n",
       "      <td>0.001692</td>\n",
       "      <td>0.003952</td>\n",
       "      <td>-0.000375</td>\n",
       "      <td>-0.000183</td>\n",
       "      <td>0.004676</td>\n",
       "      <td>-0.003467</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.031861</td>\n",
       "      <td>-0.014388</td>\n",
       "      <td>-0.117806</td>\n",
       "      <td>-0.034738</td>\n",
       "      <td>0.053094</td>\n",
       "      <td>0.058872</td>\n",
       "      <td>0.052305</td>\n",
       "      <td>-0.003304</td>\n",
       "      <td>-0.002133</td>\n",
       "      <td>0.002147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_22</th>\n",
       "      <td>-0.027759</td>\n",
       "      <td>0.002276</td>\n",
       "      <td>0.001570</td>\n",
       "      <td>-0.000340</td>\n",
       "      <td>0.000372</td>\n",
       "      <td>0.003376</td>\n",
       "      <td>0.002382</td>\n",
       "      <td>-0.004313</td>\n",
       "      <td>0.001903</td>\n",
       "      <td>-0.003631</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.029876</td>\n",
       "      <td>0.064433</td>\n",
       "      <td>-0.034676</td>\n",
       "      <td>-0.033941</td>\n",
       "      <td>0.017067</td>\n",
       "      <td>0.074124</td>\n",
       "      <td>0.007588</td>\n",
       "      <td>0.000215</td>\n",
       "      <td>-0.003564</td>\n",
       "      <td>-0.001424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_23</th>\n",
       "      <td>-0.014000</td>\n",
       "      <td>0.002620</td>\n",
       "      <td>0.000318</td>\n",
       "      <td>0.001686</td>\n",
       "      <td>-0.000380</td>\n",
       "      <td>-0.001802</td>\n",
       "      <td>-0.000808</td>\n",
       "      <td>0.004144</td>\n",
       "      <td>-0.002345</td>\n",
       "      <td>0.000375</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.012546</td>\n",
       "      <td>-0.006114</td>\n",
       "      <td>-0.135128</td>\n",
       "      <td>-0.004845</td>\n",
       "      <td>-0.013325</td>\n",
       "      <td>0.015166</td>\n",
       "      <td>0.015224</td>\n",
       "      <td>-0.001210</td>\n",
       "      <td>0.002033</td>\n",
       "      <td>0.002063</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_24</th>\n",
       "      <td>0.005797</td>\n",
       "      <td>0.000821</td>\n",
       "      <td>-0.005225</td>\n",
       "      <td>0.001244</td>\n",
       "      <td>-0.001452</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>-0.006791</td>\n",
       "      <td>-0.001291</td>\n",
       "      <td>-0.008513</td>\n",
       "      <td>0.002891</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.062896</td>\n",
       "      <td>-0.045889</td>\n",
       "      <td>-0.047124</td>\n",
       "      <td>-0.066610</td>\n",
       "      <td>0.061886</td>\n",
       "      <td>-0.001177</td>\n",
       "      <td>-0.014377</td>\n",
       "      <td>-0.000156</td>\n",
       "      <td>-0.000564</td>\n",
       "      <td>-0.006614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_25</th>\n",
       "      <td>-0.013174</td>\n",
       "      <td>-0.000442</td>\n",
       "      <td>0.006098</td>\n",
       "      <td>-0.000776</td>\n",
       "      <td>-0.003115</td>\n",
       "      <td>-0.005054</td>\n",
       "      <td>-0.004380</td>\n",
       "      <td>0.000326</td>\n",
       "      <td>0.001356</td>\n",
       "      <td>0.002226</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.031335</td>\n",
       "      <td>-0.032927</td>\n",
       "      <td>-0.063899</td>\n",
       "      <td>-0.042549</td>\n",
       "      <td>0.161833</td>\n",
       "      <td>-0.040495</td>\n",
       "      <td>-0.060890</td>\n",
       "      <td>-0.001036</td>\n",
       "      <td>-0.001842</td>\n",
       "      <td>0.000115</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_26</th>\n",
       "      <td>-0.006416</td>\n",
       "      <td>0.001145</td>\n",
       "      <td>0.000198</td>\n",
       "      <td>-0.001624</td>\n",
       "      <td>-0.001073</td>\n",
       "      <td>0.001797</td>\n",
       "      <td>-0.002796</td>\n",
       "      <td>0.000015</td>\n",
       "      <td>0.000532</td>\n",
       "      <td>0.001391</td>\n",
       "      <td>...</td>\n",
       "      <td>0.007155</td>\n",
       "      <td>-0.048667</td>\n",
       "      <td>0.034254</td>\n",
       "      <td>0.001358</td>\n",
       "      <td>-0.068537</td>\n",
       "      <td>-0.027346</td>\n",
       "      <td>-0.065563</td>\n",
       "      <td>-0.011480</td>\n",
       "      <td>0.005756</td>\n",
       "      <td>0.006256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_27</th>\n",
       "      <td>0.024825</td>\n",
       "      <td>0.003951</td>\n",
       "      <td>-0.000521</td>\n",
       "      <td>-0.006204</td>\n",
       "      <td>-0.002832</td>\n",
       "      <td>0.003983</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>-0.000810</td>\n",
       "      <td>0.003700</td>\n",
       "      <td>-0.001473</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.030731</td>\n",
       "      <td>-0.007523</td>\n",
       "      <td>-0.053658</td>\n",
       "      <td>-0.024252</td>\n",
       "      <td>0.076556</td>\n",
       "      <td>-0.142719</td>\n",
       "      <td>-0.156574</td>\n",
       "      <td>-0.004698</td>\n",
       "      <td>-0.001143</td>\n",
       "      <td>0.004994</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_28</th>\n",
       "      <td>-0.025925</td>\n",
       "      <td>0.003406</td>\n",
       "      <td>-0.002781</td>\n",
       "      <td>0.002607</td>\n",
       "      <td>-0.001192</td>\n",
       "      <td>-0.006133</td>\n",
       "      <td>-0.003260</td>\n",
       "      <td>-0.005168</td>\n",
       "      <td>-0.006443</td>\n",
       "      <td>0.006447</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000700</td>\n",
       "      <td>-0.012232</td>\n",
       "      <td>0.054993</td>\n",
       "      <td>-0.009612</td>\n",
       "      <td>0.044536</td>\n",
       "      <td>0.056673</td>\n",
       "      <td>0.021915</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>-0.004227</td>\n",
       "      <td>-0.002682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_29</th>\n",
       "      <td>0.000518</td>\n",
       "      <td>-0.002147</td>\n",
       "      <td>-0.001127</td>\n",
       "      <td>0.001588</td>\n",
       "      <td>-0.002447</td>\n",
       "      <td>0.003420</td>\n",
       "      <td>-0.004111</td>\n",
       "      <td>0.003277</td>\n",
       "      <td>0.001267</td>\n",
       "      <td>-0.001804</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002510</td>\n",
       "      <td>-0.026039</td>\n",
       "      <td>-0.020485</td>\n",
       "      <td>-0.012323</td>\n",
       "      <td>0.059228</td>\n",
       "      <td>-0.075472</td>\n",
       "      <td>-0.060477</td>\n",
       "      <td>0.014480</td>\n",
       "      <td>-0.007148</td>\n",
       "      <td>-0.010535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>vgg16_image_pca_feature_30</th>\n",
       "      <td>0.011036</td>\n",
       "      <td>-0.000232</td>\n",
       "      <td>0.003316</td>\n",
       "      <td>-0.002749</td>\n",
       "      <td>-0.000286</td>\n",
       "      <td>0.002653</td>\n",
       "      <td>0.005978</td>\n",
       "      <td>0.002327</td>\n",
       "      <td>0.002301</td>\n",
       "      <td>-0.000741</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.062834</td>\n",
       "      <td>0.027602</td>\n",
       "      <td>-0.086456</td>\n",
       "      <td>-0.060343</td>\n",
       "      <td>-0.075135</td>\n",
       "      <td>-0.089351</td>\n",
       "      <td>-0.086642</td>\n",
       "      <td>0.005317</td>\n",
       "      <td>0.011827</td>\n",
       "      <td>0.008544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_resnet_tsvd_0</th>\n",
       "      <td>-0.033303</td>\n",
       "      <td>-0.015938</td>\n",
       "      <td>0.194827</td>\n",
       "      <td>-0.023191</td>\n",
       "      <td>0.085014</td>\n",
       "      <td>-0.029754</td>\n",
       "      <td>-0.143653</td>\n",
       "      <td>-0.140541</td>\n",
       "      <td>-0.010778</td>\n",
       "      <td>0.014839</td>\n",
       "      <td>...</td>\n",
       "      <td>0.498831</td>\n",
       "      <td>-0.016859</td>\n",
       "      <td>0.100934</td>\n",
       "      <td>0.527092</td>\n",
       "      <td>-0.027949</td>\n",
       "      <td>-0.013923</td>\n",
       "      <td>-0.012299</td>\n",
       "      <td>0.002085</td>\n",
       "      <td>-0.007154</td>\n",
       "      <td>-0.009587</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_resnet_tsvd_1</th>\n",
       "      <td>-0.002556</td>\n",
       "      <td>0.104910</td>\n",
       "      <td>-0.134015</td>\n",
       "      <td>0.047253</td>\n",
       "      <td>0.269618</td>\n",
       "      <td>0.038206</td>\n",
       "      <td>0.019145</td>\n",
       "      <td>0.143119</td>\n",
       "      <td>-0.018757</td>\n",
       "      <td>0.104230</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.013331</td>\n",
       "      <td>-0.007914</td>\n",
       "      <td>-0.014936</td>\n",
       "      <td>-0.013403</td>\n",
       "      <td>-0.014488</td>\n",
       "      <td>0.009775</td>\n",
       "      <td>-0.015181</td>\n",
       "      <td>-0.006733</td>\n",
       "      <td>0.010129</td>\n",
       "      <td>0.012442</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_resnet_tsvd_2</th>\n",
       "      <td>-0.014933</td>\n",
       "      <td>-0.016233</td>\n",
       "      <td>0.153236</td>\n",
       "      <td>-0.009453</td>\n",
       "      <td>0.078366</td>\n",
       "      <td>-0.049482</td>\n",
       "      <td>-0.145313</td>\n",
       "      <td>-0.085963</td>\n",
       "      <td>-0.002974</td>\n",
       "      <td>0.059234</td>\n",
       "      <td>...</td>\n",
       "      <td>0.054521</td>\n",
       "      <td>-0.014720</td>\n",
       "      <td>0.629718</td>\n",
       "      <td>0.057500</td>\n",
       "      <td>-0.023055</td>\n",
       "      <td>-0.013981</td>\n",
       "      <td>-0.008718</td>\n",
       "      <td>0.008380</td>\n",
       "      <td>-0.008352</td>\n",
       "      <td>-0.005044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_inception_tsvd_0</th>\n",
       "      <td>-0.020061</td>\n",
       "      <td>-0.026740</td>\n",
       "      <td>0.175061</td>\n",
       "      <td>-0.006717</td>\n",
       "      <td>0.083497</td>\n",
       "      <td>-0.052784</td>\n",
       "      <td>-0.166914</td>\n",
       "      <td>-0.109156</td>\n",
       "      <td>-0.010759</td>\n",
       "      <td>0.066957</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.022502</td>\n",
       "      <td>-0.018466</td>\n",
       "      <td>0.665214</td>\n",
       "      <td>0.079304</td>\n",
       "      <td>-0.027179</td>\n",
       "      <td>-0.010828</td>\n",
       "      <td>-0.005160</td>\n",
       "      <td>0.008730</td>\n",
       "      <td>-0.009669</td>\n",
       "      <td>-0.007150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_inception_tsvd_1</th>\n",
       "      <td>-0.034509</td>\n",
       "      <td>-0.016344</td>\n",
       "      <td>0.174510</td>\n",
       "      <td>-0.029241</td>\n",
       "      <td>0.084966</td>\n",
       "      <td>-0.031581</td>\n",
       "      <td>-0.136386</td>\n",
       "      <td>-0.129130</td>\n",
       "      <td>-0.007587</td>\n",
       "      <td>0.008726</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.017118</td>\n",
       "      <td>0.079617</td>\n",
       "      <td>0.511013</td>\n",
       "      <td>-0.026224</td>\n",
       "      <td>-0.014561</td>\n",
       "      <td>-0.012102</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>-0.005855</td>\n",
       "      <td>-0.008160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_inception_tsvd_2</th>\n",
       "      <td>-0.040940</td>\n",
       "      <td>-0.155211</td>\n",
       "      <td>-0.054680</td>\n",
       "      <td>0.033030</td>\n",
       "      <td>-0.014753</td>\n",
       "      <td>-0.074676</td>\n",
       "      <td>0.164593</td>\n",
       "      <td>-0.035673</td>\n",
       "      <td>0.008872</td>\n",
       "      <td>-0.015901</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.017118</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.018430</td>\n",
       "      <td>-0.016842</td>\n",
       "      <td>0.008757</td>\n",
       "      <td>-0.035657</td>\n",
       "      <td>-0.035363</td>\n",
       "      <td>0.005543</td>\n",
       "      <td>0.000625</td>\n",
       "      <td>-0.002207</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_xception_tsvd_0</th>\n",
       "      <td>-0.018907</td>\n",
       "      <td>-0.024092</td>\n",
       "      <td>0.183707</td>\n",
       "      <td>-0.009767</td>\n",
       "      <td>0.084109</td>\n",
       "      <td>-0.056995</td>\n",
       "      <td>-0.164042</td>\n",
       "      <td>-0.113163</td>\n",
       "      <td>-0.011020</td>\n",
       "      <td>0.064436</td>\n",
       "      <td>...</td>\n",
       "      <td>0.079617</td>\n",
       "      <td>-0.018430</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.023716</td>\n",
       "      <td>-0.028531</td>\n",
       "      <td>-0.011861</td>\n",
       "      <td>-0.005749</td>\n",
       "      <td>0.009409</td>\n",
       "      <td>-0.010146</td>\n",
       "      <td>-0.007236</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_xception_tsvd_1</th>\n",
       "      <td>-0.032871</td>\n",
       "      <td>-0.015289</td>\n",
       "      <td>0.177794</td>\n",
       "      <td>-0.032573</td>\n",
       "      <td>0.085674</td>\n",
       "      <td>-0.028253</td>\n",
       "      <td>-0.142529</td>\n",
       "      <td>-0.118766</td>\n",
       "      <td>0.001085</td>\n",
       "      <td>0.020258</td>\n",
       "      <td>...</td>\n",
       "      <td>0.511013</td>\n",
       "      <td>-0.016842</td>\n",
       "      <td>-0.023716</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.027665</td>\n",
       "      <td>-0.014348</td>\n",
       "      <td>-0.010711</td>\n",
       "      <td>0.002516</td>\n",
       "      <td>-0.005992</td>\n",
       "      <td>-0.008447</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>image_top_1_xception_tsvd_2</th>\n",
       "      <td>-0.063092</td>\n",
       "      <td>-0.207773</td>\n",
       "      <td>-0.056959</td>\n",
       "      <td>0.055859</td>\n",
       "      <td>-0.021150</td>\n",
       "      <td>-0.101987</td>\n",
       "      <td>0.076480</td>\n",
       "      <td>-0.024631</td>\n",
       "      <td>0.044610</td>\n",
       "      <td>-0.047626</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.026224</td>\n",
       "      <td>0.008757</td>\n",
       "      <td>-0.028531</td>\n",
       "      <td>-0.027665</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.004567</td>\n",
       "      <td>-0.006317</td>\n",
       "      <td>0.008414</td>\n",
       "      <td>-0.011725</td>\n",
       "      <td>-0.011588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>corner</th>\n",
       "      <td>-0.025657</td>\n",
       "      <td>-0.101334</td>\n",
       "      <td>0.049915</td>\n",
       "      <td>0.046569</td>\n",
       "      <td>-0.040514</td>\n",
       "      <td>0.161868</td>\n",
       "      <td>-0.045213</td>\n",
       "      <td>0.266293</td>\n",
       "      <td>-0.065216</td>\n",
       "      <td>-0.111781</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.014561</td>\n",
       "      <td>-0.035657</td>\n",
       "      <td>-0.011861</td>\n",
       "      <td>-0.014348</td>\n",
       "      <td>-0.004567</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.756313</td>\n",
       "      <td>0.002269</td>\n",
       "      <td>-0.004174</td>\n",
       "      <td>-0.001124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>keypoint</th>\n",
       "      <td>-0.018589</td>\n",
       "      <td>-0.164998</td>\n",
       "      <td>0.143314</td>\n",
       "      <td>0.067952</td>\n",
       "      <td>-0.146763</td>\n",
       "      <td>0.240648</td>\n",
       "      <td>-0.055713</td>\n",
       "      <td>0.357230</td>\n",
       "      <td>-0.098503</td>\n",
       "      <td>-0.140270</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.012102</td>\n",
       "      <td>-0.035363</td>\n",
       "      <td>-0.005749</td>\n",
       "      <td>-0.010711</td>\n",
       "      <td>-0.006317</td>\n",
       "      <td>0.756313</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.003264</td>\n",
       "      <td>-0.008237</td>\n",
       "      <td>-0.009662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unemployment_rate</th>\n",
       "      <td>-0.007327</td>\n",
       "      <td>-0.035786</td>\n",
       "      <td>-0.004640</td>\n",
       "      <td>0.016858</td>\n",
       "      <td>0.004372</td>\n",
       "      <td>0.005020</td>\n",
       "      <td>-0.020014</td>\n",
       "      <td>-0.013813</td>\n",
       "      <td>0.007318</td>\n",
       "      <td>-0.006359</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002238</td>\n",
       "      <td>0.005543</td>\n",
       "      <td>0.009409</td>\n",
       "      <td>0.002516</td>\n",
       "      <td>0.008414</td>\n",
       "      <td>0.002269</td>\n",
       "      <td>0.003264</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.262496</td>\n",
       "      <td>-0.310529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GDP_PC_PPP</th>\n",
       "      <td>-0.006204</td>\n",
       "      <td>0.020745</td>\n",
       "      <td>-0.016885</td>\n",
       "      <td>0.007530</td>\n",
       "      <td>-0.006720</td>\n",
       "      <td>0.004308</td>\n",
       "      <td>0.004831</td>\n",
       "      <td>-0.003976</td>\n",
       "      <td>0.007302</td>\n",
       "      <td>0.016775</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.005855</td>\n",
       "      <td>0.000625</td>\n",
       "      <td>-0.010146</td>\n",
       "      <td>-0.005992</td>\n",
       "      <td>-0.011725</td>\n",
       "      <td>-0.004174</td>\n",
       "      <td>-0.008237</td>\n",
       "      <td>-0.262496</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.806588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HDI</th>\n",
       "      <td>-0.007730</td>\n",
       "      <td>0.027859</td>\n",
       "      <td>-0.025344</td>\n",
       "      <td>0.002575</td>\n",
       "      <td>0.002337</td>\n",
       "      <td>0.006604</td>\n",
       "      <td>0.009853</td>\n",
       "      <td>-0.005617</td>\n",
       "      <td>0.005305</td>\n",
       "      <td>0.019899</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.008160</td>\n",
       "      <td>-0.002207</td>\n",
       "      <td>-0.007236</td>\n",
       "      <td>-0.008447</td>\n",
       "      <td>-0.011588</td>\n",
       "      <td>-0.001124</td>\n",
       "      <td>-0.009662</td>\n",
       "      <td>-0.310529</td>\n",
       "      <td>0.806588</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>45 rows × 45 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                              deal_probability  vgg16_image_pca_feature_1  \\\n",
       "deal_probability                      1.000000                   0.198504   \n",
       "vgg16_image_pca_feature_1             0.198504                   1.000000   \n",
       "vgg16_image_pca_feature_2             0.055239                  -0.000674   \n",
       "vgg16_image_pca_feature_3            -0.062594                  -0.001731   \n",
       "vgg16_image_pca_feature_4            -0.112870                   0.003262   \n",
       "vgg16_image_pca_feature_5             0.024013                   0.006726   \n",
       "vgg16_image_pca_feature_6             0.012015                  -0.012170   \n",
       "vgg16_image_pca_feature_7             0.027724                  -0.005115   \n",
       "vgg16_image_pca_feature_8             0.016041                  -0.000616   \n",
       "vgg16_image_pca_feature_9            -0.025976                   0.000519   \n",
       "vgg16_image_pca_feature_10            0.035435                  -0.001191   \n",
       "vgg16_image_pca_feature_11            0.021842                  -0.001155   \n",
       "vgg16_image_pca_feature_12           -0.049615                   0.000359   \n",
       "vgg16_image_pca_feature_13           -0.044503                  -0.000432   \n",
       "vgg16_image_pca_feature_14            0.002965                   0.000750   \n",
       "vgg16_image_pca_feature_15            0.001471                   0.006521   \n",
       "vgg16_image_pca_feature_16           -0.001063                  -0.000701   \n",
       "vgg16_image_pca_feature_17           -0.031690                  -0.001435   \n",
       "vgg16_image_pca_feature_18           -0.011732                  -0.005494   \n",
       "vgg16_image_pca_feature_19            0.025820                  -0.001093   \n",
       "vgg16_image_pca_feature_20            0.002769                  -0.001511   \n",
       "vgg16_image_pca_feature_21            0.000634                  -0.004088   \n",
       "vgg16_image_pca_feature_22           -0.027759                   0.002276   \n",
       "vgg16_image_pca_feature_23           -0.014000                   0.002620   \n",
       "vgg16_image_pca_feature_24            0.005797                   0.000821   \n",
       "vgg16_image_pca_feature_25           -0.013174                  -0.000442   \n",
       "vgg16_image_pca_feature_26           -0.006416                   0.001145   \n",
       "vgg16_image_pca_feature_27            0.024825                   0.003951   \n",
       "vgg16_image_pca_feature_28           -0.025925                   0.003406   \n",
       "vgg16_image_pca_feature_29            0.000518                  -0.002147   \n",
       "vgg16_image_pca_feature_30            0.011036                  -0.000232   \n",
       "image_top_1_resnet_tsvd_0            -0.033303                  -0.015938   \n",
       "image_top_1_resnet_tsvd_1            -0.002556                   0.104910   \n",
       "image_top_1_resnet_tsvd_2            -0.014933                  -0.016233   \n",
       "image_top_1_inception_tsvd_0         -0.020061                  -0.026740   \n",
       "image_top_1_inception_tsvd_1         -0.034509                  -0.016344   \n",
       "image_top_1_inception_tsvd_2         -0.040940                  -0.155211   \n",
       "image_top_1_xception_tsvd_0          -0.018907                  -0.024092   \n",
       "image_top_1_xception_tsvd_1          -0.032871                  -0.015289   \n",
       "image_top_1_xception_tsvd_2          -0.063092                  -0.207773   \n",
       "corner                               -0.025657                  -0.101334   \n",
       "keypoint                             -0.018589                  -0.164998   \n",
       "unemployment_rate                    -0.007327                  -0.035786   \n",
       "GDP_PC_PPP                           -0.006204                   0.020745   \n",
       "HDI                                  -0.007730                   0.027859   \n",
       "\n",
       "                              vgg16_image_pca_feature_2  \\\n",
       "deal_probability                               0.055239   \n",
       "vgg16_image_pca_feature_1                     -0.000674   \n",
       "vgg16_image_pca_feature_2                      1.000000   \n",
       "vgg16_image_pca_feature_3                      0.004513   \n",
       "vgg16_image_pca_feature_4                     -0.002249   \n",
       "vgg16_image_pca_feature_5                     -0.004327   \n",
       "vgg16_image_pca_feature_6                     -0.008510   \n",
       "vgg16_image_pca_feature_7                     -0.003731   \n",
       "vgg16_image_pca_feature_8                     -0.007622   \n",
       "vgg16_image_pca_feature_9                      0.006898   \n",
       "vgg16_image_pca_feature_10                    -0.009936   \n",
       "vgg16_image_pca_feature_11                     0.004088   \n",
       "vgg16_image_pca_feature_12                    -0.001052   \n",
       "vgg16_image_pca_feature_13                     0.001890   \n",
       "vgg16_image_pca_feature_14                     0.003453   \n",
       "vgg16_image_pca_feature_15                     0.001252   \n",
       "vgg16_image_pca_feature_16                    -0.000679   \n",
       "vgg16_image_pca_feature_17                     0.001890   \n",
       "vgg16_image_pca_feature_18                    -0.002783   \n",
       "vgg16_image_pca_feature_19                    -0.000869   \n",
       "vgg16_image_pca_feature_20                     0.000829   \n",
       "vgg16_image_pca_feature_21                    -0.000372   \n",
       "vgg16_image_pca_feature_22                     0.001570   \n",
       "vgg16_image_pca_feature_23                     0.000318   \n",
       "vgg16_image_pca_feature_24                    -0.005225   \n",
       "vgg16_image_pca_feature_25                     0.006098   \n",
       "vgg16_image_pca_feature_26                     0.000198   \n",
       "vgg16_image_pca_feature_27                    -0.000521   \n",
       "vgg16_image_pca_feature_28                    -0.002781   \n",
       "vgg16_image_pca_feature_29                    -0.001127   \n",
       "vgg16_image_pca_feature_30                     0.003316   \n",
       "image_top_1_resnet_tsvd_0                      0.194827   \n",
       "image_top_1_resnet_tsvd_1                     -0.134015   \n",
       "image_top_1_resnet_tsvd_2                      0.153236   \n",
       "image_top_1_inception_tsvd_0                   0.175061   \n",
       "image_top_1_inception_tsvd_1                   0.174510   \n",
       "image_top_1_inception_tsvd_2                  -0.054680   \n",
       "image_top_1_xception_tsvd_0                    0.183707   \n",
       "image_top_1_xception_tsvd_1                    0.177794   \n",
       "image_top_1_xception_tsvd_2                   -0.056959   \n",
       "corner                                         0.049915   \n",
       "keypoint                                       0.143314   \n",
       "unemployment_rate                             -0.004640   \n",
       "GDP_PC_PPP                                    -0.016885   \n",
       "HDI                                           -0.025344   \n",
       "\n",
       "                              vgg16_image_pca_feature_3  \\\n",
       "deal_probability                              -0.062594   \n",
       "vgg16_image_pca_feature_1                     -0.001731   \n",
       "vgg16_image_pca_feature_2                      0.004513   \n",
       "vgg16_image_pca_feature_3                      1.000000   \n",
       "vgg16_image_pca_feature_4                     -0.004964   \n",
       "vgg16_image_pca_feature_5                      0.005331   \n",
       "vgg16_image_pca_feature_6                      0.006862   \n",
       "vgg16_image_pca_feature_7                      0.006180   \n",
       "vgg16_image_pca_feature_8                      0.001889   \n",
       "vgg16_image_pca_feature_9                     -0.003499   \n",
       "vgg16_image_pca_feature_10                     0.004563   \n",
       "vgg16_image_pca_feature_11                    -0.004426   \n",
       "vgg16_image_pca_feature_12                     0.003923   \n",
       "vgg16_image_pca_feature_13                    -0.001684   \n",
       "vgg16_image_pca_feature_14                    -0.000957   \n",
       "vgg16_image_pca_feature_15                    -0.001743   \n",
       "vgg16_image_pca_feature_16                    -0.002211   \n",
       "vgg16_image_pca_feature_17                     0.000213   \n",
       "vgg16_image_pca_feature_18                     0.005696   \n",
       "vgg16_image_pca_feature_19                    -0.000281   \n",
       "vgg16_image_pca_feature_20                     0.001670   \n",
       "vgg16_image_pca_feature_21                     0.003661   \n",
       "vgg16_image_pca_feature_22                    -0.000340   \n",
       "vgg16_image_pca_feature_23                     0.001686   \n",
       "vgg16_image_pca_feature_24                     0.001244   \n",
       "vgg16_image_pca_feature_25                    -0.000776   \n",
       "vgg16_image_pca_feature_26                    -0.001624   \n",
       "vgg16_image_pca_feature_27                    -0.006204   \n",
       "vgg16_image_pca_feature_28                     0.002607   \n",
       "vgg16_image_pca_feature_29                     0.001588   \n",
       "vgg16_image_pca_feature_30                    -0.002749   \n",
       "image_top_1_resnet_tsvd_0                     -0.023191   \n",
       "image_top_1_resnet_tsvd_1                      0.047253   \n",
       "image_top_1_resnet_tsvd_2                     -0.009453   \n",
       "image_top_1_inception_tsvd_0                  -0.006717   \n",
       "image_top_1_inception_tsvd_1                  -0.029241   \n",
       "image_top_1_inception_tsvd_2                   0.033030   \n",
       "image_top_1_xception_tsvd_0                   -0.009767   \n",
       "image_top_1_xception_tsvd_1                   -0.032573   \n",
       "image_top_1_xception_tsvd_2                    0.055859   \n",
       "corner                                         0.046569   \n",
       "keypoint                                       0.067952   \n",
       "unemployment_rate                              0.016858   \n",
       "GDP_PC_PPP                                     0.007530   \n",
       "HDI                                            0.002575   \n",
       "\n",
       "                              vgg16_image_pca_feature_4  \\\n",
       "deal_probability                              -0.112870   \n",
       "vgg16_image_pca_feature_1                      0.003262   \n",
       "vgg16_image_pca_feature_2                     -0.002249   \n",
       "vgg16_image_pca_feature_3                     -0.004964   \n",
       "vgg16_image_pca_feature_4                      1.000000   \n",
       "vgg16_image_pca_feature_5                     -0.002058   \n",
       "vgg16_image_pca_feature_6                     -0.005583   \n",
       "vgg16_image_pca_feature_7                      0.000574   \n",
       "vgg16_image_pca_feature_8                     -0.003321   \n",
       "vgg16_image_pca_feature_9                      0.001678   \n",
       "vgg16_image_pca_feature_10                    -0.002312   \n",
       "vgg16_image_pca_feature_11                     0.000014   \n",
       "vgg16_image_pca_feature_12                     0.004820   \n",
       "vgg16_image_pca_feature_13                     0.000601   \n",
       "vgg16_image_pca_feature_14                    -0.001202   \n",
       "vgg16_image_pca_feature_15                     0.000710   \n",
       "vgg16_image_pca_feature_16                    -0.000832   \n",
       "vgg16_image_pca_feature_17                    -0.004809   \n",
       "vgg16_image_pca_feature_18                    -0.001029   \n",
       "vgg16_image_pca_feature_19                    -0.000948   \n",
       "vgg16_image_pca_feature_20                    -0.000072   \n",
       "vgg16_image_pca_feature_21                     0.001692   \n",
       "vgg16_image_pca_feature_22                     0.000372   \n",
       "vgg16_image_pca_feature_23                    -0.000380   \n",
       "vgg16_image_pca_feature_24                    -0.001452   \n",
       "vgg16_image_pca_feature_25                    -0.003115   \n",
       "vgg16_image_pca_feature_26                    -0.001073   \n",
       "vgg16_image_pca_feature_27                    -0.002832   \n",
       "vgg16_image_pca_feature_28                    -0.001192   \n",
       "vgg16_image_pca_feature_29                    -0.002447   \n",
       "vgg16_image_pca_feature_30                    -0.000286   \n",
       "image_top_1_resnet_tsvd_0                      0.085014   \n",
       "image_top_1_resnet_tsvd_1                      0.269618   \n",
       "image_top_1_resnet_tsvd_2                      0.078366   \n",
       "image_top_1_inception_tsvd_0                   0.083497   \n",
       "image_top_1_inception_tsvd_1                   0.084966   \n",
       "image_top_1_inception_tsvd_2                  -0.014753   \n",
       "image_top_1_xception_tsvd_0                    0.084109   \n",
       "image_top_1_xception_tsvd_1                    0.085674   \n",
       "image_top_1_xception_tsvd_2                   -0.021150   \n",
       "corner                                        -0.040514   \n",
       "keypoint                                      -0.146763   \n",
       "unemployment_rate                              0.004372   \n",
       "GDP_PC_PPP                                    -0.006720   \n",
       "HDI                                            0.002337   \n",
       "\n",
       "                              vgg16_image_pca_feature_5  \\\n",
       "deal_probability                               0.024013   \n",
       "vgg16_image_pca_feature_1                      0.006726   \n",
       "vgg16_image_pca_feature_2                     -0.004327   \n",
       "vgg16_image_pca_feature_3                      0.005331   \n",
       "vgg16_image_pca_feature_4                     -0.002058   \n",
       "vgg16_image_pca_feature_5                      1.000000   \n",
       "vgg16_image_pca_feature_6                     -0.005496   \n",
       "vgg16_image_pca_feature_7                      0.000706   \n",
       "vgg16_image_pca_feature_8                     -0.017257   \n",
       "vgg16_image_pca_feature_9                      0.005988   \n",
       "vgg16_image_pca_feature_10                    -0.003966   \n",
       "vgg16_image_pca_feature_11                    -0.001187   \n",
       "vgg16_image_pca_feature_12                    -0.001410   \n",
       "vgg16_image_pca_feature_13                    -0.000452   \n",
       "vgg16_image_pca_feature_14                     0.004679   \n",
       "vgg16_image_pca_feature_15                    -0.000552   \n",
       "vgg16_image_pca_feature_16                     0.004305   \n",
       "vgg16_image_pca_feature_17                    -0.003370   \n",
       "vgg16_image_pca_feature_18                    -0.007096   \n",
       "vgg16_image_pca_feature_19                     0.001089   \n",
       "vgg16_image_pca_feature_20                    -0.004755   \n",
       "vgg16_image_pca_feature_21                     0.003952   \n",
       "vgg16_image_pca_feature_22                     0.003376   \n",
       "vgg16_image_pca_feature_23                    -0.001802   \n",
       "vgg16_image_pca_feature_24                     0.000045   \n",
       "vgg16_image_pca_feature_25                    -0.005054   \n",
       "vgg16_image_pca_feature_26                     0.001797   \n",
       "vgg16_image_pca_feature_27                     0.003983   \n",
       "vgg16_image_pca_feature_28                    -0.006133   \n",
       "vgg16_image_pca_feature_29                     0.003420   \n",
       "vgg16_image_pca_feature_30                     0.002653   \n",
       "image_top_1_resnet_tsvd_0                     -0.029754   \n",
       "image_top_1_resnet_tsvd_1                      0.038206   \n",
       "image_top_1_resnet_tsvd_2                     -0.049482   \n",
       "image_top_1_inception_tsvd_0                  -0.052784   \n",
       "image_top_1_inception_tsvd_1                  -0.031581   \n",
       "image_top_1_inception_tsvd_2                  -0.074676   \n",
       "image_top_1_xception_tsvd_0                   -0.056995   \n",
       "image_top_1_xception_tsvd_1                   -0.028253   \n",
       "image_top_1_xception_tsvd_2                   -0.101987   \n",
       "corner                                         0.161868   \n",
       "keypoint                                       0.240648   \n",
       "unemployment_rate                              0.005020   \n",
       "GDP_PC_PPP                                     0.004308   \n",
       "HDI                                            0.006604   \n",
       "\n",
       "                              vgg16_image_pca_feature_6  \\\n",
       "deal_probability                               0.012015   \n",
       "vgg16_image_pca_feature_1                     -0.012170   \n",
       "vgg16_image_pca_feature_2                     -0.008510   \n",
       "vgg16_image_pca_feature_3                      0.006862   \n",
       "vgg16_image_pca_feature_4                     -0.005583   \n",
       "vgg16_image_pca_feature_5                     -0.005496   \n",
       "vgg16_image_pca_feature_6                      1.000000   \n",
       "vgg16_image_pca_feature_7                     -0.005300   \n",
       "vgg16_image_pca_feature_8                     -0.005890   \n",
       "vgg16_image_pca_feature_9                     -0.000090   \n",
       "vgg16_image_pca_feature_10                    -0.001688   \n",
       "vgg16_image_pca_feature_11                     0.007404   \n",
       "vgg16_image_pca_feature_12                    -0.000007   \n",
       "vgg16_image_pca_feature_13                     0.002582   \n",
       "vgg16_image_pca_feature_14                    -0.003405   \n",
       "vgg16_image_pca_feature_15                    -0.001403   \n",
       "vgg16_image_pca_feature_16                     0.004052   \n",
       "vgg16_image_pca_feature_17                    -0.001838   \n",
       "vgg16_image_pca_feature_18                     0.001449   \n",
       "vgg16_image_pca_feature_19                     0.000213   \n",
       "vgg16_image_pca_feature_20                    -0.002766   \n",
       "vgg16_image_pca_feature_21                    -0.000375   \n",
       "vgg16_image_pca_feature_22                     0.002382   \n",
       "vgg16_image_pca_feature_23                    -0.000808   \n",
       "vgg16_image_pca_feature_24                    -0.006791   \n",
       "vgg16_image_pca_feature_25                    -0.004380   \n",
       "vgg16_image_pca_feature_26                    -0.002796   \n",
       "vgg16_image_pca_feature_27                     0.000583   \n",
       "vgg16_image_pca_feature_28                    -0.003260   \n",
       "vgg16_image_pca_feature_29                    -0.004111   \n",
       "vgg16_image_pca_feature_30                     0.005978   \n",
       "image_top_1_resnet_tsvd_0                     -0.143653   \n",
       "image_top_1_resnet_tsvd_1                      0.019145   \n",
       "image_top_1_resnet_tsvd_2                     -0.145313   \n",
       "image_top_1_inception_tsvd_0                  -0.166914   \n",
       "image_top_1_inception_tsvd_1                  -0.136386   \n",
       "image_top_1_inception_tsvd_2                   0.164593   \n",
       "image_top_1_xception_tsvd_0                   -0.164042   \n",
       "image_top_1_xception_tsvd_1                   -0.142529   \n",
       "image_top_1_xception_tsvd_2                    0.076480   \n",
       "corner                                        -0.045213   \n",
       "keypoint                                      -0.055713   \n",
       "unemployment_rate                             -0.020014   \n",
       "GDP_PC_PPP                                     0.004831   \n",
       "HDI                                            0.009853   \n",
       "\n",
       "                              vgg16_image_pca_feature_7  \\\n",
       "deal_probability                               0.027724   \n",
       "vgg16_image_pca_feature_1                     -0.005115   \n",
       "vgg16_image_pca_feature_2                     -0.003731   \n",
       "vgg16_image_pca_feature_3                      0.006180   \n",
       "vgg16_image_pca_feature_4                      0.000574   \n",
       "vgg16_image_pca_feature_5                      0.000706   \n",
       "vgg16_image_pca_feature_6                     -0.005300   \n",
       "vgg16_image_pca_feature_7                      1.000000   \n",
       "vgg16_image_pca_feature_8                      0.000284   \n",
       "vgg16_image_pca_feature_9                      0.001419   \n",
       "vgg16_image_pca_feature_10                    -0.000535   \n",
       "vgg16_image_pca_feature_11                    -0.001852   \n",
       "vgg16_image_pca_feature_12                    -0.003814   \n",
       "vgg16_image_pca_feature_13                     0.005405   \n",
       "vgg16_image_pca_feature_14                     0.001275   \n",
       "vgg16_image_pca_feature_15                     0.001511   \n",
       "vgg16_image_pca_feature_16                     0.003322   \n",
       "vgg16_image_pca_feature_17                    -0.000474   \n",
       "vgg16_image_pca_feature_18                    -0.004951   \n",
       "vgg16_image_pca_feature_19                    -0.002601   \n",
       "vgg16_image_pca_feature_20                    -0.000995   \n",
       "vgg16_image_pca_feature_21                    -0.000183   \n",
       "vgg16_image_pca_feature_22                    -0.004313   \n",
       "vgg16_image_pca_feature_23                     0.004144   \n",
       "vgg16_image_pca_feature_24                    -0.001291   \n",
       "vgg16_image_pca_feature_25                     0.000326   \n",
       "vgg16_image_pca_feature_26                     0.000015   \n",
       "vgg16_image_pca_feature_27                    -0.000810   \n",
       "vgg16_image_pca_feature_28                    -0.005168   \n",
       "vgg16_image_pca_feature_29                     0.003277   \n",
       "vgg16_image_pca_feature_30                     0.002327   \n",
       "image_top_1_resnet_tsvd_0                     -0.140541   \n",
       "image_top_1_resnet_tsvd_1                      0.143119   \n",
       "image_top_1_resnet_tsvd_2                     -0.085963   \n",
       "image_top_1_inception_tsvd_0                  -0.109156   \n",
       "image_top_1_inception_tsvd_1                  -0.129130   \n",
       "image_top_1_inception_tsvd_2                  -0.035673   \n",
       "image_top_1_xception_tsvd_0                   -0.113163   \n",
       "image_top_1_xception_tsvd_1                   -0.118766   \n",
       "image_top_1_xception_tsvd_2                   -0.024631   \n",
       "corner                                         0.266293   \n",
       "keypoint                                       0.357230   \n",
       "unemployment_rate                             -0.013813   \n",
       "GDP_PC_PPP                                    -0.003976   \n",
       "HDI                                           -0.005617   \n",
       "\n",
       "                              vgg16_image_pca_feature_8  \\\n",
       "deal_probability                               0.016041   \n",
       "vgg16_image_pca_feature_1                     -0.000616   \n",
       "vgg16_image_pca_feature_2                     -0.007622   \n",
       "vgg16_image_pca_feature_3                      0.001889   \n",
       "vgg16_image_pca_feature_4                     -0.003321   \n",
       "vgg16_image_pca_feature_5                     -0.017257   \n",
       "vgg16_image_pca_feature_6                     -0.005890   \n",
       "vgg16_image_pca_feature_7                      0.000284   \n",
       "vgg16_image_pca_feature_8                      1.000000   \n",
       "vgg16_image_pca_feature_9                      0.008373   \n",
       "vgg16_image_pca_feature_10                    -0.007754   \n",
       "vgg16_image_pca_feature_11                     0.003573   \n",
       "vgg16_image_pca_feature_12                    -0.000090   \n",
       "vgg16_image_pca_feature_13                    -0.001975   \n",
       "vgg16_image_pca_feature_14                     0.001356   \n",
       "vgg16_image_pca_feature_15                     0.001299   \n",
       "vgg16_image_pca_feature_16                     0.002829   \n",
       "vgg16_image_pca_feature_17                    -0.001583   \n",
       "vgg16_image_pca_feature_18                    -0.006836   \n",
       "vgg16_image_pca_feature_19                    -0.003443   \n",
       "vgg16_image_pca_feature_20                    -0.006825   \n",
       "vgg16_image_pca_feature_21                     0.004676   \n",
       "vgg16_image_pca_feature_22                     0.001903   \n",
       "vgg16_image_pca_feature_23                    -0.002345   \n",
       "vgg16_image_pca_feature_24                    -0.008513   \n",
       "vgg16_image_pca_feature_25                     0.001356   \n",
       "vgg16_image_pca_feature_26                     0.000532   \n",
       "vgg16_image_pca_feature_27                     0.003700   \n",
       "vgg16_image_pca_feature_28                    -0.006443   \n",
       "vgg16_image_pca_feature_29                     0.001267   \n",
       "vgg16_image_pca_feature_30                     0.002301   \n",
       "image_top_1_resnet_tsvd_0                     -0.010778   \n",
       "image_top_1_resnet_tsvd_1                     -0.018757   \n",
       "image_top_1_resnet_tsvd_2                     -0.002974   \n",
       "image_top_1_inception_tsvd_0                  -0.010759   \n",
       "image_top_1_inception_tsvd_1                  -0.007587   \n",
       "image_top_1_inception_tsvd_2                   0.008872   \n",
       "image_top_1_xception_tsvd_0                   -0.011020   \n",
       "image_top_1_xception_tsvd_1                    0.001085   \n",
       "image_top_1_xception_tsvd_2                    0.044610   \n",
       "corner                                        -0.065216   \n",
       "keypoint                                      -0.098503   \n",
       "unemployment_rate                              0.007318   \n",
       "GDP_PC_PPP                                     0.007302   \n",
       "HDI                                            0.005305   \n",
       "\n",
       "                              vgg16_image_pca_feature_9    ...     \\\n",
       "deal_probability                              -0.025976    ...      \n",
       "vgg16_image_pca_feature_1                      0.000519    ...      \n",
       "vgg16_image_pca_feature_2                      0.006898    ...      \n",
       "vgg16_image_pca_feature_3                     -0.003499    ...      \n",
       "vgg16_image_pca_feature_4                      0.001678    ...      \n",
       "vgg16_image_pca_feature_5                      0.005988    ...      \n",
       "vgg16_image_pca_feature_6                     -0.000090    ...      \n",
       "vgg16_image_pca_feature_7                      0.001419    ...      \n",
       "vgg16_image_pca_feature_8                      0.008373    ...      \n",
       "vgg16_image_pca_feature_9                      1.000000    ...      \n",
       "vgg16_image_pca_feature_10                     0.006219    ...      \n",
       "vgg16_image_pca_feature_11                    -0.000105    ...      \n",
       "vgg16_image_pca_feature_12                     0.001930    ...      \n",
       "vgg16_image_pca_feature_13                     0.001398    ...      \n",
       "vgg16_image_pca_feature_14                     0.001142    ...      \n",
       "vgg16_image_pca_feature_15                     0.000465    ...      \n",
       "vgg16_image_pca_feature_16                    -0.002261    ...      \n",
       "vgg16_image_pca_feature_17                    -0.002150    ...      \n",
       "vgg16_image_pca_feature_18                     0.004060    ...      \n",
       "vgg16_image_pca_feature_19                     0.002117    ...      \n",
       "vgg16_image_pca_feature_20                     0.005444    ...      \n",
       "vgg16_image_pca_feature_21                    -0.003467    ...      \n",
       "vgg16_image_pca_feature_22                    -0.003631    ...      \n",
       "vgg16_image_pca_feature_23                     0.000375    ...      \n",
       "vgg16_image_pca_feature_24                     0.002891    ...      \n",
       "vgg16_image_pca_feature_25                     0.002226    ...      \n",
       "vgg16_image_pca_feature_26                     0.001391    ...      \n",
       "vgg16_image_pca_feature_27                    -0.001473    ...      \n",
       "vgg16_image_pca_feature_28                     0.006447    ...      \n",
       "vgg16_image_pca_feature_29                    -0.001804    ...      \n",
       "vgg16_image_pca_feature_30                    -0.000741    ...      \n",
       "image_top_1_resnet_tsvd_0                      0.014839    ...      \n",
       "image_top_1_resnet_tsvd_1                      0.104230    ...      \n",
       "image_top_1_resnet_tsvd_2                      0.059234    ...      \n",
       "image_top_1_inception_tsvd_0                   0.066957    ...      \n",
       "image_top_1_inception_tsvd_1                   0.008726    ...      \n",
       "image_top_1_inception_tsvd_2                  -0.015901    ...      \n",
       "image_top_1_xception_tsvd_0                    0.064436    ...      \n",
       "image_top_1_xception_tsvd_1                    0.020258    ...      \n",
       "image_top_1_xception_tsvd_2                   -0.047626    ...      \n",
       "corner                                        -0.111781    ...      \n",
       "keypoint                                      -0.140270    ...      \n",
       "unemployment_rate                             -0.006359    ...      \n",
       "GDP_PC_PPP                                     0.016775    ...      \n",
       "HDI                                            0.019899    ...      \n",
       "\n",
       "                              image_top_1_inception_tsvd_1  \\\n",
       "deal_probability                                 -0.034509   \n",
       "vgg16_image_pca_feature_1                        -0.016344   \n",
       "vgg16_image_pca_feature_2                         0.174510   \n",
       "vgg16_image_pca_feature_3                        -0.029241   \n",
       "vgg16_image_pca_feature_4                         0.084966   \n",
       "vgg16_image_pca_feature_5                        -0.031581   \n",
       "vgg16_image_pca_feature_6                        -0.136386   \n",
       "vgg16_image_pca_feature_7                        -0.129130   \n",
       "vgg16_image_pca_feature_8                        -0.007587   \n",
       "vgg16_image_pca_feature_9                         0.008726   \n",
       "vgg16_image_pca_feature_10                       -0.016304   \n",
       "vgg16_image_pca_feature_11                       -0.046724   \n",
       "vgg16_image_pca_feature_12                        0.097254   \n",
       "vgg16_image_pca_feature_13                        0.028737   \n",
       "vgg16_image_pca_feature_14                       -0.010189   \n",
       "vgg16_image_pca_feature_15                       -0.070619   \n",
       "vgg16_image_pca_feature_16                        0.041170   \n",
       "vgg16_image_pca_feature_17                       -0.040690   \n",
       "vgg16_image_pca_feature_18                        0.009171   \n",
       "vgg16_image_pca_feature_19                       -0.045438   \n",
       "vgg16_image_pca_feature_20                       -0.039731   \n",
       "vgg16_image_pca_feature_21                       -0.031861   \n",
       "vgg16_image_pca_feature_22                       -0.029876   \n",
       "vgg16_image_pca_feature_23                       -0.012546   \n",
       "vgg16_image_pca_feature_24                       -0.062896   \n",
       "vgg16_image_pca_feature_25                       -0.031335   \n",
       "vgg16_image_pca_feature_26                        0.007155   \n",
       "vgg16_image_pca_feature_27                       -0.030731   \n",
       "vgg16_image_pca_feature_28                       -0.000700   \n",
       "vgg16_image_pca_feature_29                       -0.002510   \n",
       "vgg16_image_pca_feature_30                       -0.062834   \n",
       "image_top_1_resnet_tsvd_0                         0.498831   \n",
       "image_top_1_resnet_tsvd_1                        -0.013331   \n",
       "image_top_1_resnet_tsvd_2                         0.054521   \n",
       "image_top_1_inception_tsvd_0                     -0.022502   \n",
       "image_top_1_inception_tsvd_1                      1.000000   \n",
       "image_top_1_inception_tsvd_2                     -0.017118   \n",
       "image_top_1_xception_tsvd_0                       0.079617   \n",
       "image_top_1_xception_tsvd_1                       0.511013   \n",
       "image_top_1_xception_tsvd_2                      -0.026224   \n",
       "corner                                           -0.014561   \n",
       "keypoint                                         -0.012102   \n",
       "unemployment_rate                                 0.002238   \n",
       "GDP_PC_PPP                                       -0.005855   \n",
       "HDI                                              -0.008160   \n",
       "\n",
       "                              image_top_1_inception_tsvd_2  \\\n",
       "deal_probability                                 -0.040940   \n",
       "vgg16_image_pca_feature_1                        -0.155211   \n",
       "vgg16_image_pca_feature_2                        -0.054680   \n",
       "vgg16_image_pca_feature_3                         0.033030   \n",
       "vgg16_image_pca_feature_4                        -0.014753   \n",
       "vgg16_image_pca_feature_5                        -0.074676   \n",
       "vgg16_image_pca_feature_6                         0.164593   \n",
       "vgg16_image_pca_feature_7                        -0.035673   \n",
       "vgg16_image_pca_feature_8                         0.008872   \n",
       "vgg16_image_pca_feature_9                        -0.015901   \n",
       "vgg16_image_pca_feature_10                        0.056849   \n",
       "vgg16_image_pca_feature_11                        0.058678   \n",
       "vgg16_image_pca_feature_12                        0.012913   \n",
       "vgg16_image_pca_feature_13                        0.055556   \n",
       "vgg16_image_pca_feature_14                       -0.071888   \n",
       "vgg16_image_pca_feature_15                       -0.059111   \n",
       "vgg16_image_pca_feature_16                        0.013833   \n",
       "vgg16_image_pca_feature_17                        0.031371   \n",
       "vgg16_image_pca_feature_18                        0.051357   \n",
       "vgg16_image_pca_feature_19                        0.016053   \n",
       "vgg16_image_pca_feature_20                       -0.031203   \n",
       "vgg16_image_pca_feature_21                       -0.014388   \n",
       "vgg16_image_pca_feature_22                        0.064433   \n",
       "vgg16_image_pca_feature_23                       -0.006114   \n",
       "vgg16_image_pca_feature_24                       -0.045889   \n",
       "vgg16_image_pca_feature_25                       -0.032927   \n",
       "vgg16_image_pca_feature_26                       -0.048667   \n",
       "vgg16_image_pca_feature_27                       -0.007523   \n",
       "vgg16_image_pca_feature_28                       -0.012232   \n",
       "vgg16_image_pca_feature_29                       -0.026039   \n",
       "vgg16_image_pca_feature_30                        0.027602   \n",
       "image_top_1_resnet_tsvd_0                        -0.016859   \n",
       "image_top_1_resnet_tsvd_1                        -0.007914   \n",
       "image_top_1_resnet_tsvd_2                        -0.014720   \n",
       "image_top_1_inception_tsvd_0                     -0.018466   \n",
       "image_top_1_inception_tsvd_1                     -0.017118   \n",
       "image_top_1_inception_tsvd_2                      1.000000   \n",
       "image_top_1_xception_tsvd_0                      -0.018430   \n",
       "image_top_1_xception_tsvd_1                      -0.016842   \n",
       "image_top_1_xception_tsvd_2                       0.008757   \n",
       "corner                                           -0.035657   \n",
       "keypoint                                         -0.035363   \n",
       "unemployment_rate                                 0.005543   \n",
       "GDP_PC_PPP                                        0.000625   \n",
       "HDI                                              -0.002207   \n",
       "\n",
       "                              image_top_1_xception_tsvd_0  \\\n",
       "deal_probability                                -0.018907   \n",
       "vgg16_image_pca_feature_1                       -0.024092   \n",
       "vgg16_image_pca_feature_2                        0.183707   \n",
       "vgg16_image_pca_feature_3                       -0.009767   \n",
       "vgg16_image_pca_feature_4                        0.084109   \n",
       "vgg16_image_pca_feature_5                       -0.056995   \n",
       "vgg16_image_pca_feature_6                       -0.164042   \n",
       "vgg16_image_pca_feature_7                       -0.113163   \n",
       "vgg16_image_pca_feature_8                       -0.011020   \n",
       "vgg16_image_pca_feature_9                        0.064436   \n",
       "vgg16_image_pca_feature_10                      -0.043940   \n",
       "vgg16_image_pca_feature_11                       0.007624   \n",
       "vgg16_image_pca_feature_12                       0.137586   \n",
       "vgg16_image_pca_feature_13                       0.058481   \n",
       "vgg16_image_pca_feature_14                       0.000059   \n",
       "vgg16_image_pca_feature_15                      -0.070181   \n",
       "vgg16_image_pca_feature_16                       0.084856   \n",
       "vgg16_image_pca_feature_17                      -0.017308   \n",
       "vgg16_image_pca_feature_18                      -0.011322   \n",
       "vgg16_image_pca_feature_19                      -0.096741   \n",
       "vgg16_image_pca_feature_20                      -0.011680   \n",
       "vgg16_image_pca_feature_21                      -0.117806   \n",
       "vgg16_image_pca_feature_22                      -0.034676   \n",
       "vgg16_image_pca_feature_23                      -0.135128   \n",
       "vgg16_image_pca_feature_24                      -0.047124   \n",
       "vgg16_image_pca_feature_25                      -0.063899   \n",
       "vgg16_image_pca_feature_26                       0.034254   \n",
       "vgg16_image_pca_feature_27                      -0.053658   \n",
       "vgg16_image_pca_feature_28                       0.054993   \n",
       "vgg16_image_pca_feature_29                      -0.020485   \n",
       "vgg16_image_pca_feature_30                      -0.086456   \n",
       "image_top_1_resnet_tsvd_0                        0.100934   \n",
       "image_top_1_resnet_tsvd_1                       -0.014936   \n",
       "image_top_1_resnet_tsvd_2                        0.629718   \n",
       "image_top_1_inception_tsvd_0                     0.665214   \n",
       "image_top_1_inception_tsvd_1                     0.079617   \n",
       "image_top_1_inception_tsvd_2                    -0.018430   \n",
       "image_top_1_xception_tsvd_0                      1.000000   \n",
       "image_top_1_xception_tsvd_1                     -0.023716   \n",
       "image_top_1_xception_tsvd_2                     -0.028531   \n",
       "corner                                          -0.011861   \n",
       "keypoint                                        -0.005749   \n",
       "unemployment_rate                                0.009409   \n",
       "GDP_PC_PPP                                      -0.010146   \n",
       "HDI                                             -0.007236   \n",
       "\n",
       "                              image_top_1_xception_tsvd_1  \\\n",
       "deal_probability                                -0.032871   \n",
       "vgg16_image_pca_feature_1                       -0.015289   \n",
       "vgg16_image_pca_feature_2                        0.177794   \n",
       "vgg16_image_pca_feature_3                       -0.032573   \n",
       "vgg16_image_pca_feature_4                        0.085674   \n",
       "vgg16_image_pca_feature_5                       -0.028253   \n",
       "vgg16_image_pca_feature_6                       -0.142529   \n",
       "vgg16_image_pca_feature_7                       -0.118766   \n",
       "vgg16_image_pca_feature_8                        0.001085   \n",
       "vgg16_image_pca_feature_9                        0.020258   \n",
       "vgg16_image_pca_feature_10                      -0.012988   \n",
       "vgg16_image_pca_feature_11                      -0.052340   \n",
       "vgg16_image_pca_feature_12                       0.087242   \n",
       "vgg16_image_pca_feature_13                       0.034415   \n",
       "vgg16_image_pca_feature_14                      -0.015568   \n",
       "vgg16_image_pca_feature_15                      -0.075463   \n",
       "vgg16_image_pca_feature_16                       0.040491   \n",
       "vgg16_image_pca_feature_17                      -0.035713   \n",
       "vgg16_image_pca_feature_18                       0.011690   \n",
       "vgg16_image_pca_feature_19                      -0.054898   \n",
       "vgg16_image_pca_feature_20                      -0.044321   \n",
       "vgg16_image_pca_feature_21                      -0.034738   \n",
       "vgg16_image_pca_feature_22                      -0.033941   \n",
       "vgg16_image_pca_feature_23                      -0.004845   \n",
       "vgg16_image_pca_feature_24                      -0.066610   \n",
       "vgg16_image_pca_feature_25                      -0.042549   \n",
       "vgg16_image_pca_feature_26                       0.001358   \n",
       "vgg16_image_pca_feature_27                      -0.024252   \n",
       "vgg16_image_pca_feature_28                      -0.009612   \n",
       "vgg16_image_pca_feature_29                      -0.012323   \n",
       "vgg16_image_pca_feature_30                      -0.060343   \n",
       "image_top_1_resnet_tsvd_0                        0.527092   \n",
       "image_top_1_resnet_tsvd_1                       -0.013403   \n",
       "image_top_1_resnet_tsvd_2                        0.057500   \n",
       "image_top_1_inception_tsvd_0                     0.079304   \n",
       "image_top_1_inception_tsvd_1                     0.511013   \n",
       "image_top_1_inception_tsvd_2                    -0.016842   \n",
       "image_top_1_xception_tsvd_0                     -0.023716   \n",
       "image_top_1_xception_tsvd_1                      1.000000   \n",
       "image_top_1_xception_tsvd_2                     -0.027665   \n",
       "corner                                          -0.014348   \n",
       "keypoint                                        -0.010711   \n",
       "unemployment_rate                                0.002516   \n",
       "GDP_PC_PPP                                      -0.005992   \n",
       "HDI                                             -0.008447   \n",
       "\n",
       "                              image_top_1_xception_tsvd_2    corner  keypoint  \\\n",
       "deal_probability                                -0.063092 -0.025657 -0.018589   \n",
       "vgg16_image_pca_feature_1                       -0.207773 -0.101334 -0.164998   \n",
       "vgg16_image_pca_feature_2                       -0.056959  0.049915  0.143314   \n",
       "vgg16_image_pca_feature_3                        0.055859  0.046569  0.067952   \n",
       "vgg16_image_pca_feature_4                       -0.021150 -0.040514 -0.146763   \n",
       "vgg16_image_pca_feature_5                       -0.101987  0.161868  0.240648   \n",
       "vgg16_image_pca_feature_6                        0.076480 -0.045213 -0.055713   \n",
       "vgg16_image_pca_feature_7                       -0.024631  0.266293  0.357230   \n",
       "vgg16_image_pca_feature_8                        0.044610 -0.065216 -0.098503   \n",
       "vgg16_image_pca_feature_9                       -0.047626 -0.111781 -0.140270   \n",
       "vgg16_image_pca_feature_10                      -0.086506 -0.261298 -0.259910   \n",
       "vgg16_image_pca_feature_11                       0.041661 -0.070796 -0.036000   \n",
       "vgg16_image_pca_feature_12                       0.130352  0.108341  0.161320   \n",
       "vgg16_image_pca_feature_13                      -0.026629 -0.000048  0.005194   \n",
       "vgg16_image_pca_feature_14                      -0.076831  0.105507  0.078608   \n",
       "vgg16_image_pca_feature_15                       0.042678 -0.051641 -0.067026   \n",
       "vgg16_image_pca_feature_16                      -0.089641 -0.040459 -0.034711   \n",
       "vgg16_image_pca_feature_17                      -0.068507  0.013181  0.038124   \n",
       "vgg16_image_pca_feature_18                       0.098845 -0.042387 -0.083247   \n",
       "vgg16_image_pca_feature_19                       0.072088  0.042840  0.044546   \n",
       "vgg16_image_pca_feature_20                      -0.045357  0.070068  0.083423   \n",
       "vgg16_image_pca_feature_21                       0.053094  0.058872  0.052305   \n",
       "vgg16_image_pca_feature_22                       0.017067  0.074124  0.007588   \n",
       "vgg16_image_pca_feature_23                      -0.013325  0.015166  0.015224   \n",
       "vgg16_image_pca_feature_24                       0.061886 -0.001177 -0.014377   \n",
       "vgg16_image_pca_feature_25                       0.161833 -0.040495 -0.060890   \n",
       "vgg16_image_pca_feature_26                      -0.068537 -0.027346 -0.065563   \n",
       "vgg16_image_pca_feature_27                       0.076556 -0.142719 -0.156574   \n",
       "vgg16_image_pca_feature_28                       0.044536  0.056673  0.021915   \n",
       "vgg16_image_pca_feature_29                       0.059228 -0.075472 -0.060477   \n",
       "vgg16_image_pca_feature_30                      -0.075135 -0.089351 -0.086642   \n",
       "image_top_1_resnet_tsvd_0                       -0.027949 -0.013923 -0.012299   \n",
       "image_top_1_resnet_tsvd_1                       -0.014488  0.009775 -0.015181   \n",
       "image_top_1_resnet_tsvd_2                       -0.023055 -0.013981 -0.008718   \n",
       "image_top_1_inception_tsvd_0                    -0.027179 -0.010828 -0.005160   \n",
       "image_top_1_inception_tsvd_1                    -0.026224 -0.014561 -0.012102   \n",
       "image_top_1_inception_tsvd_2                     0.008757 -0.035657 -0.035363   \n",
       "image_top_1_xception_tsvd_0                     -0.028531 -0.011861 -0.005749   \n",
       "image_top_1_xception_tsvd_1                     -0.027665 -0.014348 -0.010711   \n",
       "image_top_1_xception_tsvd_2                      1.000000 -0.004567 -0.006317   \n",
       "corner                                          -0.004567  1.000000  0.756313   \n",
       "keypoint                                        -0.006317  0.756313  1.000000   \n",
       "unemployment_rate                                0.008414  0.002269  0.003264   \n",
       "GDP_PC_PPP                                      -0.011725 -0.004174 -0.008237   \n",
       "HDI                                             -0.011588 -0.001124 -0.009662   \n",
       "\n",
       "                              unemployment_rate  GDP_PC_PPP       HDI  \n",
       "deal_probability                      -0.007327   -0.006204 -0.007730  \n",
       "vgg16_image_pca_feature_1             -0.035786    0.020745  0.027859  \n",
       "vgg16_image_pca_feature_2             -0.004640   -0.016885 -0.025344  \n",
       "vgg16_image_pca_feature_3              0.016858    0.007530  0.002575  \n",
       "vgg16_image_pca_feature_4              0.004372   -0.006720  0.002337  \n",
       "vgg16_image_pca_feature_5              0.005020    0.004308  0.006604  \n",
       "vgg16_image_pca_feature_6             -0.020014    0.004831  0.009853  \n",
       "vgg16_image_pca_feature_7             -0.013813   -0.003976 -0.005617  \n",
       "vgg16_image_pca_feature_8              0.007318    0.007302  0.005305  \n",
       "vgg16_image_pca_feature_9             -0.006359    0.016775  0.019899  \n",
       "vgg16_image_pca_feature_10            -0.009784   -0.001185 -0.011437  \n",
       "vgg16_image_pca_feature_11             0.010515    0.006814  0.000376  \n",
       "vgg16_image_pca_feature_12            -0.000516   -0.008541 -0.006505  \n",
       "vgg16_image_pca_feature_13             0.002673    0.006895  0.007179  \n",
       "vgg16_image_pca_feature_14            -0.001148   -0.001210  0.005084  \n",
       "vgg16_image_pca_feature_15            -0.007526   -0.000863  0.002631  \n",
       "vgg16_image_pca_feature_16            -0.005801    0.002328  0.008010  \n",
       "vgg16_image_pca_feature_17             0.001464   -0.007527 -0.008833  \n",
       "vgg16_image_pca_feature_18             0.004063    0.002658  0.002711  \n",
       "vgg16_image_pca_feature_19             0.002148   -0.005539 -0.009171  \n",
       "vgg16_image_pca_feature_20             0.001910    0.002989  0.001895  \n",
       "vgg16_image_pca_feature_21            -0.003304   -0.002133  0.002147  \n",
       "vgg16_image_pca_feature_22             0.000215   -0.003564 -0.001424  \n",
       "vgg16_image_pca_feature_23            -0.001210    0.002033  0.002063  \n",
       "vgg16_image_pca_feature_24            -0.000156   -0.000564 -0.006614  \n",
       "vgg16_image_pca_feature_25            -0.001036   -0.001842  0.000115  \n",
       "vgg16_image_pca_feature_26            -0.011480    0.005756  0.006256  \n",
       "vgg16_image_pca_feature_27            -0.004698   -0.001143  0.004994  \n",
       "vgg16_image_pca_feature_28             0.006667   -0.004227 -0.002682  \n",
       "vgg16_image_pca_feature_29             0.014480   -0.007148 -0.010535  \n",
       "vgg16_image_pca_feature_30             0.005317    0.011827  0.008544  \n",
       "image_top_1_resnet_tsvd_0              0.002085   -0.007154 -0.009587  \n",
       "image_top_1_resnet_tsvd_1             -0.006733    0.010129  0.012442  \n",
       "image_top_1_resnet_tsvd_2              0.008380   -0.008352 -0.005044  \n",
       "image_top_1_inception_tsvd_0           0.008730   -0.009669 -0.007150  \n",
       "image_top_1_inception_tsvd_1           0.002238   -0.005855 -0.008160  \n",
       "image_top_1_inception_tsvd_2           0.005543    0.000625 -0.002207  \n",
       "image_top_1_xception_tsvd_0            0.009409   -0.010146 -0.007236  \n",
       "image_top_1_xception_tsvd_1            0.002516   -0.005992 -0.008447  \n",
       "image_top_1_xception_tsvd_2            0.008414   -0.011725 -0.011588  \n",
       "corner                                 0.002269   -0.004174 -0.001124  \n",
       "keypoint                               0.003264   -0.008237 -0.009662  \n",
       "unemployment_rate                      1.000000   -0.262496 -0.310529  \n",
       "GDP_PC_PPP                            -0.262496    1.000000  0.806588  \n",
       "HDI                                   -0.310529    0.806588  1.000000  \n",
       "\n",
       "[45 rows x 45 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.loc[train.image_top_1 != -1,['deal_probability']+merge_cols].corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# label_price_features.csv\\nlabel_price = pd.read_csv('label_price_features.csv')\\n\\ntrain = train.merge(label_price[['item_id', 'price_pred']], on='item_id', how='left')\\ntest = test.merge(label_price[['item_id','price_pred']], on='item_id', how='left')\\n\\ndel label_price; gc.collect()\\n\\nmerge_cols.extend(['price_pred'])\\n\""
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# label_price_features.csv\n",
    "label_price = pd.read_csv('label_price_features.csv')\n",
    "\n",
    "train = train.merge(label_price[['item_id', 'price_pred']], on='item_id', how='left')\n",
    "test = test.merge(label_price[['item_id','price_pred']], on='item_id', how='left')\n",
    "\n",
    "del label_price; gc.collect()\n",
    "\n",
    "merge_cols.extend(['price_pred'])\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfile_prefix = ['image_top_1_resnet', 'image_top_1_inception', 'image_top_1_xception']\\nfor fp in file_prefix:\\n    new_col = fp+'_confidence'\\n    merge_cols.append(new_col)\\n    \\n    train_merge = pd.read_pickle(fp+'_train')\\n    train_merge['image'] = train_merge['image'].map(lambda x: x.replace('.jpg', ''))\\n    train = train.merge(train_merge[['image', new_col]], on='image', how='left')\\n    \\n    test_merge = pd.read_pickle(fp+'_test')\\n    test_merge['image'] =test_merge['image'].map(lambda x: x.replace('.jpg', ''))\\n    test = test.merge(test_merge[['image', new_col]], on='image', how='left')\\n        \\n    del train_merge, test_merge; gc.collect()\\n    print(fp, ' processed')\\n\""
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# img confidence score: hurt: high confidence might indicate the wrong catgory...\n",
    "\n",
    "'''\n",
    "file_prefix = ['image_top_1_resnet', 'image_top_1_inception', 'image_top_1_xception']\n",
    "for fp in file_prefix:\n",
    "    new_col = fp+'_confidence'\n",
    "    merge_cols.append(new_col)\n",
    "    \n",
    "    train_merge = pd.read_pickle(fp+'_train')\n",
    "    train_merge['image'] = train_merge['image'].map(lambda x: x.replace('.jpg', ''))\n",
    "    train = train.merge(train_merge[['image', new_col]], on='image', how='left')\n",
    "    \n",
    "    test_merge = pd.read_pickle(fp+'_test')\n",
    "    test_merge['image'] =test_merge['image'].map(lambda x: x.replace('.jpg', ''))\n",
    "    test = test.merge(test_merge[['image', new_col]], on='image', how='left')\n",
    "        \n",
    "    del train_merge, test_merge; gc.collect()\n",
    "    print(fp, ' processed')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nselect_cols = ['image_top_1_resnet_confidence', 'image_top_1_inception_confidence', 'image_top_1_xception_confidence']\\n\\nfor df in [train, test]:\\n    df['image_top_1_resnet_confidence_bin'] = df['image_top_1_resnet_confidence'] > 0.5\\n    df['image_top_1_inception_confidence_bin'] = df['image_top_1_inception_confidence'] > 0.5\\n    df['image_top_1_xception_confidence_bin'] = df['image_top_1_xception_confidence'] > 0.5\\n    df['image_confidence_mean'] = df[select_cols].mean(axis=1)\\n    df['image_confidence_med'] = df[select_cols].median(axis=1)\\n    df['image_confidence_max'] = df[select_cols].max(axis=1)\\n    df['image_confidence_min'] = df[select_cols].min(axis=1)\\n    df['image_confidence_std'] = df[select_cols].std(axis=1)\\n    df['image_confidence_vote_count'] = df[['image_top_1_resnet_confidence_bin',\\n                                            'image_top_1_inception_confidence_bin',\\n                                            'image_top_1_xception_confidence_bin']].sum(axis=1)\\n    \\nmerge_cols.extend(['image_top_1_resnet_confidence_bin',\\n                   'image_top_1_inception_confidence_bin',\\n                   'image_top_1_xception_confidence_bin',\\n                   'image_confidence_mean',\\n                   'image_confidence_med',\\n                   'image_confidence_max',\\n                   'image_confidence_min',\\n                   'image_confidence_std',\\n                   'image_confidence_vote_count'\\n                  ])\\n\""
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "select_cols = ['image_top_1_resnet_confidence', 'image_top_1_inception_confidence', 'image_top_1_xception_confidence']\n",
    "\n",
    "for df in [train, test]:\n",
    "    df['image_top_1_resnet_confidence_bin'] = df['image_top_1_resnet_confidence'] > 0.5\n",
    "    df['image_top_1_inception_confidence_bin'] = df['image_top_1_inception_confidence'] > 0.5\n",
    "    df['image_top_1_xception_confidence_bin'] = df['image_top_1_xception_confidence'] > 0.5\n",
    "    df['image_confidence_mean'] = df[select_cols].mean(axis=1)\n",
    "    df['image_confidence_med'] = df[select_cols].median(axis=1)\n",
    "    df['image_confidence_max'] = df[select_cols].max(axis=1)\n",
    "    df['image_confidence_min'] = df[select_cols].min(axis=1)\n",
    "    df['image_confidence_std'] = df[select_cols].std(axis=1)\n",
    "    df['image_confidence_vote_count'] = df[['image_top_1_resnet_confidence_bin',\n",
    "                                            'image_top_1_inception_confidence_bin',\n",
    "                                            'image_top_1_xception_confidence_bin']].sum(axis=1)\n",
    "    \n",
    "merge_cols.extend(['image_top_1_resnet_confidence_bin',\n",
    "                   'image_top_1_inception_confidence_bin',\n",
    "                   'image_top_1_xception_confidence_bin',\n",
    "                   'image_confidence_mean',\n",
    "                   'image_confidence_med',\n",
    "                   'image_confidence_max',\n",
    "                   'image_confidence_min',\n",
    "                   'image_confidence_std',\n",
    "                   'image_confidence_vote_count'\n",
    "                  ])\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "186"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# geo features: https://www.kaggle.com/frankherfert/region-and-city-details-with-lat-lon-and-clusters/notebook\n",
    "\n",
    "city_region_features = pd.read_csv('avito_region_city_features.csv')\n",
    "\n",
    "train.loc[:, 'city_region'] = train['city']# + ' ' + train['region']\n",
    "test.loc[:, 'city_region'] = test['city']# + ' ' + test['region']\n",
    "\n",
    "cols = ['latitude', 'longitude', 'lat_lon_hdbscan_cluster_05_03', 'lat_lon_hdbscan_cluster_10_03', 'lat_lon_hdbscan_cluster_20_03']\n",
    "train = train.merge(city_region_features[cols+['city_region']], on='city_region', how='left')\n",
    "test = test.merge(city_region_features[cols+['city_region']], on='city_region', how='left')\n",
    "\n",
    "train.drop('city_region', axis=1, inplace=True)\n",
    "test.drop('city_region', axis=1, inplace=True)\n",
    "del city_region_features; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "merge_cols.extend(cols)\n",
    "merge_cat.extend(['lat_lon_hdbscan_cluster_05_03', 'lat_lon_hdbscan_cluster_10_03', 'lat_lon_hdbscan_cluster_20_03'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cityxcatxusertype  processed\n",
      "imgxcityxcat  processed\n",
      "imgxisqnxusertype  processed\n"
     ]
    }
   ],
   "source": [
    "# fm features: self train for feature interaction\n",
    "\n",
    "file_prefix = ['cityxcatxusertype', 'imgxcityxcat', 'imgxisqnxusertype']\n",
    "for fp in file_prefix:\n",
    "    train_merge = pd.read_csv('train_'+fp+'_features.csv')\n",
    "    for col in train_merge.columns:\n",
    "        train.loc[:, fp+col] = train_merge[col]\n",
    "        merge_cols.append(fp+col)\n",
    "        \n",
    "    test_merge = pd.read_csv('test_'+fp+'_features.csv')\n",
    "    for col in test_merge.columns:\n",
    "        test.loc[:, fp+col] = test_merge[col]\n",
    "        \n",
    "    del train_merge, test_merge; gc.collect()\n",
    "    print(fp, ' processed')\n",
    "    \n",
    "# remove less significant features:\n",
    "for f in['imgxisqnxusertypeuser_type_fm_factor_1', 'imgxisqnxusertypeuser_type_fm_bias']:\n",
    "    train.drop(f, axis=1, inplace=True)\n",
    "    test.drop(f, axis=1, inplace=True)\n",
    "    merge_cols.remove(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "146"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## img meta\n",
    "\n",
    "def clean_img(img):\n",
    "    return img.split('.')[0]\n",
    "\n",
    "img_size = pd.read_pickle('image_size')\n",
    "img_size.columns = ['image', 'img_size']\n",
    "img_size['image'] = img_size['image'].apply(clean_img)\n",
    "\n",
    "img_info = pd.read_pickle('ImageInfo')\n",
    "img_info['image'] = img_info['image'].apply(clean_img)\n",
    "\n",
    "train = train.merge(img_size, how='left', on='image')\n",
    "train = train.merge(img_info, how='left', on='image')\n",
    "test = test.merge(img_size, how='left', on='image')\n",
    "test = test.merge(img_info, how='left', on='image')\n",
    "\n",
    "merge_cols.extend(img_size.columns[img_size.columns != 'image'].tolist())\n",
    "merge_cols.extend(img_info.columns[img_info.columns != 'image'].tolist())\n",
    "   \n",
    "del img_size, img_info; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1503424, 11)\n",
      "nasnet_nima_  processed\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nfor f in ['nasnet_nima_mean']:\\n    train.drop(f, axis=1, inplace=True)\\n    test.drop(f, axis=1, inplace=True)\\n    merge_cols.remove(f)\\n\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## nima scores\n",
    "\n",
    "file_prefix = ['nasnet_nima_']\n",
    "for fp in file_prefix:\n",
    "    train_merge = pd.read_csv('data/train_'+fp+'features_ver2.csv')\n",
    "    print(train_merge.shape)\n",
    "    for col in train_merge.columns:\n",
    "        if col == 'image':\n",
    "            continue\n",
    "        train.loc[:, fp+col] = train_merge[col]\n",
    "        merge_cols.append(fp+col)\n",
    "        \n",
    "    test_merge = pd.read_csv('data/test_'+fp+'features_ver2.csv')\n",
    "    for col in test_merge.columns:\n",
    "        if col == 'image':\n",
    "            continue\n",
    "        test.loc[:, fp+col] = test_merge[col]\n",
    "        \n",
    "    del train_merge, test_merge; gc.collect()\n",
    "    print(fp, ' processed')\n",
    "\n",
    "# remove nasnet_nima_mean\n",
    "'''\n",
    "for f in ['nasnet_nima_mean']:\n",
    "    train.drop(f, axis=1, inplace=True)\n",
    "    test.drop(f, axis=1, inplace=True)\n",
    "    merge_cols.remove(f)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1503424, 3)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "97"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# active features\n",
    "\n",
    "active_features = ['active_price_pred', 'active_duration_pred', 'active_ad_count_pred']\n",
    "\n",
    "train_merge = pd.read_csv('train_active_features_concated.csv')\n",
    "print(train_merge.shape)\n",
    "for col in active_features:\n",
    "    train.loc[:, col] = train_merge[col]\n",
    "    merge_cols.append(col)\n",
    "\n",
    "test_merge = pd.read_csv('test_active_features_concated.csv')\n",
    "for col in active_features:\n",
    "    test.loc[:, col] = test_merge[col]\n",
    "\n",
    "del train_merge, test_merge; gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\ntop_features = [\\n    'price_diff_1',\\n    'region_ME',\\n    'score_diff_4',\\n    'user_type_pcate_cate_cumcount',\\n    'title_num_chars',\\n    'region_day_count',\\n    'MORECOUNT_2',\\n    'seq_region_count',\\n    'user_id_city_region_cumcount',\\n    'price_diff_5',\\n    'price_diff_3',\\n    'seq_city_count']\\n\\ntrain_merge = pd.read_pickle('zijun_top50_train.pkl.gz')\\nprint(train_merge.shape)\\nfor col in top_features:\\n    train.loc[:, col] = train_merge[col].values\\n    merge_cols.append(col)\\n\\ntest_merge = pd.read_pickle('zijun_top50_test.pkl.gz')\\nfor col in top_features:\\n    test.loc[:, col] = test_merge[col].values\\n\\ndel train_merge, test_merge; gc.collect()\\n\""
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# zijun top features\n",
    "'''\n",
    "top_features = [\n",
    "    'price_diff_1',\n",
    "    'region_ME',\n",
    "    'score_diff_4',\n",
    "    'user_type_pcate_cate_cumcount',\n",
    "    'title_num_chars',\n",
    "    'region_day_count',\n",
    "    'MORECOUNT_2',\n",
    "    'seq_region_count',\n",
    "    'user_id_city_region_cumcount',\n",
    "    'price_diff_5',\n",
    "    'price_diff_3',\n",
    "    'seq_city_count']\n",
    "\n",
    "train_merge = pd.read_pickle('zijun_top50_train.pkl.gz')\n",
    "print(train_merge.shape)\n",
    "for col in top_features:\n",
    "    train.loc[:, col] = train_merge[col].values\n",
    "    merge_cols.append(col)\n",
    "\n",
    "test_merge = pd.read_pickle('zijun_top50_test.pkl.gz')\n",
    "for col in top_features:\n",
    "    test.loc[:, col] = test_merge[col].values\n",
    "\n",
    "del train_merge, test_merge; gc.collect()\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# split local test data after merge\n",
    "if SPLIT_TEST:\n",
    "    del test; gc.collect()\n",
    "    \n",
    "    import random\n",
    "    test_ix = random.sample(list(np.arange(train.shape[0])), 375000)\n",
    "    test = train.loc[test_ix,:].reset_index(drop=True)\n",
    "    y_test = y_train[test_ix]\n",
    "    \n",
    "    new_train_filter = ~train.index.isin(test_ix)\n",
    "    train = train.loc[new_train_filter,:].reset_index(drop=True)\n",
    "    y_train = y_train[new_train_filter]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vgg16_image_pca_feature_1                       112588\n",
      "vgg16_image_pca_feature_2                       112588\n",
      "vgg16_image_pca_feature_3                       112588\n",
      "vgg16_image_pca_feature_4                       112588\n",
      "vgg16_image_pca_feature_5                       112588\n",
      "vgg16_image_pca_feature_6                       112588\n",
      "vgg16_image_pca_feature_7                       112588\n",
      "vgg16_image_pca_feature_8                       112588\n",
      "vgg16_image_pca_feature_9                       112588\n",
      "vgg16_image_pca_feature_10                      112588\n",
      "vgg16_image_pca_feature_11                      112588\n",
      "vgg16_image_pca_feature_12                      112588\n",
      "vgg16_image_pca_feature_13                      112588\n",
      "vgg16_image_pca_feature_14                      112588\n",
      "vgg16_image_pca_feature_15                      112588\n",
      "vgg16_image_pca_feature_16                      112588\n",
      "vgg16_image_pca_feature_17                      112588\n",
      "vgg16_image_pca_feature_18                      112588\n",
      "vgg16_image_pca_feature_19                      112588\n",
      "vgg16_image_pca_feature_20                      112588\n",
      "vgg16_image_pca_feature_21                      112588\n",
      "vgg16_image_pca_feature_22                      112588\n",
      "vgg16_image_pca_feature_23                      112588\n",
      "vgg16_image_pca_feature_24                      112588\n",
      "vgg16_image_pca_feature_25                      112588\n",
      "vgg16_image_pca_feature_26                      112588\n",
      "vgg16_image_pca_feature_27                      112588\n",
      "vgg16_image_pca_feature_28                      112588\n",
      "vgg16_image_pca_feature_29                      112588\n",
      "vgg16_image_pca_feature_30                      112588\n",
      "image_top_1_resnet_tsvd_0                       112588\n",
      "image_top_1_resnet_tsvd_1                       112588\n",
      "image_top_1_resnet_tsvd_2                       112588\n",
      "image_top_1_inception_tsvd_0                    112588\n",
      "image_top_1_inception_tsvd_1                    112588\n",
      "image_top_1_inception_tsvd_2                    112588\n",
      "image_top_1_xception_tsvd_0                     112588\n",
      "image_top_1_xception_tsvd_1                     112588\n",
      "image_top_1_xception_tsvd_2                     112588\n",
      "corner                                               0\n",
      "keypoint                                             0\n",
      "unemployment_rate                                    0\n",
      "GDP_PC_PPP                                           0\n",
      "HDI                                                  0\n",
      "latitude                                             0\n",
      "longitude                                            0\n",
      "lat_lon_hdbscan_cluster_05_03                        0\n",
      "lat_lon_hdbscan_cluster_10_03                        0\n",
      "lat_lon_hdbscan_cluster_20_03                        0\n",
      "cityxcatxusertypeitem_num                            0\n",
      "cityxcatxusertypecity_fm_factor_0                    0\n",
      "cityxcatxusertypecity_fm_factor_1                    0\n",
      "cityxcatxusertypecategory_name_fm_factor_0           0\n",
      "cityxcatxusertypecategory_name_fm_factor_1           0\n",
      "cityxcatxusertypeuser_type_fm_factor_0               0\n",
      "cityxcatxusertypeuser_type_fm_factor_1               0\n",
      "cityxcatxusertypecity_fm_bias                        0\n",
      "cityxcatxusertypecategory_name_fm_bias               0\n",
      "cityxcatxusertypeuser_type_fm_bias                   0\n",
      "imgxcityxcatitem_num                                 0\n",
      "imgxcityxcatimage_top_1_fm_factor_0                  0\n",
      "imgxcityxcatimage_top_1_fm_factor_1                  0\n",
      "imgxcityxcatcity_fm_factor_0                         0\n",
      "imgxcityxcatcity_fm_factor_1                         0\n",
      "imgxcityxcatcategory_name_fm_factor_0                0\n",
      "imgxcityxcatcategory_name_fm_factor_1                0\n",
      "imgxcityxcatimage_top_1_fm_bias                      0\n",
      "imgxcityxcatcity_fm_bias                             0\n",
      "imgxcityxcatcategory_name_fm_bias                    0\n",
      "imgxisqnxusertypeitem_num                            0\n",
      "imgxisqnxusertypeimage_top_1_fm_factor_0             0\n",
      "imgxisqnxusertypeimage_top_1_fm_factor_1             0\n",
      "imgxisqnxusertypeitem_seq_number_fm_factor_0         0\n",
      "imgxisqnxusertypeitem_seq_number_fm_factor_1         0\n",
      "imgxisqnxusertypeuser_type_fm_factor_0               0\n",
      "imgxisqnxusertypeimage_top_1_fm_bias                 0\n",
      "imgxisqnxusertypeitem_seq_number_fm_bias             0\n",
      "img_size                                        112588\n",
      "b_intensity_mean                                112588\n",
      "b_intensity_median                              112588\n",
      "b_intensity_std                                 112588\n",
      "g_intensity_mean                                112588\n",
      "g_intensity_median                              112588\n",
      "g_intensity_std                                 112588\n",
      "gray_intensity_mean                             112588\n",
      "gray_intensity_median                           112588\n",
      "gray_intensity_std                              112588\n",
      "height                                          112588\n",
      "r_intensity_mean                                112588\n",
      "r_intensity_median                              112588\n",
      "r_intensity_std                                 112588\n",
      "width                                           112588\n",
      "nasnet_nima_med                                      0\n",
      "nasnet_nima_std                                      0\n",
      "nasnet_nima_max                                      0\n",
      "nasnet_nima_min                                      0\n",
      "nasnet_nima_1_quartile                               0\n",
      "nasnet_nima_3_quartile                               0\n",
      "nasnet_nima_13_quartile_diff                         0\n",
      "nasnet_nima_max_min_diff                             0\n",
      "nasnet_nima_non_max_mean                             0\n",
      "nasnet_nima_max_non_max_mean_diff                    0\n",
      "active_price_pred                                    0\n",
      "active_duration_pred                                 0\n",
      "active_ad_count_pred                                 0\n",
      "dtype: int64\n",
      "vgg16_image_pca_feature_1                       42609\n",
      "vgg16_image_pca_feature_2                       42609\n",
      "vgg16_image_pca_feature_3                       42609\n",
      "vgg16_image_pca_feature_4                       42609\n",
      "vgg16_image_pca_feature_5                       42609\n",
      "vgg16_image_pca_feature_6                       42609\n",
      "vgg16_image_pca_feature_7                       42609\n",
      "vgg16_image_pca_feature_8                       42609\n",
      "vgg16_image_pca_feature_9                       42609\n",
      "vgg16_image_pca_feature_10                      42609\n",
      "vgg16_image_pca_feature_11                      42609\n",
      "vgg16_image_pca_feature_12                      42609\n",
      "vgg16_image_pca_feature_13                      42609\n",
      "vgg16_image_pca_feature_14                      42609\n",
      "vgg16_image_pca_feature_15                      42609\n",
      "vgg16_image_pca_feature_16                      42609\n",
      "vgg16_image_pca_feature_17                      42609\n",
      "vgg16_image_pca_feature_18                      42609\n",
      "vgg16_image_pca_feature_19                      42609\n",
      "vgg16_image_pca_feature_20                      42609\n",
      "vgg16_image_pca_feature_21                      42609\n",
      "vgg16_image_pca_feature_22                      42609\n",
      "vgg16_image_pca_feature_23                      42609\n",
      "vgg16_image_pca_feature_24                      42609\n",
      "vgg16_image_pca_feature_25                      42609\n",
      "vgg16_image_pca_feature_26                      42609\n",
      "vgg16_image_pca_feature_27                      42609\n",
      "vgg16_image_pca_feature_28                      42609\n",
      "vgg16_image_pca_feature_29                      42609\n",
      "vgg16_image_pca_feature_30                      42609\n",
      "image_top_1_resnet_tsvd_0                       42609\n",
      "image_top_1_resnet_tsvd_1                       42609\n",
      "image_top_1_resnet_tsvd_2                       42609\n",
      "image_top_1_inception_tsvd_0                    42609\n",
      "image_top_1_inception_tsvd_1                    42609\n",
      "image_top_1_inception_tsvd_2                    42609\n",
      "image_top_1_xception_tsvd_0                     42609\n",
      "image_top_1_xception_tsvd_1                     42609\n",
      "image_top_1_xception_tsvd_2                     42609\n",
      "corner                                              0\n",
      "keypoint                                            0\n",
      "unemployment_rate                                   0\n",
      "GDP_PC_PPP                                          0\n",
      "HDI                                                 0\n",
      "latitude                                            0\n",
      "longitude                                           0\n",
      "lat_lon_hdbscan_cluster_05_03                       0\n",
      "lat_lon_hdbscan_cluster_10_03                       0\n",
      "lat_lon_hdbscan_cluster_20_03                       0\n",
      "cityxcatxusertypeitem_num                           0\n",
      "cityxcatxusertypecity_fm_factor_0                   0\n",
      "cityxcatxusertypecity_fm_factor_1                   0\n",
      "cityxcatxusertypecategory_name_fm_factor_0          0\n",
      "cityxcatxusertypecategory_name_fm_factor_1          0\n",
      "cityxcatxusertypeuser_type_fm_factor_0              0\n",
      "cityxcatxusertypeuser_type_fm_factor_1              0\n",
      "cityxcatxusertypecity_fm_bias                       0\n",
      "cityxcatxusertypecategory_name_fm_bias              0\n",
      "cityxcatxusertypeuser_type_fm_bias                  0\n",
      "imgxcityxcatitem_num                                0\n",
      "imgxcityxcatimage_top_1_fm_factor_0                 0\n",
      "imgxcityxcatimage_top_1_fm_factor_1                 0\n",
      "imgxcityxcatcity_fm_factor_0                        0\n",
      "imgxcityxcatcity_fm_factor_1                        0\n",
      "imgxcityxcatcategory_name_fm_factor_0               0\n",
      "imgxcityxcatcategory_name_fm_factor_1               0\n",
      "imgxcityxcatimage_top_1_fm_bias                     0\n",
      "imgxcityxcatcity_fm_bias                            0\n",
      "imgxcityxcatcategory_name_fm_bias                   0\n",
      "imgxisqnxusertypeitem_num                           0\n",
      "imgxisqnxusertypeimage_top_1_fm_factor_0            0\n",
      "imgxisqnxusertypeimage_top_1_fm_factor_1            0\n",
      "imgxisqnxusertypeitem_seq_number_fm_factor_0        0\n",
      "imgxisqnxusertypeitem_seq_number_fm_factor_1        0\n",
      "imgxisqnxusertypeuser_type_fm_factor_0              0\n",
      "imgxisqnxusertypeimage_top_1_fm_bias                0\n",
      "imgxisqnxusertypeitem_seq_number_fm_bias            0\n",
      "img_size                                        42609\n",
      "b_intensity_mean                                42609\n",
      "b_intensity_median                              42609\n",
      "b_intensity_std                                 42609\n",
      "g_intensity_mean                                42609\n",
      "g_intensity_median                              42609\n",
      "g_intensity_std                                 42609\n",
      "gray_intensity_mean                             42609\n",
      "gray_intensity_median                           42609\n",
      "gray_intensity_std                              42609\n",
      "height                                          42609\n",
      "r_intensity_mean                                42609\n",
      "r_intensity_median                              42609\n",
      "r_intensity_std                                 42609\n",
      "width                                           42609\n",
      "nasnet_nima_med                                     0\n",
      "nasnet_nima_std                                     0\n",
      "nasnet_nima_max                                     0\n",
      "nasnet_nima_min                                     0\n",
      "nasnet_nima_1_quartile                              0\n",
      "nasnet_nima_3_quartile                              0\n",
      "nasnet_nima_13_quartile_diff                        0\n",
      "nasnet_nima_max_min_diff                            0\n",
      "nasnet_nima_non_max_mean                            0\n",
      "nasnet_nima_max_non_max_mean_diff                   0\n",
      "active_price_pred                                   0\n",
      "active_duration_pred                                0\n",
      "active_ad_count_pred                                0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "if merge_cols != []:\n",
    "    for df in [train, test]:\n",
    "        print(df.loc[:, merge_cols].isnull().sum())\n",
    "        df.loc[:, merge_cols] = df[merge_cols].fillna(-1.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "target = 'deal_probability'\n",
    "predictors = [\n",
    "    'num_desc_punct', \n",
    "    'words_vs_unique_description', 'num_unique_words_description', 'num_unique_words_title', 'num_words_description', 'num_words_title',\n",
    "    'avg_times_up_user', 'avg_days_up_user', 'med_times_up_user', 'med_days_up_user', 'n_user_items', \n",
    "    'price', 'item_seq_number'\n",
    "] + merge_cols\n",
    "\n",
    "# , 'user_type',\n",
    "categorical = [\n",
    "    'image_top_1', 'param_1', 'param_2', 'param_3', \n",
    "    'city', 'region', 'category_name', 'parent_category_name', 'user_type',\n",
    "] \n",
    "\n",
    "predictors = predictors + categorical\n",
    "categorical = categorical + merge_cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['num_desc_punct',\n",
       "  'words_vs_unique_description',\n",
       "  'num_unique_words_description',\n",
       "  'num_unique_words_title',\n",
       "  'num_words_description',\n",
       "  'num_words_title',\n",
       "  'avg_times_up_user',\n",
       "  'avg_days_up_user',\n",
       "  'med_times_up_user',\n",
       "  'med_days_up_user',\n",
       "  'n_user_items',\n",
       "  'price',\n",
       "  'item_seq_number',\n",
       "  'vgg16_image_pca_feature_1',\n",
       "  'vgg16_image_pca_feature_2',\n",
       "  'vgg16_image_pca_feature_3',\n",
       "  'vgg16_image_pca_feature_4',\n",
       "  'vgg16_image_pca_feature_5',\n",
       "  'vgg16_image_pca_feature_6',\n",
       "  'vgg16_image_pca_feature_7',\n",
       "  'vgg16_image_pca_feature_8',\n",
       "  'vgg16_image_pca_feature_9',\n",
       "  'vgg16_image_pca_feature_10',\n",
       "  'vgg16_image_pca_feature_11',\n",
       "  'vgg16_image_pca_feature_12',\n",
       "  'vgg16_image_pca_feature_13',\n",
       "  'vgg16_image_pca_feature_14',\n",
       "  'vgg16_image_pca_feature_15',\n",
       "  'vgg16_image_pca_feature_16',\n",
       "  'vgg16_image_pca_feature_17',\n",
       "  'vgg16_image_pca_feature_18',\n",
       "  'vgg16_image_pca_feature_19',\n",
       "  'vgg16_image_pca_feature_20',\n",
       "  'vgg16_image_pca_feature_21',\n",
       "  'vgg16_image_pca_feature_22',\n",
       "  'vgg16_image_pca_feature_23',\n",
       "  'vgg16_image_pca_feature_24',\n",
       "  'vgg16_image_pca_feature_25',\n",
       "  'vgg16_image_pca_feature_26',\n",
       "  'vgg16_image_pca_feature_27',\n",
       "  'vgg16_image_pca_feature_28',\n",
       "  'vgg16_image_pca_feature_29',\n",
       "  'vgg16_image_pca_feature_30',\n",
       "  'image_top_1_resnet_tsvd_0',\n",
       "  'image_top_1_resnet_tsvd_1',\n",
       "  'image_top_1_resnet_tsvd_2',\n",
       "  'image_top_1_inception_tsvd_0',\n",
       "  'image_top_1_inception_tsvd_1',\n",
       "  'image_top_1_inception_tsvd_2',\n",
       "  'image_top_1_xception_tsvd_0',\n",
       "  'image_top_1_xception_tsvd_1',\n",
       "  'image_top_1_xception_tsvd_2',\n",
       "  'corner',\n",
       "  'keypoint',\n",
       "  'unemployment_rate',\n",
       "  'GDP_PC_PPP',\n",
       "  'HDI',\n",
       "  'latitude',\n",
       "  'longitude',\n",
       "  'lat_lon_hdbscan_cluster_05_03',\n",
       "  'lat_lon_hdbscan_cluster_10_03',\n",
       "  'lat_lon_hdbscan_cluster_20_03',\n",
       "  'cityxcatxusertypeitem_num',\n",
       "  'cityxcatxusertypecity_fm_factor_0',\n",
       "  'cityxcatxusertypecity_fm_factor_1',\n",
       "  'cityxcatxusertypecategory_name_fm_factor_0',\n",
       "  'cityxcatxusertypecategory_name_fm_factor_1',\n",
       "  'cityxcatxusertypeuser_type_fm_factor_0',\n",
       "  'cityxcatxusertypeuser_type_fm_factor_1',\n",
       "  'cityxcatxusertypecity_fm_bias',\n",
       "  'cityxcatxusertypecategory_name_fm_bias',\n",
       "  'cityxcatxusertypeuser_type_fm_bias',\n",
       "  'imgxcityxcatitem_num',\n",
       "  'imgxcityxcatimage_top_1_fm_factor_0',\n",
       "  'imgxcityxcatimage_top_1_fm_factor_1',\n",
       "  'imgxcityxcatcity_fm_factor_0',\n",
       "  'imgxcityxcatcity_fm_factor_1',\n",
       "  'imgxcityxcatcategory_name_fm_factor_0',\n",
       "  'imgxcityxcatcategory_name_fm_factor_1',\n",
       "  'imgxcityxcatimage_top_1_fm_bias',\n",
       "  'imgxcityxcatcity_fm_bias',\n",
       "  'imgxcityxcatcategory_name_fm_bias',\n",
       "  'imgxisqnxusertypeitem_num',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_factor_0',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_factor_1',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_factor_0',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_factor_1',\n",
       "  'imgxisqnxusertypeuser_type_fm_factor_0',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_bias',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_bias',\n",
       "  'img_size',\n",
       "  'b_intensity_mean',\n",
       "  'b_intensity_median',\n",
       "  'b_intensity_std',\n",
       "  'g_intensity_mean',\n",
       "  'g_intensity_median',\n",
       "  'g_intensity_std',\n",
       "  'gray_intensity_mean',\n",
       "  'gray_intensity_median',\n",
       "  'gray_intensity_std',\n",
       "  'height',\n",
       "  'r_intensity_mean',\n",
       "  'r_intensity_median',\n",
       "  'r_intensity_std',\n",
       "  'width',\n",
       "  'nasnet_nima_med',\n",
       "  'nasnet_nima_std',\n",
       "  'nasnet_nima_max',\n",
       "  'nasnet_nima_min',\n",
       "  'nasnet_nima_1_quartile',\n",
       "  'nasnet_nima_3_quartile',\n",
       "  'nasnet_nima_13_quartile_diff',\n",
       "  'nasnet_nima_max_min_diff',\n",
       "  'nasnet_nima_non_max_mean',\n",
       "  'nasnet_nima_max_non_max_mean_diff',\n",
       "  'active_price_pred',\n",
       "  'active_duration_pred',\n",
       "  'active_ad_count_pred',\n",
       "  'image_top_1',\n",
       "  'param_1',\n",
       "  'param_2',\n",
       "  'param_3',\n",
       "  'city',\n",
       "  'region',\n",
       "  'category_name',\n",
       "  'parent_category_name',\n",
       "  'user_type'],\n",
       " ['image_top_1',\n",
       "  'param_1',\n",
       "  'param_2',\n",
       "  'param_3',\n",
       "  'city',\n",
       "  'region',\n",
       "  'category_name',\n",
       "  'parent_category_name',\n",
       "  'user_type',\n",
       "  'lat_lon_hdbscan_cluster_05_03',\n",
       "  'lat_lon_hdbscan_cluster_10_03',\n",
       "  'lat_lon_hdbscan_cluster_20_03'])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictors, categorical"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### More feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfor df in [train, test]:\\n    df['param_join'] = df['param_1'].astype(str) + ' ' + df['param_2'].astype(str) + ' ' + df['param_3'].astype(str)\\n                                                                                                         \\ncategorical.append('param_join') \\npredictors.append('param_join')\\n\""
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# join params\n",
    "'''\n",
    "for df in [train, test]:\n",
    "    df['param_join'] = df['param_1'].astype(str) + ' ' + df['param_2'].astype(str) + ' ' + df['param_3'].astype(str)\n",
    "                                                                                                         \n",
    "categorical.append('param_join') \n",
    "predictors.append('param_join')\n",
    "'''\n",
    "#remove original param\n",
    "#\n",
    "#for cat in ['param_1', 'param_2', 'param_3']:\n",
    "#    categorical.remove(cat) \n",
    "#    predictors.remove(cat)\n",
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfrom copy import deepcopy as cp\\ncol = 'user_id'\\nuser_vc = train[col].append(test[col]).value_counts()\\nold_user_vc = cp(user_vc)\\nuser_vc = user_vc[user_vc>30]\\nusers = user_vc.index\\nuser_filter = users.isin(train[col]) & users.isin(test[col])\\n\\nprint('Original feature types # = {}, new feature types # = {}'.format(\\n        len(old_user_vc), pd.Series(user_filter).value_counts()[1]))\\n\\nprint(train.loc[train[col].isin(users[user_filter]), col].value_counts().sort_index())\\nprint(test.loc[test[col].isin(users[user_filter]), col].value_counts().sort_index())\\nprint(train.loc[train[col].isin(users[user_filter]), :].shape[0]/train.shape[0]*100.)\\nprint(test.loc[test[col].isin(users[user_filter]), :].shape[0]/test.shape[0]*100.)\\n\\ntrain.loc[~train[col].isin(users[user_filter]), col] = 'others_users'\\ntest.loc[~test[col].isin(users[user_filter]), col] = 'others_users'\\n\\ngp = train.groupby('user_id')\\n\\nfor col in ['item_seq_number']: # 'price', , 'num_desc_punct'\\n    mean_mapping, std_mapping = gp[col].mean(), gp[col].std()\\n    \\n    train.loc[:, col+'_user_mean'] = train['user_id'].map(mean_mapping).fillna(0.)\\n    test.loc[:, col+'_user_mean'] = test['user_id'].map(mean_mapping).fillna(0.)\\n    \\n    train.loc[:, col+'_user_std'] = train['user_id'].map(std_mapping).fillna(0.)\\n    test.loc[:, col+'_user_std'] = test['user_id'].map(std_mapping).fillna(0.)\\n    \\n    predictors.extend([col+'_user_mean', col+'_user_std'])\\n    print(col+' processed')\\n\""
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#user features\n",
    "'''\n",
    "from copy import deepcopy as cp\n",
    "col = 'user_id'\n",
    "user_vc = train[col].append(test[col]).value_counts()\n",
    "old_user_vc = cp(user_vc)\n",
    "user_vc = user_vc[user_vc>30]\n",
    "users = user_vc.index\n",
    "user_filter = users.isin(train[col]) & users.isin(test[col])\n",
    "\n",
    "print('Original feature types # = {}, new feature types # = {}'.format(\n",
    "        len(old_user_vc), pd.Series(user_filter).value_counts()[1]))\n",
    "\n",
    "print(train.loc[train[col].isin(users[user_filter]), col].value_counts().sort_index())\n",
    "print(test.loc[test[col].isin(users[user_filter]), col].value_counts().sort_index())\n",
    "print(train.loc[train[col].isin(users[user_filter]), :].shape[0]/train.shape[0]*100.)\n",
    "print(test.loc[test[col].isin(users[user_filter]), :].shape[0]/test.shape[0]*100.)\n",
    "\n",
    "train.loc[~train[col].isin(users[user_filter]), col] = 'others_users'\n",
    "test.loc[~test[col].isin(users[user_filter]), col] = 'others_users'\n",
    "\n",
    "gp = train.groupby('user_id')\n",
    "\n",
    "for col in ['item_seq_number']: # 'price', , 'num_desc_punct'\n",
    "    mean_mapping, std_mapping = gp[col].mean(), gp[col].std()\n",
    "    \n",
    "    train.loc[:, col+'_user_mean'] = train['user_id'].map(mean_mapping).fillna(0.)\n",
    "    test.loc[:, col+'_user_mean'] = test['user_id'].map(mean_mapping).fillna(0.)\n",
    "    \n",
    "    train.loc[:, col+'_user_std'] = train['user_id'].map(std_mapping).fillna(0.)\n",
    "    test.loc[:, col+'_user_std'] = test['user_id'].map(std_mapping).fillna(0.)\n",
    "    \n",
    "    predictors.extend([col+'_user_mean', col+'_user_std'])\n",
    "    print(col+' processed')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "partial = pd.concat([train[categorical+['price', 'item_seq_number']],\n",
    "                    test[categorical+['price', 'item_seq_number']]]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nfor cat in ['image_top_1', 'city', 'param_join', 'category_name']:\\n    gp = partial.groupby(cat)\\n  \\n    for df in [train, test]:\\n        #df.loc[:, cat+'_price_avg'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].mean())).fillna(-1.)\\n        #df.loc[:, cat+'_price_med'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].median())).fillna(-1.)\\n        #df.loc[:, cat+'_price_sum'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].sum())).fillna(-1.)\\n        #df.loc[:, cat+'_price_std'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].std())).fillna(-1.)\\n        \\n        df.loc[:, cat+'_price_na_count'] = df[cat].map(gp['price'].apply(lambda x: x[x<0].count())).fillna(-1.)\\n        df.loc[:, cat+'_item_seq_num_avg'] = df[cat].map(gp['item_seq_number'].mean()).fillna(-1.)\\n        df.loc[:, cat+'_item_seq_num_med'] = df[cat].map(gp['item_seq_number'].median()).fillna(-1.)\\n        df.loc[:, cat+'_item_seq_num_sum'] = df[cat].map(gp['item_seq_number'].sum()).fillna(-1.)\\n        df.loc[:, cat+'_item_seq_num_unique_count'] =                                             df[cat].map(gp['item_seq_number'].apply(lambda x: pd.Series(x).nunique())).fillna(-1.)\\n        df.loc[:, cat+'_item_seq_num_std'] = df[cat].map(gp['item_seq_number'].std()).fillna(-1.)\\n\\n    predictors = predictors +         [cat+'_price_na_count',\\n         cat+'_item_seq_num_avg',\\n         cat+'_item_seq_num_sum',\\n         cat+'_item_seq_num_unique_count',\\n         cat+'_item_seq_num_std']\\n        \\n    del gp; gc.collect()\\n    print(cat, ' processed')\\n\""
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# price and item_seq_number enc\n",
    "'''\n",
    "for cat in ['image_top_1', 'city', 'param_join', 'category_name']:\n",
    "    gp = partial.groupby(cat)\n",
    "  \n",
    "    for df in [train, test]:\n",
    "        #df.loc[:, cat+'_price_avg'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].mean())).fillna(-1.)\n",
    "        #df.loc[:, cat+'_price_med'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].median())).fillna(-1.)\n",
    "        #df.loc[:, cat+'_price_sum'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].sum())).fillna(-1.)\n",
    "        #df.loc[:, cat+'_price_std'] = df[cat].map(gp['price'].apply(lambda x: x[x>=0].std())).fillna(-1.)\n",
    "        \n",
    "        df.loc[:, cat+'_price_na_count'] = df[cat].map(gp['price'].apply(lambda x: x[x<0].count())).fillna(-1.)\n",
    "        df.loc[:, cat+'_item_seq_num_avg'] = df[cat].map(gp['item_seq_number'].mean()).fillna(-1.)\n",
    "        df.loc[:, cat+'_item_seq_num_med'] = df[cat].map(gp['item_seq_number'].median()).fillna(-1.)\n",
    "        df.loc[:, cat+'_item_seq_num_sum'] = df[cat].map(gp['item_seq_number'].sum()).fillna(-1.)\n",
    "        df.loc[:, cat+'_item_seq_num_unique_count'] = \\\n",
    "                                            df[cat].map(gp['item_seq_number'].apply(lambda x: pd.Series(x).nunique())).fillna(-1.)\n",
    "        df.loc[:, cat+'_item_seq_num_std'] = df[cat].map(gp['item_seq_number'].std()).fillna(-1.)\n",
    "\n",
    "    predictors = predictors + \\\n",
    "        [cat+'_price_na_count',\n",
    "         cat+'_item_seq_num_avg',\n",
    "         cat+'_item_seq_num_sum',\n",
    "         cat+'_item_seq_num_unique_count',\n",
    "         cat+'_item_seq_num_std']\n",
    "        \n",
    "    del gp; gc.collect()\n",
    "    print(cat, ' processed')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n# binned price\\n\\nsorted_prices = sorted(partial.price.values)\\n_, bins = pd.qcut(sorted_prices, 80, labels=None, retbins=True, precision=3, duplicates='drop')\\npartial['price_binned'] =     pd.cut(partial.price, bins=bins, labels=np.arange(len(bins)-1), retbins=False)\\npartial['price_binned'] = partial['price_binned'].astype(int)\\nprint(partial['price_binned'].value_counts())\\n\\ntrain.loc[:, 'price_binned'] = partial['price_binned'].values[:train.shape[0]]\\ntest.loc[:, 'price_binned'] = partial['price_binned'].values[train.shape[0]:]\\n\\ncategorical.append('price_binned')\\npredictors.append('price_binned')\\n\""
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "# binned price\n",
    "\n",
    "sorted_prices = sorted(partial.price.values)\n",
    "_, bins = pd.qcut(sorted_prices, 80, labels=None, retbins=True, precision=3, duplicates='drop')\n",
    "partial['price_binned'] = \\\n",
    "    pd.cut(partial.price, bins=bins, labels=np.arange(len(bins)-1), retbins=False)\n",
    "partial['price_binned'] = partial['price_binned'].astype(int)\n",
    "print(partial['price_binned'].value_counts())\n",
    "\n",
    "train.loc[:, 'price_binned'] = partial['price_binned'].values[:train.shape[0]]\n",
    "test.loc[:, 'price_binned'] = partial['price_binned'].values[train.shape[0]:]\n",
    "\n",
    "categorical.append('price_binned')\n",
    "predictors.append('price_binned')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nsorted_isqn = sorted(partial.item_seq_number.values)\\n\\n_, bins = pd.qcut(sorted_isqn, 1000, labels=None, retbins=True, precision=3, duplicates='drop')\\npartial['isqn_binned'] =     pd.cut(partial.item_seq_number.values, bins=bins, labels=np.arange(len(bins)-1), retbins=False)\\npartial['isqn_binned'] = partial['isqn_binned'].astype(int)\\nprint(partial['isqn_binned'].value_counts())\\n\\ntrain.loc[:, 'isqn_binned'] = partial['isqn_binned'].values[:train.shape[0]]\\ntest.loc[:, 'isqn_binned'] = partial['isqn_binned'].values[train.shape[0]:]\\n\\ncategorical.append('isqn_binned')\\npredictors.append('isqn_binned')\\n\""
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# binned isqn\n",
    "\n",
    "'''\n",
    "sorted_isqn = sorted(partial.item_seq_number.values)\n",
    "\n",
    "_, bins = pd.qcut(sorted_isqn, 1000, labels=None, retbins=True, precision=3, duplicates='drop')\n",
    "partial['isqn_binned'] = \\\n",
    "    pd.cut(partial.item_seq_number.values, bins=bins, labels=np.arange(len(bins)-1), retbins=False)\n",
    "partial['isqn_binned'] = partial['isqn_binned'].astype(int)\n",
    "print(partial['isqn_binned'].value_counts())\n",
    "\n",
    "train.loc[:, 'isqn_binned'] = partial['isqn_binned'].values[:train.shape[0]]\n",
    "test.loc[:, 'isqn_binned'] = partial['isqn_binned'].values[train.shape[0]:]\n",
    "\n",
    "categorical.append('isqn_binned')\n",
    "predictors.append('isqn_binned')\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def add_noise(series, noise_level): \n",
    "    return series * (1 + noise_level * np.random.randn(len(series)))\n",
    "\n",
    "def target_encode(trn_series=None, \n",
    "                  tst_series=None, \n",
    "                  target=None, \n",
    "                  min_samples_leaf=1, \n",
    "                  smoothing=1,\n",
    "                  noise_level=0):\n",
    "                      \n",
    "    assert len(trn_series) == len(target)\n",
    "    assert trn_series.name == tst_series.name\n",
    "    temp = pd.concat([trn_series, target], axis=1)\n",
    "    # Compute target mean \n",
    "    averages = temp.groupby(by=trn_series.name)[target.name].agg([\"mean\", \"count\"])\n",
    "    # Compute smoothing\n",
    "    smoothing = 1 / (1 + np.exp(-(averages[\"count\"] - min_samples_leaf) / smoothing))\n",
    "    # Apply average function to all target data\n",
    "    prior = target.mean()\n",
    "    # The bigger the count the less full_avg is taken into account\n",
    "    averages[target.name] = prior * (1 - smoothing) + averages[\"mean\"] * smoothing\n",
    "    averages.drop([\"mean\", \"count\"], axis=1, inplace=True)\n",
    "    # Apply averages to trn and tst series\n",
    "    ft_trn_series = pd.merge(\n",
    "        trn_series.to_frame(trn_series.name),\n",
    "        averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n",
    "        on=trn_series.name,\n",
    "        how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n",
    "    # pd.merge does not keep the index so restore it\n",
    "    ft_trn_series.index = trn_series.index \n",
    "    ft_tst_series = pd.merge(\n",
    "        tst_series.to_frame(tst_series.name),\n",
    "        averages.reset_index().rename(columns={'index': target.name, target.name: 'average'}),\n",
    "        on=tst_series.name,\n",
    "        how='left')['average'].rename(trn_series.name + '_mean').fillna(prior)\n",
    "    # pd.merge does not keep the index so restore it\n",
    "    ft_tst_series.index = tst_series.index\n",
    "    \n",
    "    return add_noise(ft_trn_series, noise_level), add_noise(ft_tst_series, noise_level)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\n#label encode\\nfor feature in categorical:\\n    print(f'Transforming {feature}...')\\n    encoder = LabelEncoder()\\n    encoder.fit(train[feature].append(test[feature]).astype(str))\\n    \\n    train[feature] = encoder.transform(train[feature].astype(str))\\n    test[feature] = encoder.transform(test[feature].astype(str))\\n\\n#cnt encode\\nfor feature in categorical:\\n    if feature == 'user_type':\\n        continue\\n        \\n    gp = train.groupby(feature)[feature].count()\\n    train.loc[:, feature+'_cnt'] = train[feature].map(gp)\\n    test.loc[:, feature+'_cnt'] = test[feature].map(gp)\\n    predictors.append(feature+'_cnt')\\n    \\n    del gp; gc.collect()\\n\\n\\n#target encode\\nfrom sklearn.model_selection import KFold\\nfrom scipy.stats import kurtosis\\nfold_num = 5\\nkf = KFold(fold_num, shuffle=True, random_state=SEED)\\n\\nfor col in categorical+['user_id']:    \\n    \\n    #train.loc[:, col+'_dp_mean_enc'] = np.zeros((train.shape[0],))\\n    #train.loc[:, col+'_dp_std_enc'] = np.zeros((train.shape[0],))\\n    #test.loc[:, col+'_dp_mean_enc'] = np.zeros((test.shape[0],))\\n    #test.loc[:, col+'_dp_std_enc'] = np.zeros((test.shape[0],))\\n    ##test.loc[:, col+'_dp_kurtosis_enc'] = np.zeros((test.shape[0],))\\n    #\\n    #for train_ix, val_ix in kf.split(train):\\n    #    tr_X = train.loc[train_ix, :]\\n    #\\n    #    gp = tr_X.groupby(col)['deal_probability']\\n    #    mapping, mapping_std = gp.mean(), gp.std()\\n    #    mapping_kurtosis = gp.apply(lambda x: kurtosis(x, nan_policy='omit'))\\n    #    \\n    #    train.loc[val_ix, col+'_dp_mean_enc'] = train.loc[val_ix, col].map(mapping).fillna(-1.)\\n    #    train.loc[val_ix, col+'_dp_std_enc'] = train.loc[val_ix, col].map(mapping_std).fillna(-1.)\\n    #    #train.loc[val_ix, col+'_dp_kurtosis_enc'] = train.loc[val_ix, col].map(mapping_kurtosis).fillna(-1.)\\n    #    \\n    #    test.loc[:, col+'_dp_mean_enc'] += test.loc[:, col].map(mapping).fillna(-1.)\\n    #    test.loc[:, col+'_dp_std_enc'] += test.loc[:, col].map(mapping_std).fillna(-1.)\\n    #    #test.loc[:, col+'_dp_kurtosis_enc'] += test.loc[:, col].map(mapping_kurtosis).fillna(-1.)\\n    # \\n    #    del gp, mapping, mapping_std, mapping_kurtosis; gc.collect()\\n        \\n    #test.loc[:, col+'_dp_mean_enc'] /= fold_num\\n    #test.loc[:, col+'_dp_std_enc'] /= fold_num\\n    #print(col + ' processed.')\\n    #predictors.extend([col+'_dp_mean_enc', col+'_dp_std_enc'])\\n\\n    trn, sub = target_encode(train[col], \\n                             test[col], \\n                             target=pd.Series(data=y_train, name='deal_probability'),  \\n                             min_samples_leaf=100,\\n                             smoothing=10,\\n                             noise_level=0.01)\\n                            \\n    train.loc[:, col+'_dp_mean_enc'] = trn.astype('float32')\\n    test.loc[:, col+'_dp_mean_enc'] = sub.astype('float32')\\n    predictors.extend([col+'_dp_mean_enc'])  \\n\""
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# cat encoding\n",
    "\n",
    "'''\n",
    "#label encode\n",
    "for feature in categorical:\n",
    "    print(f'Transforming {feature}...')\n",
    "    encoder = LabelEncoder()\n",
    "    encoder.fit(train[feature].append(test[feature]).astype(str))\n",
    "    \n",
    "    train[feature] = encoder.transform(train[feature].astype(str))\n",
    "    test[feature] = encoder.transform(test[feature].astype(str))\n",
    "\n",
    "#cnt encode\n",
    "for feature in categorical:\n",
    "    if feature == 'user_type':\n",
    "        continue\n",
    "        \n",
    "    gp = train.groupby(feature)[feature].count()\n",
    "    train.loc[:, feature+'_cnt'] = train[feature].map(gp)\n",
    "    test.loc[:, feature+'_cnt'] = test[feature].map(gp)\n",
    "    predictors.append(feature+'_cnt')\n",
    "    \n",
    "    del gp; gc.collect()\n",
    "\n",
    "\n",
    "#target encode\n",
    "from sklearn.model_selection import KFold\n",
    "from scipy.stats import kurtosis\n",
    "fold_num = 5\n",
    "kf = KFold(fold_num, shuffle=True, random_state=SEED)\n",
    "\n",
    "for col in categorical+['user_id']:    \n",
    "    \n",
    "    #train.loc[:, col+'_dp_mean_enc'] = np.zeros((train.shape[0],))\n",
    "    #train.loc[:, col+'_dp_std_enc'] = np.zeros((train.shape[0],))\n",
    "    #test.loc[:, col+'_dp_mean_enc'] = np.zeros((test.shape[0],))\n",
    "    #test.loc[:, col+'_dp_std_enc'] = np.zeros((test.shape[0],))\n",
    "    ##test.loc[:, col+'_dp_kurtosis_enc'] = np.zeros((test.shape[0],))\n",
    "    #\n",
    "    #for train_ix, val_ix in kf.split(train):\n",
    "    #    tr_X = train.loc[train_ix, :]\n",
    "    #\n",
    "    #    gp = tr_X.groupby(col)['deal_probability']\n",
    "    #    mapping, mapping_std = gp.mean(), gp.std()\n",
    "    #    mapping_kurtosis = gp.apply(lambda x: kurtosis(x, nan_policy='omit'))\n",
    "    #    \n",
    "    #    train.loc[val_ix, col+'_dp_mean_enc'] = train.loc[val_ix, col].map(mapping).fillna(-1.)\n",
    "    #    train.loc[val_ix, col+'_dp_std_enc'] = train.loc[val_ix, col].map(mapping_std).fillna(-1.)\n",
    "    #    #train.loc[val_ix, col+'_dp_kurtosis_enc'] = train.loc[val_ix, col].map(mapping_kurtosis).fillna(-1.)\n",
    "    #    \n",
    "    #    test.loc[:, col+'_dp_mean_enc'] += test.loc[:, col].map(mapping).fillna(-1.)\n",
    "    #    test.loc[:, col+'_dp_std_enc'] += test.loc[:, col].map(mapping_std).fillna(-1.)\n",
    "    #    #test.loc[:, col+'_dp_kurtosis_enc'] += test.loc[:, col].map(mapping_kurtosis).fillna(-1.)\n",
    "    # \n",
    "    #    del gp, mapping, mapping_std, mapping_kurtosis; gc.collect()\n",
    "        \n",
    "    #test.loc[:, col+'_dp_mean_enc'] /= fold_num\n",
    "    #test.loc[:, col+'_dp_std_enc'] /= fold_num\n",
    "    #print(col + ' processed.')\n",
    "    #predictors.extend([col+'_dp_mean_enc', col+'_dp_std_enc'])\n",
    "\n",
    "    trn, sub = target_encode(train[col], \n",
    "                             test[col], \n",
    "                             target=pd.Series(data=y_train, name='deal_probability'),  \n",
    "                             min_samples_leaf=100,\n",
    "                             smoothing=10,\n",
    "                             noise_level=0.01)\n",
    "                            \n",
    "    train.loc[:, col+'_dp_mean_enc'] = trn.astype('float32')\n",
    "    test.loc[:, col+'_dp_mean_enc'] = sub.astype('float32')\n",
    "    predictors.extend([col+'_dp_mean_enc'])  \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\npredictors = [f for f in predictors if f not in mean_enc_cols]\\ncategorical = []\\n'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# remove all categorical variables\n",
    "'''\n",
    "predictors = [f for f in predictors if f not in mean_enc_cols]\n",
    "categorical = []\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def reduce_mem_usage(df):\n",
    "    \"\"\" iterate through all the columns of a dataframe and modify the data type\n",
    "        to reduce memory usage.        \n",
    "    \"\"\"\n",
    "    start_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n",
    "    \n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtype\n",
    "        \n",
    "        if col_type != object:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "        else: df[col] = df[col].astype('category')\n",
    "\n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
    "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory usage of dataframe is 1284.66 MB\n",
      "Memory usage after optimization is: 429.30 MB\n",
      "Decreased by 66.6%\n",
      "Memory usage of dataframe is 430.58 MB\n",
      "Memory usage after optimization is: 150.76 MB\n",
      "Decreased by 65.0%\n"
     ]
    }
   ],
   "source": [
    "train.drop(['activation_date', 'item_id', 'user_id', 'title', 'description'], axis=1, inplace=True)\n",
    "test.drop(['activation_date', 'item_id', 'user_id', 'title', 'description'], axis=1, inplace=True)\n",
    "\n",
    "train = reduce_mem_usage(train)\n",
    "test = reduce_mem_usage(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['num_desc_punct',\n",
       "  'words_vs_unique_description',\n",
       "  'num_unique_words_description',\n",
       "  'num_unique_words_title',\n",
       "  'num_words_description',\n",
       "  'num_words_title',\n",
       "  'avg_times_up_user',\n",
       "  'avg_days_up_user',\n",
       "  'med_times_up_user',\n",
       "  'med_days_up_user',\n",
       "  'n_user_items',\n",
       "  'price',\n",
       "  'item_seq_number',\n",
       "  'vgg16_image_pca_feature_1',\n",
       "  'vgg16_image_pca_feature_2',\n",
       "  'vgg16_image_pca_feature_3',\n",
       "  'vgg16_image_pca_feature_4',\n",
       "  'vgg16_image_pca_feature_5',\n",
       "  'vgg16_image_pca_feature_6',\n",
       "  'vgg16_image_pca_feature_7',\n",
       "  'vgg16_image_pca_feature_8',\n",
       "  'vgg16_image_pca_feature_9',\n",
       "  'vgg16_image_pca_feature_10',\n",
       "  'vgg16_image_pca_feature_11',\n",
       "  'vgg16_image_pca_feature_12',\n",
       "  'vgg16_image_pca_feature_13',\n",
       "  'vgg16_image_pca_feature_14',\n",
       "  'vgg16_image_pca_feature_15',\n",
       "  'vgg16_image_pca_feature_16',\n",
       "  'vgg16_image_pca_feature_17',\n",
       "  'vgg16_image_pca_feature_18',\n",
       "  'vgg16_image_pca_feature_19',\n",
       "  'vgg16_image_pca_feature_20',\n",
       "  'vgg16_image_pca_feature_21',\n",
       "  'vgg16_image_pca_feature_22',\n",
       "  'vgg16_image_pca_feature_23',\n",
       "  'vgg16_image_pca_feature_24',\n",
       "  'vgg16_image_pca_feature_25',\n",
       "  'vgg16_image_pca_feature_26',\n",
       "  'vgg16_image_pca_feature_27',\n",
       "  'vgg16_image_pca_feature_28',\n",
       "  'vgg16_image_pca_feature_29',\n",
       "  'vgg16_image_pca_feature_30',\n",
       "  'image_top_1_resnet_tsvd_0',\n",
       "  'image_top_1_resnet_tsvd_1',\n",
       "  'image_top_1_resnet_tsvd_2',\n",
       "  'image_top_1_inception_tsvd_0',\n",
       "  'image_top_1_inception_tsvd_1',\n",
       "  'image_top_1_inception_tsvd_2',\n",
       "  'image_top_1_xception_tsvd_0',\n",
       "  'image_top_1_xception_tsvd_1',\n",
       "  'image_top_1_xception_tsvd_2',\n",
       "  'corner',\n",
       "  'keypoint',\n",
       "  'unemployment_rate',\n",
       "  'GDP_PC_PPP',\n",
       "  'HDI',\n",
       "  'latitude',\n",
       "  'longitude',\n",
       "  'lat_lon_hdbscan_cluster_05_03',\n",
       "  'lat_lon_hdbscan_cluster_10_03',\n",
       "  'lat_lon_hdbscan_cluster_20_03',\n",
       "  'cityxcatxusertypeitem_num',\n",
       "  'cityxcatxusertypecity_fm_factor_0',\n",
       "  'cityxcatxusertypecity_fm_factor_1',\n",
       "  'cityxcatxusertypecategory_name_fm_factor_0',\n",
       "  'cityxcatxusertypecategory_name_fm_factor_1',\n",
       "  'cityxcatxusertypeuser_type_fm_factor_0',\n",
       "  'cityxcatxusertypeuser_type_fm_factor_1',\n",
       "  'cityxcatxusertypecity_fm_bias',\n",
       "  'cityxcatxusertypecategory_name_fm_bias',\n",
       "  'cityxcatxusertypeuser_type_fm_bias',\n",
       "  'imgxcityxcatitem_num',\n",
       "  'imgxcityxcatimage_top_1_fm_factor_0',\n",
       "  'imgxcityxcatimage_top_1_fm_factor_1',\n",
       "  'imgxcityxcatcity_fm_factor_0',\n",
       "  'imgxcityxcatcity_fm_factor_1',\n",
       "  'imgxcityxcatcategory_name_fm_factor_0',\n",
       "  'imgxcityxcatcategory_name_fm_factor_1',\n",
       "  'imgxcityxcatimage_top_1_fm_bias',\n",
       "  'imgxcityxcatcity_fm_bias',\n",
       "  'imgxcityxcatcategory_name_fm_bias',\n",
       "  'imgxisqnxusertypeitem_num',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_factor_0',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_factor_1',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_factor_0',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_factor_1',\n",
       "  'imgxisqnxusertypeuser_type_fm_factor_0',\n",
       "  'imgxisqnxusertypeimage_top_1_fm_bias',\n",
       "  'imgxisqnxusertypeitem_seq_number_fm_bias',\n",
       "  'img_size',\n",
       "  'b_intensity_mean',\n",
       "  'b_intensity_median',\n",
       "  'b_intensity_std',\n",
       "  'g_intensity_mean',\n",
       "  'g_intensity_median',\n",
       "  'g_intensity_std',\n",
       "  'gray_intensity_mean',\n",
       "  'gray_intensity_median',\n",
       "  'gray_intensity_std',\n",
       "  'height',\n",
       "  'r_intensity_mean',\n",
       "  'r_intensity_median',\n",
       "  'r_intensity_std',\n",
       "  'width',\n",
       "  'nasnet_nima_med',\n",
       "  'nasnet_nima_std',\n",
       "  'nasnet_nima_max',\n",
       "  'nasnet_nima_min',\n",
       "  'nasnet_nima_1_quartile',\n",
       "  'nasnet_nima_3_quartile',\n",
       "  'nasnet_nima_13_quartile_diff',\n",
       "  'nasnet_nima_max_min_diff',\n",
       "  'nasnet_nima_non_max_mean',\n",
       "  'nasnet_nima_max_non_max_mean_diff',\n",
       "  'active_price_pred',\n",
       "  'active_duration_pred',\n",
       "  'active_ad_count_pred',\n",
       "  'image_top_1',\n",
       "  'param_1',\n",
       "  'param_2',\n",
       "  'param_3',\n",
       "  'city',\n",
       "  'region',\n",
       "  'category_name',\n",
       "  'parent_category_name',\n",
       "  'user_type'],\n",
       " ['image_top_1',\n",
       "  'param_1',\n",
       "  'param_2',\n",
       "  'param_3',\n",
       "  'city',\n",
       "  'region',\n",
       "  'category_name',\n",
       "  'parent_category_name',\n",
       "  'user_type',\n",
       "  'lat_lon_hdbscan_cluster_05_03',\n",
       "  'lat_lon_hdbscan_cluster_10_03',\n",
       "  'lat_lon_hdbscan_cluster_20_03'],\n",
       " 127)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictors, categorical, len(predictors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import KFold\n",
    "from tqdm import tqdm\n",
    "SEED=411 \n",
    "kf = KFold(5, shuffle=True, random_state=SEED)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nremove_cols = [\\n    'cityxcatxusertypecategory_name_fm_factor_0',\\n    'cityxcatxusertypecategory_name_fm_factor_1',\\n    'cityxcatxusertypecategory_name_fm_bias',\\n    'cityxcatxusertypeuser_type_fm_bias',\\n    'imgxcityxcatcategory_name_fm_factor_0',\\n    'imgxcityxcatcategory_name_fm_factor_1',\\n    'imgxisqnxusertypeuser_type_fm_factor_0', \\n    'user_type',\\n    'image_top_1_item_seq_num_unique_count',\\n    'category_name_price_na_count',\\n    'category_name_item_seq_num_sum', 'category_name_item_seq_num_unique_count',\\n    'image_top_1_cnt', 'param_2_cnt', 'param_3_cnt', 'category_name_cnt',\\n    'parent_category_name_cnt', 'param_join_cnt']\\n\\nfor col in remove_cols:\\n    if col in predictors:\\n        predictors.remove(col)\\n    if col in categorical:\\n        categorical.remove(col)\\n    if col in train.columns.tolist():\\n        train.drop(col, axis=1, inplace=True)\\n        test.drop(col, axis=1, inplace=True)\\n\""
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "remove_cols = [\n",
    "    'cityxcatxusertypecategory_name_fm_factor_0',\n",
    "    'cityxcatxusertypecategory_name_fm_factor_1',\n",
    "    'cityxcatxusertypecategory_name_fm_bias',\n",
    "    'cityxcatxusertypeuser_type_fm_bias',\n",
    "    'imgxcityxcatcategory_name_fm_factor_0',\n",
    "    'imgxcityxcatcategory_name_fm_factor_1',\n",
    "    'imgxisqnxusertypeuser_type_fm_factor_0', \n",
    "    'user_type',\n",
    "    'image_top_1_item_seq_num_unique_count',\n",
    "    'category_name_price_na_count',\n",
    "    'category_name_item_seq_num_sum', 'category_name_item_seq_num_unique_count',\n",
    "    'image_top_1_cnt', 'param_2_cnt', 'param_3_cnt', 'category_name_cnt',\n",
    "    'parent_category_name_cnt', 'param_join_cnt']\n",
    "\n",
    "for col in remove_cols:\n",
    "    if col in predictors:\n",
    "        predictors.remove(col)\n",
    "    if col in categorical:\n",
    "        categorical.remove(col)\n",
    "    if col in train.columns.tolist():\n",
    "        train.drop(col, axis=1, inplace=True)\n",
    "        test.drop(col, axis=1, inplace=True)\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Modeling: Local test or dump features for future other learnings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "if SPLIT_TEST is True:\n",
    "    import pickle\n",
    "    train_text = pickle.load(open('train_text_dense_new', 'rb'))\n",
    "    \n",
    "    new_test = np.hstack([train_text[test_ix,:], test[predictors].values])\n",
    "    new_train = np.hstack([train_text[new_train_filter,:], train[predictors].values])\n",
    "    \n",
    "    text_feature_names = ['text_'+str(i) for i in range(train_text.shape[1])]\n",
    "    predictors = np.hstack([\n",
    "        text_feature_names,\n",
    "        predictors,\n",
    "    ])\n",
    "    print('predictors updated!')\n",
    "    \n",
    "    del train_text, train, test; gc.collect()\n",
    "    train, test = new_train, new_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def feature_select(rounds):\n",
    "    global predictors, categorical\n",
    "    \n",
    "    for i in range(rounds):\n",
    "        rounds = 2000\n",
    "        early_stop_rounds = 100\n",
    "        trial_seed = 0\n",
    "\n",
    "        if SPLIT_TEST is True:\n",
    "            test_pred = np.zeros((test.shape[0],))\n",
    "            \n",
    "        feature_importances = None\n",
    "        for train_ix, val_ix in kf.split(train):\n",
    "            trial_seed += 1\n",
    "            params = {\n",
    "                'objective' : 'regression',\n",
    "                'metric' : 'rmse',\n",
    "                'num_leaves' : 31,\n",
    "                'max_depth': 5,\n",
    "                'learning_rate' : 0.1,\n",
    "                'bagging_fraction': 1.,\n",
    "                'bagging_freq': 1,    \n",
    "                'feature_fraction' : .6,\n",
    "                'feature_fraction_seed': trial_seed,\n",
    "                'reg_alpha': 0.0, \n",
    "                'reg_lambda': 0.0,\n",
    "                'verbosity' : -1,\n",
    "                'n_jobs': 4\n",
    "            }\n",
    "\n",
    "            dtrain = lgb.Dataset(train[train_ix, :], label=y_train[train_ix],\n",
    "                                 feature_name=list(predictors), \n",
    "                                 categorical_feature=categorical)\n",
    "            dvalid = lgb.Dataset(train[val_ix, :], label=y_train[val_ix],\n",
    "                                 feature_name=list(predictors), \n",
    "                                 categorical_feature=categorical)\n",
    "\n",
    "            model = lgb.train(params, dtrain, \n",
    "                              valid_sets=[dtrain, dvalid], \n",
    "                              valid_names=['train', 'valid'],\n",
    "                              num_boost_round=rounds, \n",
    "                              early_stopping_rounds=early_stop_rounds, \n",
    "                              verbose_eval=100)\n",
    "            \n",
    "            test_pred += model.predict(test)\n",
    "            \n",
    "            current_feat_imp = model.feature_importance(importance_type='gain')\n",
    "            pd.Series(current_feat_imp).hist()\n",
    "            plt.show()\n",
    "\n",
    "            if feature_importances is None:\n",
    "                feature_importances = current_feat_imp\n",
    "            else:\n",
    "                feature_importances += current_feat_imp\n",
    "\n",
    "            del dtrain, dvalid; gc.collect()\n",
    "        \n",
    "        if SPLIT_TEST is True:\n",
    "            test_pred /= 5\n",
    "            test_pred = np.clip(test_pred, 0., 1.)\n",
    "            print('test loss=', ((y_test-test_pred)**2.).mean()**.5)\n",
    "\n",
    "        th = 10 # unit: percent\n",
    "        th = np.percentile(feature_importances, th)\n",
    "        print('threshold=', th)\n",
    "        print('Original dense features # =', len(feature_importances))\n",
    "        print('Removed feature # =', (feature_importances<=th).astype(int).sum())\n",
    "        print('Removed feature importances =', feature_importances[feature_importances<=th])\n",
    "        print('Removed features:\\n', np.array(predictors)[feature_importances<=th])\n",
    "    \n",
    "        df = pd.DataFrame()\n",
    "        df['feature_name'] = predictors\n",
    "        df['feature_importances'] = feature_importances\n",
    "        print(df.sort_values('feature_importances', ascending=False).reset_index(drop=True))\n",
    "        \n",
    "        predictors = list(predictors)\n",
    "        for f in np.array(predictors)[feature_importances<=th]:\n",
    "            if f in categorical:\n",
    "                categorical.remove(f)\n",
    "            if f in predictors:\n",
    "                predictors.remove(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "105\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['vgg16_image_pca_feature_1',\n",
       " 'vgg16_image_pca_feature_2',\n",
       " 'vgg16_image_pca_feature_3',\n",
       " 'vgg16_image_pca_feature_4',\n",
       " 'vgg16_image_pca_feature_5',\n",
       " 'vgg16_image_pca_feature_6',\n",
       " 'vgg16_image_pca_feature_7',\n",
       " 'vgg16_image_pca_feature_8',\n",
       " 'vgg16_image_pca_feature_9',\n",
       " 'vgg16_image_pca_feature_10',\n",
       " 'vgg16_image_pca_feature_11',\n",
       " 'vgg16_image_pca_feature_12',\n",
       " 'vgg16_image_pca_feature_13',\n",
       " 'vgg16_image_pca_feature_14',\n",
       " 'vgg16_image_pca_feature_15',\n",
       " 'vgg16_image_pca_feature_16',\n",
       " 'vgg16_image_pca_feature_17',\n",
       " 'vgg16_image_pca_feature_18',\n",
       " 'vgg16_image_pca_feature_19',\n",
       " 'vgg16_image_pca_feature_20',\n",
       " 'vgg16_image_pca_feature_21',\n",
       " 'vgg16_image_pca_feature_22',\n",
       " 'vgg16_image_pca_feature_23',\n",
       " 'vgg16_image_pca_feature_24',\n",
       " 'vgg16_image_pca_feature_25',\n",
       " 'vgg16_image_pca_feature_26',\n",
       " 'vgg16_image_pca_feature_27',\n",
       " 'vgg16_image_pca_feature_28',\n",
       " 'vgg16_image_pca_feature_29',\n",
       " 'vgg16_image_pca_feature_30',\n",
       " 'image_top_1_resnet_tsvd_0',\n",
       " 'image_top_1_resnet_tsvd_1',\n",
       " 'image_top_1_resnet_tsvd_2',\n",
       " 'image_top_1_inception_tsvd_0',\n",
       " 'image_top_1_inception_tsvd_1',\n",
       " 'image_top_1_inception_tsvd_2',\n",
       " 'image_top_1_xception_tsvd_0',\n",
       " 'image_top_1_xception_tsvd_1',\n",
       " 'image_top_1_xception_tsvd_2',\n",
       " 'corner',\n",
       " 'keypoint',\n",
       " 'unemployment_rate',\n",
       " 'GDP_PC_PPP',\n",
       " 'HDI',\n",
       " 'latitude',\n",
       " 'longitude',\n",
       " 'lat_lon_hdbscan_cluster_05_03',\n",
       " 'lat_lon_hdbscan_cluster_10_03',\n",
       " 'lat_lon_hdbscan_cluster_20_03',\n",
       " 'cityxcatxusertypeitem_num',\n",
       " 'cityxcatxusertypecity_fm_factor_0',\n",
       " 'cityxcatxusertypecity_fm_factor_1',\n",
       " 'cityxcatxusertypecategory_name_fm_factor_0',\n",
       " 'cityxcatxusertypecategory_name_fm_factor_1',\n",
       " 'cityxcatxusertypeuser_type_fm_factor_0',\n",
       " 'cityxcatxusertypeuser_type_fm_factor_1',\n",
       " 'cityxcatxusertypecity_fm_bias',\n",
       " 'cityxcatxusertypecategory_name_fm_bias',\n",
       " 'cityxcatxusertypeuser_type_fm_bias',\n",
       " 'imgxcityxcatitem_num',\n",
       " 'imgxcityxcatimage_top_1_fm_factor_0',\n",
       " 'imgxcityxcatimage_top_1_fm_factor_1',\n",
       " 'imgxcityxcatcity_fm_factor_0',\n",
       " 'imgxcityxcatcity_fm_factor_1',\n",
       " 'imgxcityxcatcategory_name_fm_factor_0',\n",
       " 'imgxcityxcatcategory_name_fm_factor_1',\n",
       " 'imgxcityxcatimage_top_1_fm_bias',\n",
       " 'imgxcityxcatcity_fm_bias',\n",
       " 'imgxcityxcatcategory_name_fm_bias',\n",
       " 'imgxisqnxusertypeitem_num',\n",
       " 'imgxisqnxusertypeimage_top_1_fm_factor_0',\n",
       " 'imgxisqnxusertypeimage_top_1_fm_factor_1',\n",
       " 'imgxisqnxusertypeitem_seq_number_fm_factor_0',\n",
       " 'imgxisqnxusertypeitem_seq_number_fm_factor_1',\n",
       " 'imgxisqnxusertypeuser_type_fm_factor_0',\n",
       " 'imgxisqnxusertypeimage_top_1_fm_bias',\n",
       " 'imgxisqnxusertypeitem_seq_number_fm_bias',\n",
       " 'img_size',\n",
       " 'b_intensity_mean',\n",
       " 'b_intensity_median',\n",
       " 'b_intensity_std',\n",
       " 'g_intensity_mean',\n",
       " 'g_intensity_median',\n",
       " 'g_intensity_std',\n",
       " 'gray_intensity_mean',\n",
       " 'gray_intensity_median',\n",
       " 'gray_intensity_std',\n",
       " 'height',\n",
       " 'r_intensity_mean',\n",
       " 'r_intensity_median',\n",
       " 'r_intensity_std',\n",
       " 'width',\n",
       " 'nasnet_nima_med',\n",
       " 'nasnet_nima_std',\n",
       " 'nasnet_nima_max',\n",
       " 'nasnet_nima_min',\n",
       " 'nasnet_nima_1_quartile',\n",
       " 'nasnet_nima_3_quartile',\n",
       " 'nasnet_nima_13_quartile_diff',\n",
       " 'nasnet_nima_max_min_diff',\n",
       " 'nasnet_nima_non_max_mean',\n",
       " 'nasnet_nima_max_non_max_mean_diff',\n",
       " 'active_price_pred',\n",
       " 'active_duration_pred',\n",
       " 'active_ad_count_pred']"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(len(merge_cols))\n",
    "merge_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# dump features only if not in test mode\n",
    "if SPLIT_TEST is False:\n",
    "    import pickle\n",
    "    pickle.dump(train[merge_cols], open('train_selected_features', 'wb'))\n",
    "    pickle.dump(test[merge_cols], open('test_selected_features', 'wb'))\n",
    "    #pickle.dump(categorical, open('selected_cat_feature_names', 'wb'))\n",
    "else:\n",
    "    feature_select(1)\n",
    "    print(predictors)\n",
    "    print(categorical)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature Selection Local CV Records (Simple lgb trees)\n",
    "* baseline = 0.221558\n",
    "* +last all (smoothing mean enc) = cv: 0.22127, local test: 0.2209x\n",
    "* +last all (20 fold oof mean enc) = cv: 0.219657, local test: 0.2200x\n",
    "* +last all (20 fold oof mean enc) + geo = cv: 0.219868, local test: 0.21955\n",
    "* +impute image top 1 and agg features + label price feature = 0.21888\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "#### Old experiments\n",
    "* +user feature => .\n",
    "* +joined param123 => .\n",
    "* +price&item encode =>.\n",
    "* price(na_count only)&item encode => .\n",
    "* user_type cnt (city+param_join only)&price(na_count only)&item(exclude parant_cat, param_3, user_type) encode => .\n",
    "* remove user_type count, remove user mean\\std, + binned_price => bad a lot, cut!\n",
    "* remove user_type count, remove user mean\\std, + binned_isqn => bad a lot, cut!\n",
    "* remove user_type count, remove user mean\\std, + fm => .\n",
    "* remove user_type count, use price_na+isqn encode with only (city, category_name, img_top_1, param_join), remove useless fm features => .\n",
    "* +img meta =>.\n",
    "* +nima => .\n",
    "* +nima (nima mean removed) => .\n",
    "* +active features => .\n",
    "* +zijun top features => .\n",
    "* +kurtosis encode => ."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}