vote_feature_selection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Goal : find the better set of features\n",
    "\n",
    "--\n",
    "\n",
    "Thanks to all the feature importances given by all the ML algorithms we test.\n",
    "\n",
    "\n",
    "#### TODO :\n",
    "- create a importance score based on the importance from the csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tuxml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### I- First we have to import all the datasets with features importances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first tree + some data processing => keep all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1118"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_tree = np.array(pd.read_csv(\"feature_net.csv\", skiprows = 1, names = ['features', 'imp'])['features'])\n",
    "len(fi_tree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The actual tree => keep all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1051"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_tree2 = np.array(pd.read_csv(\"feature_importance.csv\", skiprows = 1, names = ['features', 'imp'])['features'])\n",
    "len(fi_tree2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe another Decision Tree? => keep all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "369"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_tree3 = np.array(pd.read_csv(\"feature_importanceDT.csv\", skiprows = 1, names = ['features', 'imp'])['features'])\n",
    "len(fi_tree3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random forest => keep 99.9% of the predictive power"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4229"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we select the features based on random forest importance\n",
    "df_selection = pd.read_csv('feature_importanceRF.csv', names = ['features', 'imp'], skiprows = 0)\n",
    "selection = df_selection['features']\n",
    "imp = df_selection['imp']\n",
    "\n",
    "# we select the more important features until we get more than alpha % of the 'predictive power'\n",
    "alpha = 0.999\n",
    "sorted_imp = sorted(imp, reverse = True)\n",
    "sum_imp = 0\n",
    "i = 0\n",
    "\n",
    "while sum_imp < alpha:\n",
    "    sum_imp+=sorted_imp[i]\n",
    "    i+=1\n",
    "threshold_imp = sorted_imp[i]\n",
    "\n",
    "fi_rf = selection[imp>threshold_imp]\n",
    "\n",
    "len(fi_rf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Probably ElasticNet => keep the 1000 first features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_elan = np.array(pd.read_csv(\"feature_importanceEN.csv\", skiprows = 1, names = ['features', 'imp'])['features'])[0:1000]\n",
    "len(fi_elan)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Probably Gradient Boosting => keep them all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "160"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_gb = np.array(pd.read_csv(\"feature_importanceGB.csv\", skiprows = 1, names = ['features', 'imp'])['features'])\n",
    "len(fi_gb)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "I guess Linear Regression => keep the 1000 first features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_linreg = np.array(pd.read_csv(\"feature_importanceLR.csv\", skiprows = 1, names = ['features', 'imp'])['features'])[0:1000]\n",
    "len(fi_linreg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lasso => keep the 1000 first features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_lasso= np.array(pd.read_csv(\"feature_importanceLasso.csv\", skiprows = 1, names = ['features', 'imp'])['features'])[0:1000]\n",
    "len(fi_lasso)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Ridge => keep the 1000 first features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_ridge = np.array(pd.read_csv(\"feature_importanceRidge.csv\", skiprows = 1, names = ['features', 'imp'])['features'])[0:1000]\n",
    "len(fi_ridge)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Correlation => we keep the 1000 with vmlinux highest correlated features (absolute value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fi_correlation = pd.read_csv(\"correlations_vmlinux.csv\", skiprows= 1, names = ['features', 'imp'])\n",
    "fi_correlation['imp'] =  np.abs(fi_correlation['imp'])\n",
    "fi_corr = np.array(fi_correlation.sort_values(by = 'imp', ascending = False)['features'][0:1000])\n",
    "len(fi_corr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Welch test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "fi_welch = pd.read_csv(\"welch_test_output.csv\", skiprows = 1, names = ['features','imp'])['features']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## II- Let's the ML vote!\n",
    "\n",
    "For each feature, we count how many time it's been considered as important for one ml algo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "features = tuxml.load_dataset()\n",
    "\n",
    "list_features = features.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\llesoil\\Documents\\Anaconda3\\envs\\tensor_gpu\\lib\\site-packages\\ipykernel_launcher.py:4: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  after removing the cwd from sys.path.\n"
     ]
    }
   ],
   "source": [
    "count = []\n",
    "for f in list_features:\n",
    "    sum_ml = 0\n",
    "    if f in fi_tree:\n",
    "        sum_ml+=1\n",
    "    if f in fi_tree2:\n",
    "        sum_ml+=1\n",
    "    if f in fi_tree3:\n",
    "        sum_ml+=1\n",
    "    if f in fi_rf:\n",
    "        sum_ml+=1\n",
    "    if f in fi_elan:\n",
    "        sum_ml+=1\n",
    "    if f in fi_gb:\n",
    "        sum_ml+=1\n",
    "    if f in fi_linreg:\n",
    "        sum_ml+=1\n",
    "    if f in fi_lasso:\n",
    "        sum_ml+=1\n",
    "    if f in fi_ridge:\n",
    "        sum_ml+=1\n",
    "    if f in fi_corr:\n",
    "        sum_ml+=1\n",
    "    if f in fi_welch:\n",
    "        sum_ml+=1\n",
    "    count.append(sum_ml)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>features</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>X86_LOCAL_APIC</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OPENVSWITCH</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>TEXTSEARCH_FSM</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>LOCKDEP_SUPPORT</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GENERIC_CLOCKEVENTS_MIN_ADJUST</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         features count\n",
       "0                  X86_LOCAL_APIC     1\n",
       "1                     OPENVSWITCH     1\n",
       "2                  TEXTSEARCH_FSM     1\n",
       "3                 LOCKDEP_SUPPORT     1\n",
       "4  GENERIC_CLOCKEVENTS_MIN_ADJUST     1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vote = pd.DataFrame(np.transpose([list_features, count]), columns = ['features', 'count'])\n",
    "vote.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>features</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7111</th>\n",
       "      <td>DEBUG_INFO_SPLIT</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12438</th>\n",
       "      <td>GCOV_PROFILE_ALL</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7828</th>\n",
       "      <td>KASAN_OUTLINE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5822</th>\n",
       "      <td>RANDOMIZE_BASE</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>618</th>\n",
       "      <td>KASAN</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2691</th>\n",
       "      <td>UBSAN_SANITIZE_ALL</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7084</th>\n",
       "      <td>DEBUG_INFO_REDUCED</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6131</th>\n",
       "      <td>X86_VSMP</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5829</th>\n",
       "      <td>X86_NEED_RELOCS</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5622</th>\n",
       "      <td>DEBUG_INFO</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4825</th>\n",
       "      <td>DRM_I915</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2698</th>\n",
       "      <td>UBSAN_ALIGNMENT</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1468</th>\n",
       "      <td>COMPILE_TEST</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12217</th>\n",
       "      <td>SCSI_DMA</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8554</th>\n",
       "      <td>KCOV_INSTRUMENT_ALL</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>729</th>\n",
       "      <td>RFS_ACCEL</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9774</th>\n",
       "      <td>INFINIBAND_ISER</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1550</th>\n",
       "      <td>BLOCK</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5902</th>\n",
       "      <td>XFS_DEBUG</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3996</th>\n",
       "      <td>BE2NET</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  features count\n",
       "7111      DEBUG_INFO_SPLIT     7\n",
       "12438     GCOV_PROFILE_ALL     7\n",
       "7828         KASAN_OUTLINE     7\n",
       "5822        RANDOMIZE_BASE     7\n",
       "618                  KASAN     7\n",
       "2691    UBSAN_SANITIZE_ALL     7\n",
       "7084    DEBUG_INFO_REDUCED     7\n",
       "6131              X86_VSMP     6\n",
       "5829       X86_NEED_RELOCS     6\n",
       "5622            DEBUG_INFO     6\n",
       "4825              DRM_I915     5\n",
       "2698       UBSAN_ALIGNMENT     5\n",
       "1468          COMPILE_TEST     5\n",
       "12217             SCSI_DMA     5\n",
       "8554   KCOV_INSTRUMENT_ALL     5\n",
       "729              RFS_ACCEL     5\n",
       "9774       INFINIBAND_ISER     5\n",
       "1550                 BLOCK     5\n",
       "5902             XFS_DEBUG     5\n",
       "3996                BE2NET     4"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res_vote = vote.sort_values(by='count', ascending = False)\n",
    "res_vote[0:20]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Maybe the 20 most important features?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Small set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "273\n"
     ]
    }
   ],
   "source": [
    "small_feature_set = np.array(res_vote['features'])[np.where(res_vote['count']>2)]\n",
    "print(len(small_feature_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.savetxt(\"small_feature_set.csv\", big_feature_set, delimiter=\",\", fmt='%s')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Bigger set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1376\n"
     ]
    }
   ],
   "source": [
    "big_feature_set = np.array(res_vote['features'])[np.where(res_vote['count']>1)]\n",
    "print(len(big_feature_set))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.savetxt(\"big_feature_set.csv\", big_feature_set, delimiter=\",\", fmt='%s')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}