Text Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Import libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "%matplotlib inline\n",
    "import sqlite3    ## SQL Interface\n",
    "import pickle     ## Used to save your data - Converts objects to byte stream and vice versa\n",
    "\n",
    "## Modules to perform Text Preprocessing\n",
    "import re\n",
    "import nltk\n",
    "from nltk.stem.snowball import SnowballStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SEM 1 SGPA</th>\n",
       "      <th>SEM 1 KT</th>\n",
       "      <th>SEM 2 SGPA</th>\n",
       "      <th>SEM 2 KT</th>\n",
       "      <th>SEM 3 SGPA</th>\n",
       "      <th>SEM 3 KT</th>\n",
       "      <th>SEM 4 SGPA</th>\n",
       "      <th>SEM 4 KT</th>\n",
       "      <th>SEM 5 SGPA</th>\n",
       "      <th>SEM 5 KT</th>\n",
       "      <th>...</th>\n",
       "      <th>Mode_Of_Transportation</th>\n",
       "      <th>2_hrs_lect</th>\n",
       "      <th>Submissions</th>\n",
       "      <th>5_hrs_lect</th>\n",
       "      <th>5_hrs_pracs</th>\n",
       "      <th>Coaching_classes</th>\n",
       "      <th>Teacher's Feedback</th>\n",
       "      <th>Label</th>\n",
       "      <th>Scocial_Skills</th>\n",
       "      <th>Average pointer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>7.93</td>\n",
       "      <td>0</td>\n",
       "      <td>7.93</td>\n",
       "      <td>0</td>\n",
       "      <td>7.38</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.40</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Good student</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.58000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7.75</td>\n",
       "      <td>0</td>\n",
       "      <td>7.78</td>\n",
       "      <td>0</td>\n",
       "      <td>7.68</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.93</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.33</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Good leadership skills</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>8.24625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6.15</td>\n",
       "      <td>0</td>\n",
       "      <td>6.09</td>\n",
       "      <td>0</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Hard working</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.39250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5.75</td>\n",
       "      <td>0</td>\n",
       "      <td>6.54</td>\n",
       "      <td>0</td>\n",
       "      <td>5.90</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.98</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.00</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Responsible</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.18375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5.67</td>\n",
       "      <td>0</td>\n",
       "      <td>6.81</td>\n",
       "      <td>0</td>\n",
       "      <td>6.60</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.50</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.70</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Good leadership skills</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>6.27250</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 31 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SEM 1 SGPA  SEM 1 KT  SEM 2 SGPA  SEM 2 KT  SEM 3 SGPA  SEM 3 KT  \\\n",
       "0        7.93         0        7.93         0        7.38       0.0   \n",
       "1        7.75         0        7.78         0        7.68       0.0   \n",
       "2        6.15         0        6.09         0        6.00       0.0   \n",
       "3        5.75         0        6.54         0        5.90       0.0   \n",
       "4        5.67         0        6.81         0        6.60       2.0   \n",
       "\n",
       "   SEM 4 SGPA  SEM 4 KT  SEM 5 SGPA  SEM 5 KT       ...         \\\n",
       "0        7.30       0.0        7.40         0       ...          \n",
       "1        8.93       0.0        8.33         0       ...          \n",
       "2        7.10       0.0        6.10         0       ...          \n",
       "3        5.98       0.0        6.00         0       ...          \n",
       "4        6.50       0.0        6.70         0       ...          \n",
       "\n",
       "   Mode_Of_Transportation  2_hrs_lect  Submissions  5_hrs_lect  5_hrs_pracs  \\\n",
       "0                       1           0            1           1            0   \n",
       "1                       1           0            1           1            1   \n",
       "2                       3           1            1           1            1   \n",
       "3                       1           0            0           0            1   \n",
       "4                       1           0            1           1            1   \n",
       "\n",
       "   Coaching_classes      Teacher's Feedback  Label  Scocial_Skills  \\\n",
       "0                 0            Good student      1               1   \n",
       "1                 0  Good leadership skills      1               0   \n",
       "2                 0            Hard working      1               0   \n",
       "3                 1             Responsible      1               0   \n",
       "4                 0  Good leadership skills      1               1   \n",
       "\n",
       "   Average pointer  \n",
       "0          7.58000  \n",
       "1          8.24625  \n",
       "2          6.39250  \n",
       "3          6.18375  \n",
       "4          6.27250  \n",
       "\n",
       "[5 rows x 31 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('student_info2','rb') as pickle_file:\n",
    "    data = pickle.load(pickle_file)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['SEM 1 SGPA', 'SEM 1 KT', 'SEM 2 SGPA', 'SEM 2 KT', 'SEM 3 SGPA',\n",
       "       'SEM 3 KT', 'SEM 4 SGPA', 'SEM 4 KT', 'SEM 5 SGPA', 'SEM 5 KT',\n",
       "       'SEM 6 SGPA', 'SEM 6 KT', 'SEM 7 SGPA', 'SEM 7 KT', 'SEM 8 SGPA',\n",
       "       'Hours_On_Assignment', 'Hours_On_Studies', 'Travel_Time', 'Attendance',\n",
       "       'Internet_Availability', 'Internet_Speed', 'Mode_Of_Transportation',\n",
       "       '2_hrs_lect', 'Submissions', '5_hrs_lect', '5_hrs_pracs',\n",
       "       'Coaching_classes', 'Teacher's Feedback', 'Label', 'Scocial_Skills',\n",
       "       'Average pointer'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final = data.sample(frac=1,random_state = 123).reset_index(drop=True)  #Shuffling the DataFrame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    162\n",
       "0    100\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Pratheesh\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:1362: UserWarning: The spaces in these column names will not be changed. In pandas versions < 0.14, spaces were converted to underscores.\n",
      "  chunksize=chunksize, dtype=dtype)\n"
     ]
    }
   ],
   "source": [
    "conn = sqlite3.connect('final.sqlite')    #Saving the changes made above into a new sqlite file\n",
    "c=conn.cursor()\n",
    "final.to_sql('Reviews', conn, if_exists='replace')\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>SEM 1 SGPA</th>\n",
       "      <th>SEM 1 KT</th>\n",
       "      <th>SEM 2 SGPA</th>\n",
       "      <th>SEM 2 KT</th>\n",
       "      <th>SEM 3 SGPA</th>\n",
       "      <th>SEM 3 KT</th>\n",
       "      <th>SEM 4 SGPA</th>\n",
       "      <th>SEM 4 KT</th>\n",
       "      <th>SEM 5 SGPA</th>\n",
       "      <th>...</th>\n",
       "      <th>Mode_Of_Transportation</th>\n",
       "      <th>2_hrs_lect</th>\n",
       "      <th>Submissions</th>\n",
       "      <th>5_hrs_lect</th>\n",
       "      <th>5_hrs_pracs</th>\n",
       "      <th>Coaching_classes</th>\n",
       "      <th>Teacher's Feedback</th>\n",
       "      <th>Label</th>\n",
       "      <th>Scocial_Skills</th>\n",
       "      <th>Average pointer</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7.10</td>\n",
       "      <td>0</td>\n",
       "      <td>6.85</td>\n",
       "      <td>0</td>\n",
       "      <td>7.20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.60</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Disciplined and hard working</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.13125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>7.20</td>\n",
       "      <td>0</td>\n",
       "      <td>7.10</td>\n",
       "      <td>0</td>\n",
       "      <td>6.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.14</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Good leadership skills</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.05375</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3.80</td>\n",
       "      <td>4</td>\n",
       "      <td>4.20</td>\n",
       "      <td>2</td>\n",
       "      <td>4.20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.80</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.35</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Very talkative and Poor attendance</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.05875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>6.91</td>\n",
       "      <td>1</td>\n",
       "      <td>6.60</td>\n",
       "      <td>2</td>\n",
       "      <td>5.82</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.71</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Respectful to Authority and Others</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.50500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3.50</td>\n",
       "      <td>1</td>\n",
       "      <td>3.40</td>\n",
       "      <td>2</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.10</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.63</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Bunk lectuer</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.81500</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 32 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  SEM 1 SGPA  SEM 1 KT  SEM 2 SGPA  SEM 2 KT  SEM 3 SGPA  SEM 3 KT  \\\n",
       "0      0        7.10         0        6.85         0        7.20       1.0   \n",
       "1      1        7.20         0        7.10         0        6.11       0.0   \n",
       "2      2        3.80         4        4.20         2        4.20       1.0   \n",
       "3      3        6.91         1        6.60         2        5.82       1.0   \n",
       "4      4        3.50         1        3.40         2        4.00       2.0   \n",
       "\n",
       "   SEM 4 SGPA  SEM 4 KT  SEM 5 SGPA       ...         Mode_Of_Transportation  \\\n",
       "0        7.30       0.0        7.60       ...                              1   \n",
       "1        6.67       0.0        7.14       ...                              1   \n",
       "2        3.80       2.0        4.35       ...                              1   \n",
       "3        6.10       0.0        7.71       ...                              1   \n",
       "4        4.10       2.0        3.63       ...                              1   \n",
       "\n",
       "   2_hrs_lect  Submissions  5_hrs_lect  5_hrs_pracs  Coaching_classes  \\\n",
       "0           0            0           0            0                 0   \n",
       "1           1            1           0            0                 1   \n",
       "2           0            1           0            0                 0   \n",
       "3           0            1           0            1                 0   \n",
       "4           0            0           0            0                 0   \n",
       "\n",
       "                   Teacher's Feedback  Label  Scocial_Skills  Average pointer  \n",
       "0        Disciplined and hard working      1               0          7.13125  \n",
       "1              Good leadership skills      1               1          7.05375  \n",
       "2  Very talkative and Poor attendance      0               0          4.05875  \n",
       "3  Respectful to Authority and Others      1               0          6.50500  \n",
       "4                        Bunk lectuer      0               0          3.81500  \n",
       "\n",
       "[5 rows x 32 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conn = sqlite3.connect('final.sqlite')  #Loading the sqlite file for future use\n",
    "final = pd.read_sql_query(\"\"\"SELECT * FROM Reviews\"\"\", conn)\n",
    "conn.close()\n",
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanpunc(sentence):\n",
    "    '''This function cleans all the punctuation or special characters from a given sentence'''\n",
    "    cleaned = re.sub(r'[?|@|!|^|%|\\'|\"|#]',r'',sentence)\n",
    "    cleaned = re.sub(r'[.|,|)|(|\\|/]',r' ',cleaned)\n",
    "    return  cleaned"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocessing(series):\n",
    "    '''The function takes a Pandas Series object containing text in all the cells\n",
    "       And performs following Preprocessing steps on each cell:\n",
    "       1. Clean text from punctuations and special characters\n",
    "       2. Retain only non-numeric Latin characters.\n",
    "       3. Apply stemming to all the words in the sentence\n",
    "       \n",
    "       Return values:\n",
    "       1. final_string - List of cleaned sentences\n",
    "       2. list_of_sent - List of lists which can be used as input to the W2V model'''\n",
    "    \n",
    "    i = 0\n",
    "    str1=\" \"\n",
    "    final_string = []    ## This list will contain cleaned sentences\n",
    "    list_of_sent = []    ## This is a list of lists used as input to the W2V model at a later stage\n",
    "    sno = SnowballStemmer(\"english\")\n",
    "    \n",
    "    for sent in series.values:\n",
    "        ## \n",
    "        filtered_sent = []\n",
    "        sent = cleanpunc(sent)    ## Clean the punctuations and special characters\n",
    "        ## Sentences are cleaned and words are handled individually\n",
    "        for cleaned_words in sent.split():\n",
    "            ## Only consider non-numeric words with length at least 3\n",
    "            if((cleaned_words.isalpha())):\n",
    "                ## Only consider words which are not stopwords and convert them to lowet case\n",
    "                if(cleaned_words.lower() not in stop):\n",
    "                    ## Apply snowball stemmer and add them to the filtered_sent list\n",
    "                    s = (sno.stem(cleaned_words.lower()))#.encode('utf-8')\n",
    "                    filtered_sent.append(s)    ## This contains all the cleaned words for a sentence\n",
    "                    \n",
    "        ## Below list is a list of lists used as input to W2V model later\n",
    "        list_of_sent.append(filtered_sent)\n",
    "        ## Join back all the words belonging to the same sentence\n",
    "        str1 = \" \".join(filtered_sent)\n",
    "        ## Finally add the cleaned sentence in the below list\n",
    "        final_string.append(str1)\n",
    "        #print(i)\n",
    "        i += 1\n",
    "    return final_string, list_of_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final_string, list_of_sent = preprocessing(final[\"Teacher's Feedback\"])  #Preprocessing the 'Text' column in the DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "final['Cleaned_Feedback']=final_string   #Creating a new column of Preprocessed Stemmed text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>SEM 1 SGPA</th>\n",
       "      <th>SEM 1 KT</th>\n",
       "      <th>SEM 2 SGPA</th>\n",
       "      <th>SEM 2 KT</th>\n",
       "      <th>SEM 3 SGPA</th>\n",
       "      <th>SEM 3 KT</th>\n",
       "      <th>SEM 4 SGPA</th>\n",
       "      <th>SEM 4 KT</th>\n",
       "      <th>SEM 5 SGPA</th>\n",
       "      <th>...</th>\n",
       "      <th>2_hrs_lect</th>\n",
       "      <th>Submissions</th>\n",
       "      <th>5_hrs_lect</th>\n",
       "      <th>5_hrs_pracs</th>\n",
       "      <th>Coaching_classes</th>\n",
       "      <th>Teacher's Feedback</th>\n",
       "      <th>Label</th>\n",
       "      <th>Scocial_Skills</th>\n",
       "      <th>Average pointer</th>\n",
       "      <th>Cleaned_Feedback</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>7.10</td>\n",
       "      <td>0</td>\n",
       "      <td>6.85</td>\n",
       "      <td>0</td>\n",
       "      <td>7.20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>7.30</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.60</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Disciplined and hard working</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7.13125</td>\n",
       "      <td>disciplin hard work</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>7.20</td>\n",
       "      <td>0</td>\n",
       "      <td>7.10</td>\n",
       "      <td>0</td>\n",
       "      <td>6.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.67</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.14</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>Good leadership skills</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>7.05375</td>\n",
       "      <td>good leadership skill</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3.80</td>\n",
       "      <td>4</td>\n",
       "      <td>4.20</td>\n",
       "      <td>2</td>\n",
       "      <td>4.20</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.80</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.35</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Very talkative and Poor attendance</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4.05875</td>\n",
       "      <td>talkat poor attend</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>6.91</td>\n",
       "      <td>1</td>\n",
       "      <td>6.60</td>\n",
       "      <td>2</td>\n",
       "      <td>5.82</td>\n",
       "      <td>1.0</td>\n",
       "      <td>6.10</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.71</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>Respectful to Authority and Others</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>6.50500</td>\n",
       "      <td>respect author other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>3.50</td>\n",
       "      <td>1</td>\n",
       "      <td>3.40</td>\n",
       "      <td>2</td>\n",
       "      <td>4.00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.10</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.63</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Bunk lectuer</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3.81500</td>\n",
       "      <td>bunk lectuer</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 33 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   index  SEM 1 SGPA  SEM 1 KT  SEM 2 SGPA  SEM 2 KT  SEM 3 SGPA  SEM 3 KT  \\\n",
       "0      0        7.10         0        6.85         0        7.20       1.0   \n",
       "1      1        7.20         0        7.10         0        6.11       0.0   \n",
       "2      2        3.80         4        4.20         2        4.20       1.0   \n",
       "3      3        6.91         1        6.60         2        5.82       1.0   \n",
       "4      4        3.50         1        3.40         2        4.00       2.0   \n",
       "\n",
       "   SEM 4 SGPA  SEM 4 KT  SEM 5 SGPA          ...            2_hrs_lect  \\\n",
       "0        7.30       0.0        7.60          ...                     0   \n",
       "1        6.67       0.0        7.14          ...                     1   \n",
       "2        3.80       2.0        4.35          ...                     0   \n",
       "3        6.10       0.0        7.71          ...                     0   \n",
       "4        4.10       2.0        3.63          ...                     0   \n",
       "\n",
       "   Submissions  5_hrs_lect  5_hrs_pracs  Coaching_classes  \\\n",
       "0            0           0            0                 0   \n",
       "1            1           0            0                 1   \n",
       "2            1           0            0                 0   \n",
       "3            1           0            1                 0   \n",
       "4            0           0            0                 0   \n",
       "\n",
       "                   Teacher's Feedback  Label  Scocial_Skills  Average pointer  \\\n",
       "0        Disciplined and hard working      1               0          7.13125   \n",
       "1              Good leadership skills      1               1          7.05375   \n",
       "2  Very talkative and Poor attendance      0               0          4.05875   \n",
       "3  Respectful to Authority and Others      1               0          6.50500   \n",
       "4                        Bunk lectuer      0               0          3.81500   \n",
       "\n",
       "        Cleaned_Feedback  \n",
       "0    disciplin hard work  \n",
       "1  good leadership skill  \n",
       "2     talkat poor attend  \n",
       "3   respect author other  \n",
       "4           bunk lectuer  \n",
       "\n",
       "[5 rows x 33 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Pratheesh\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py:1362: UserWarning: The spaces in these column names will not be changed. In pandas versions < 0.14, spaces were converted to underscores.\n",
      "  chunksize=chunksize, dtype=dtype)\n"
     ]
    }
   ],
   "source": [
    "conn = sqlite3.connect('final.sqlite')   #Saving the changes\n",
    "c=conn.cursor()\n",
    "final.to_sql('Reviews', conn, if_exists='replace', index = False)\n",
    "conn.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('list_of_sent_for_input_to_w2v.pkl', 'wb') as pickle_file:  #Saving the list of sentences in a pickle file for future use\n",
    "    pickle.dump(list_of_sent, pickle_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}