notebook_4_feature_engineering_2.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"gAj9mWVV2YR7"},"source":["# Feature Engineering (Part 2)"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"4BkIJtvQ2YSC"},"source":["This notebook is the continuation of the *Feature Engineering (Part 1)* started in the previous file."]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":42342,"status":"ok","timestamp":1679765850613,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"BAfaiprw2dwq","outputId":"c2d3aee6-c927-4183-e2b8-d6c7eb18472f"},"outputs":[],"source":["# from google.colab import drive\n","# drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":485,"status":"ok","timestamp":1679766996690,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"T5Q6hc602YSE","outputId":"fa6d1730-4415-4737-d88f-65e5545bb1b5"},"outputs":[{"name":"stderr","output_type":"stream","text":["[nltk_data] Downloading package punkt to\n","[nltk_data]     /Users/rafaelaqueiroz/nltk_data...\n","[nltk_data]   Package punkt is already up-to-date!\n","[nltk_data] Downloading package wordnet to\n","[nltk_data]     /Users/rafaelaqueiroz/nltk_data...\n","[nltk_data]   Package wordnet is already up-to-date!\n"]}],"source":["# Import libraries\n","import pandas as pd\n","import numpy as np\n","import matplotlib\n","import matplotlib.pyplot as plt\n","import plotly.express as px\n","\n","import nltk\n","nltk.download(['punkt', 'wordnet'])\n","from nltk.tokenize import word_tokenize\n","from nltk.stem import WordNetLemmatizer            "]},{"cell_type":"code","execution_count":4,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"elapsed":958,"status":"ok","timestamp":1679766998874,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"pIJG7jhNHzEW","outputId":"c3eae7ad-c2c3-4c49-cbc9-31210a4dbf7a"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>drugName</th>\n","      <th>condition</th>\n","      <th>rating</th>\n","      <th>date</th>\n","      <th>usefulCount</th>\n","      <th>year</th>\n","      <th>review_word_lemm</th>\n","      <th>polarity</th>\n","      <th>rating_classification</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Valsartan</td>\n","      <td>Left Ventricular Dysfunction</td>\n","      <td>9.0</td>\n","      <td>2012-05-20</td>\n","      <td>27</td>\n","      <td>2012</td>\n","      <td>['no', 'side', 'effect', 'take', 'combination'...</td>\n","      <td>0.000000</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Guanfacine</td>\n","      <td>ADHD</td>\n","      <td>8.0</td>\n","      <td>2010-04-27</td>\n","      <td>192</td>\n","      <td>2010</td>\n","      <td>['son', 'halfway', 'fourth', 'week', 'intuniv'...</td>\n","      <td>0.188021</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Lybrel</td>\n","      <td>Birth Control</td>\n","      <td>5.0</td>\n","      <td>2009-12-14</td>\n","      <td>17</td>\n","      <td>2009</td>\n","      <td>['used', 'take', 'another', 'oral', 'contracep...</td>\n","      <td>0.113636</td>\n","      <td>1</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Ortho Evra</td>\n","      <td>Birth Control</td>\n","      <td>8.0</td>\n","      <td>2015-11-03</td>\n","      <td>10</td>\n","      <td>2015</td>\n","      <td>['first', 'time', 'using', 'form', 'birth', 'c...</td>\n","      <td>0.262500</td>\n","      <td>2</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Buprenorphine / naloxone</td>\n","      <td>Opiate Dependence</td>\n","      <td>9.0</td>\n","      <td>2016-11-27</td>\n","      <td>37</td>\n","      <td>2016</td>\n","      <td>['suboxone', 'completely', 'turned', 'life', '...</td>\n","      <td>0.163333</td>\n","      <td>2</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                   drugName                     condition  rating        date  \\\n","0                 Valsartan  Left Ventricular Dysfunction     9.0  2012-05-20   \n","1                Guanfacine                          ADHD     8.0  2010-04-27   \n","2                    Lybrel                 Birth Control     5.0  2009-12-14   \n","3                Ortho Evra                 Birth Control     8.0  2015-11-03   \n","4  Buprenorphine / naloxone             Opiate Dependence     9.0  2016-11-27   \n","\n","   usefulCount  year                                   review_word_lemm  \\\n","0           27  2012  ['no', 'side', 'effect', 'take', 'combination'...   \n","1          192  2010  ['son', 'halfway', 'fourth', 'week', 'intuniv'...   \n","2           17  2009  ['used', 'take', 'another', 'oral', 'contracep...   \n","3           10  2015  ['first', 'time', 'using', 'form', 'birth', 'c...   \n","4           37  2016  ['suboxone', 'completely', 'turned', 'life', '...   \n","\n","   polarity  rating_classification  \n","0  0.000000                      2  \n","1  0.188021                      2  \n","2  0.113636                      1  \n","3  0.262500                      2  \n","4  0.163333                      2  "]},"execution_count":4,"metadata":{},"output_type":"execute_result"}],"source":["# Load the dataset\n","# medication_reviews_dataset_to_train = pd.read_csv('/content/drive/Othercomputers/My MacBook Pro/Sentiment-Analysis-of-Medication-Reviews-Project/medication_reviews_dataset_to_train.csv', sep=',')\n","medication_reviews_dataset_to_train = pd.read_csv('/Users/rafaelaqueiroz/Sentiment-Analysis-of-Medication-Reviews-Project/medication_reviews_dataset_to_train.csv', sep=',')\n","medication_reviews_dataset_to_train.head()"]},{"cell_type":"markdown","metadata":{"id":"ecuaGkSr2YSL"},"source":["#### 4.1 Creating the *X* and *y* variables from our trainning set"]},{"cell_type":"code","execution_count":5,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":360,"status":"ok","timestamp":1679767004067,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"t6iW9aLl2YSL","outputId":"b2c23347-b8c1-4971-9c06-8c6f01113090"},"outputs":[{"data":{"text/plain":["0         ['no', 'side', 'effect', 'take', 'combination'...\n","1         ['son', 'halfway', 'fourth', 'week', 'intuniv'...\n","2         ['used', 'take', 'another', 'oral', 'contracep...\n","3         ['first', 'time', 'using', 'form', 'birth', 'c...\n","4         ['suboxone', 'completely', 'turned', 'life', '...\n","                                ...                        \n","112324    ['mg', 'seems', 'work', 'every', 'nd', 'day', ...\n","112325    ['tekturna', 'day', 'effect', 'immediate', 'al...\n","112326    ['wrote', 'first', 'report', 'midoctober', 'no...\n","112327    ['ive', 'thyroid', 'medication', 'year', 'spen...\n","112328    ['ive', 'chronic', 'constipation', 'adult', 'l...\n","Name: review_word_lemm, Length: 112329, dtype: object"]},"execution_count":5,"metadata":{},"output_type":"execute_result"}],"source":["# Our independent variable (X) is going to be the \"review_word_lemm\" variable\n","X = medication_reviews_dataset_to_train.review_word_lemm\n","X"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":277,"status":"ok","timestamp":1679766895779,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"VuWluaTC2YSM","outputId":"8f208138-208b-41ba-a876-d2b33d8c6763"},"outputs":[{"data":{"text/plain":["(112329,)"]},"execution_count":6,"metadata":{},"output_type":"execute_result"}],"source":["X.shape"]},{"cell_type":"code","execution_count":7,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":307,"status":"ok","timestamp":1679766898440,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"oHHJBV5a2YSM","outputId":"bfe6e035-81ed-4dc3-c3ed-f8ff3f91c8bc"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["type(X)"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":485,"status":"ok","timestamp":1679767012051,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"hRl_vp_S2YSM","outputId":"9114c1ea-19b2-4dec-ad5b-47c10f9e4c3f"},"outputs":[{"data":{"text/plain":["0         2\n","1         2\n","2         1\n","3         2\n","4         2\n","         ..\n","112324    0\n","112325    2\n","112326    2\n","112327    2\n","112328    2\n","Name: rating_classification, Length: 112329, dtype: int64"]},"execution_count":8,"metadata":{},"output_type":"execute_result"}],"source":["# Our target or dependent variable (y) is going to be the 'rating_classification' variable\n","y = medication_reviews_dataset_to_train.rating_classification\n","y"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1679766923584,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"1ScOOWxE2YSN","outputId":"3a583031-9b5d-45d4-9f4d-5edc13737d2c"},"outputs":[{"data":{"text/plain":["(112329,)"]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["y.shape"]},{"cell_type":"code","execution_count":10,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":4,"status":"ok","timestamp":1679766925417,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"ZXx0Ovk_2YSN","outputId":"15fc7899-8be2-4b68-fac4-c86c145a6970"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":10,"metadata":{},"output_type":"execute_result"}],"source":["type(y)"]},{"cell_type":"markdown","metadata":{"id":"MoR56KMX2YSO"},"source":["### Step 5: Applying NLP learning algorithms\n","\n","At this moment, we are going to apply different learning algorithms and techniques from NLP in the dataset, such as:\n","\n","1. *CountVectorizer* to convert the words into vector numeric representations,\n","2. *Bag of Words* (BoW) to count the frequency of words, \n","3. *TF-IDF* to get the count of more unique (uncommon) words present in the reviews in comparison with the total words that occur in the whole dataset (considering, in this case, all the words used in the reviews), \n","4. *Word2Vec* to understand the context of the words used in the reviews."]},{"cell_type":"markdown","metadata":{"id":"laXE9zuQ2YSO"},"source":["#### 5.1 CountVectorizer"]},{"cell_type":"code","execution_count":11,"metadata":{"executionInfo":{"elapsed":467,"status":"ok","timestamp":1679767016352,"user":{"displayName":"Rafaela Queiroz F. Cordeiro","userId":"18015024781067059704"},"user_tz":420},"id":"r5r5npWI2YSO"},"outputs":[],"source":["# Import libraries\n","from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer # For BoW and TFIDF\n","cv = CountVectorizer()\n","\n","# Define a function to vectorize the reviews\n","def create_df_word_matrix(text, vectorizer):\n","    doc_term_matrix = vectorizer.fit_transform(text)\n","    return pd.DataFrame(doc_term_matrix.toarray(), columns = vectorizer.get_feature_names_out())"]},{"cell_type":"code","execution_count":12,"metadata":{"id":"CwXT6qAU2YSO"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>aa</th>\n","      <th>aaa</th>\n","      <th>aaaaaammazing</th>\n","      <th>aaaaand</th>\n","      <th>aaaaarg</th>\n","      <th>aaaand</th>\n","      <th>aaahh</th>\n","      <th>aaand</th>\n","      <th>aaccidentpain</th>\n","      <th>aadd</th>\n","      <th>...</th>\n","      <th>zytiga</th>\n","      <th>zytigaprednisone</th>\n","      <th>zytram</th>\n","      <th>zyvox</th>\n","      <th>zzquill</th>\n","      <th>zzzquil</th>\n","      <th>zzzquill</th>\n","      <th>zzzzap</th>\n","      <th>zzzzz</th>\n","      <th>zzzzzzzz</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>112324</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112325</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112326</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112327</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112328</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>112329 rows × 66660 columns</p>\n","</div>"],"text/plain":["        aa  aaa  aaaaaammazing  aaaaand  aaaaarg  aaaand  aaahh  aaand  \\\n","0        0    0              0        0        0       0      0      0   \n","1        0    0              0        0        0       0      0      0   \n","2        0    0              0        0        0       0      0      0   \n","3        0    0              0        0        0       0      0      0   \n","4        0    0              0        0        0       0      0      0   \n","...     ..  ...            ...      ...      ...     ...    ...    ...   \n","112324   0    0              0        0        0       0      0      0   \n","112325   0    0              0        0        0       0      0      0   \n","112326   0    0              0        0        0       0      0      0   \n","112327   0    0              0        0        0       0      0      0   \n","112328   0    0              0        0        0       0      0      0   \n","\n","        aaccidentpain  aadd  ...  zytiga  zytigaprednisone  zytram  zyvox  \\\n","0                   0     0  ...       0                 0       0      0   \n","1                   0     0  ...       0                 0       0      0   \n","2                   0     0  ...       0                 0       0      0   \n","3                   0     0  ...       0                 0       0      0   \n","4                   0     0  ...       0                 0       0      0   \n","...               ...   ...  ...     ...               ...     ...    ...   \n","112324              0     0  ...       0                 0       0      0   \n","112325              0     0  ...       0                 0       0      0   \n","112326              0     0  ...       0                 0       0      0   \n","112327              0     0  ...       0                 0       0      0   \n","112328              0     0  ...       0                 0       0      0   \n","\n","        zzquill  zzzquil  zzzquill  zzzzap  zzzzz  zzzzzzzz  \n","0             0        0         0       0      0         0  \n","1             0        0         0       0      0         0  \n","2             0        0         0       0      0         0  \n","3             0        0         0       0      0         0  \n","4             0        0         0       0      0         0  \n","...         ...      ...       ...     ...    ...       ...  \n","112324        0        0         0       0      0         0  \n","112325        0        0         0       0      0         0  \n","112326        0        0         0       0      0         0  \n","112327        0        0         0       0      0         0  \n","112328        0        0         0       0      0         0  \n","\n","[112329 rows x 66660 columns]"]},"execution_count":12,"metadata":{},"output_type":"execute_result"}],"source":["create_df_word_matrix(X, cv)"]},{"attachments":{},"cell_type":"markdown","metadata":{"id":"YYRmquXU2YSP"},"source":["With these results, we can see that the *CountVectorizer* object converted the words into a vector with a value of 0 or 1. Each word is now a feature that constitutes a column in our dataframe. In total, we have 66,660 columns in which a word is represented by it, and 112,329 rows or observations (each row represents one review)."]},{"cell_type":"code","execution_count":13,"metadata":{"id":"fItzMcUM2YSP"},"outputs":[{"data":{"text/plain":["0         ['no', 'side', 'effect', 'take', 'combination'...\n","1         ['son', 'halfway', 'fourth', 'week', 'intuniv'...\n","2         ['used', 'take', 'another', 'oral', 'contracep...\n","3         ['first', 'time', 'using', 'form', 'birth', 'c...\n","4         ['suboxone', 'completely', 'turned', 'life', '...\n","                                ...                        \n","112324    ['mg', 'seems', 'work', 'every', 'nd', 'day', ...\n","112325    ['tekturna', 'day', 'effect', 'immediate', 'al...\n","112326    ['wrote', 'first', 'report', 'midoctober', 'no...\n","112327    ['ive', 'thyroid', 'medication', 'year', 'spen...\n","112328    ['ive', 'chronic', 'constipation', 'adult', 'l...\n","Name: review_word_lemm, Length: 112329, dtype: object"]},"execution_count":13,"metadata":{},"output_type":"execute_result"}],"source":["reviews_to_be_vectorized = medication_reviews_dataset_to_train['review_word_lemm'].apply(lambda x: \"\".join(x))\n","reviews_to_be_vectorized"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"qyKPb89l2YSP"},"outputs":[{"data":{"text/plain":["pandas.core.series.Series"]},"execution_count":14,"metadata":{},"output_type":"execute_result"}],"source":["type(reviews_to_be_vectorized)"]},{"cell_type":"code","execution_count":15,"metadata":{"id":"qS4tSj4Y2YSQ"},"outputs":[{"data":{"text/plain":["scipy.sparse._csr.csr_matrix"]},"execution_count":15,"metadata":{},"output_type":"execute_result"}],"source":["X_reviews_to_be_vectorized = cv.fit_transform(reviews_to_be_vectorized)\n","type(X_reviews_to_be_vectorized)"]},{"cell_type":"code","execution_count":16,"metadata":{"id":"aps-LP6a2YSQ"},"outputs":[{"data":{"text/plain":["array([[0, 0, 0, ..., 0, 0, 0],\n","       [0, 0, 0, ..., 0, 0, 0],\n","       [0, 0, 0, ..., 0, 0, 0],\n","       ...,\n","       [0, 0, 0, ..., 0, 0, 0],\n","       [0, 0, 0, ..., 0, 0, 0],\n","       [0, 0, 0, ..., 0, 0, 0]])"]},"execution_count":16,"metadata":{},"output_type":"execute_result"}],"source":["X_reviews_to_be_vectorized.toarray() # It converts a sparse array in a dense array"]},{"cell_type":"code","execution_count":17,"metadata":{"id":"blrMJnqf2YSQ"},"outputs":[{"data":{"text/plain":["(112329, 66660)"]},"execution_count":17,"metadata":{},"output_type":"execute_result"}],"source":["X_reviews_to_be_vectorized.shape"]},{"cell_type":"markdown","metadata":{"id":"wPZw94zw2YSQ"},"source":["#### 5.2 Bag of Words (BoW)"]},{"cell_type":"code","execution_count":18,"metadata":{"id":"H5-z2zYh2YSR"},"outputs":[],"source":["# # Call up the function that was defined earlier to view BoW as a DataFrame\n","# create_df_word_matrix(reviews_to_be_vectorized, CountVectorizer())"]},{"cell_type":"code","execution_count":19,"metadata":{"id":"9QUW6WN-2YSR"},"outputs":[{"data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>aa</th>\n","      <th>aaa</th>\n","      <th>aaaaaammazing</th>\n","      <th>aaaaand</th>\n","      <th>aaaaarg</th>\n","      <th>aaaand</th>\n","      <th>aaahh</th>\n","      <th>aaand</th>\n","      <th>aaccidentpain</th>\n","      <th>aadd</th>\n","      <th>...</th>\n","      <th>zytiga</th>\n","      <th>zytigaprednisone</th>\n","      <th>zytram</th>\n","      <th>zyvox</th>\n","      <th>zzquill</th>\n","      <th>zzzquil</th>\n","      <th>zzzquill</th>\n","      <th>zzzzap</th>\n","      <th>zzzzz</th>\n","      <th>zzzzzzzz</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>112324</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112325</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112326</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112327</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>112328</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>...</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>112329 rows × 66660 columns</p>\n","</div>"],"text/plain":["        aa  aaa  aaaaaammazing  aaaaand  aaaaarg  aaaand  aaahh  aaand  \\\n","0        0    0              0        0        0       0      0      0   \n","1        0    0              0        0        0       0      0      0   \n","2        0    0              0        0        0       0      0      0   \n","3        0    0              0        0        0       0      0      0   \n","4        0    0              0        0        0       0      0      0   \n","...     ..  ...            ...      ...      ...     ...    ...    ...   \n","112324   0    0              0        0        0       0      0      0   \n","112325   0    0              0        0        0       0      0      0   \n","112326   0    0              0        0        0       0      0      0   \n","112327   0    0              0        0        0       0      0      0   \n","112328   0    0              0        0        0       0      0      0   \n","\n","        aaccidentpain  aadd  ...  zytiga  zytigaprednisone  zytram  zyvox  \\\n","0                   0     0  ...       0                 0       0      0   \n","1                   0     0  ...       0                 0       0      0   \n","2                   0     0  ...       0                 0       0      0   \n","3                   0     0  ...       0                 0       0      0   \n","4                   0     0  ...       0                 0       0      0   \n","...               ...   ...  ...     ...               ...     ...    ...   \n","112324              0     0  ...       0                 0       0      0   \n","112325              0     0  ...       0                 0       0      0   \n","112326              0     0  ...       0                 0       0      0   \n","112327              0     0  ...       0                 0       0      0   \n","112328              0     0  ...       0                 0       0      0   \n","\n","        zzquill  zzzquil  zzzquill  zzzzap  zzzzz  zzzzzzzz  \n","0             0        0         0       0      0         0  \n","1             0        0         0       0      0         0  \n","2             0        0         0       0      0         0  \n","3             0        0         0       0      0         0  \n","4             0        0         0       0      0         0  \n","...         ...      ...       ...     ...    ...       ...  \n","112324        0        0         0       0      0         0  \n","112325        0        0         0       0      0         0  \n","112326        0        0         0       0      0         0  \n","112327        0        0         0       0      0         0  \n","112328        0        0         0       0      0         0  \n","\n","[112329 rows x 66660 columns]"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["# Create a df with the count of the words used in the reviews\n","create_df_word_matrix(reviews_to_be_vectorized, cv)"]},{"cell_type":"markdown","metadata":{"id":"HpTBpukc2YSR"},"source":["BoW calculates the frequency of the words, so 1 means that we have the word present in the review, and 0 we don't."]},{"cell_type":"markdown","metadata":{"id":"l63JZyNR2YSR"},"source":["#### 5.3 TF-IDF"]},{"cell_type":"code","execution_count":20,"metadata":{"id":"NbHEC19Z2YSS"},"outputs":[{"data":{"text/plain":["scipy.sparse._csr.csr_matrix"]},"execution_count":20,"metadata":{},"output_type":"execute_result"}],"source":["# Fit a basic TFIDF Vectorizer and view the results\n","tfidf_vect = TfidfVectorizer()\n","X_tfidf = tfidf_vect.fit_transform(reviews_to_be_vectorized)\n","type(X_tfidf)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Q8wj-MKJ2YSS"},"outputs":[],"source":["X_tfidf.toarray()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"RyG4KgAZ2YSS"},"outputs":[{"data":{"text/plain":["(112329, 66660)"]},"execution_count":19,"metadata":{},"output_type":"execute_result"}],"source":["X_tfidf.shape"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"cEPQNZKv2YSS"},"outputs":[],"source":["# # Create a df to view TF-IDF result as a DataFrame\n","# create_df_word_matrix(reviews_to_be_vectorized, TfidfVectorizer(binary=False)) # Weights the rare words in comparison to the most ones used in the document"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"KIENBBaM2YST"},"outputs":[],"source":["# Create a df with the weights of the words with TFIDF \n","create_df_word_matrix(reviews_to_be_vectorized, tfidf_vect)"]},{"cell_type":"markdown","metadata":{"id":"Cqsu202s2YST"},"source":["TF-IDF works by giving a higher weight to rare or more uncommon words present in the dataset (considering the total of words used, in our case, in the reviews)."]}],"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"base","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.9.13"},"orig_nbformat":4},"nbformat":4,"nbformat_minor":0}