test2_reviews.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\doguc\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from gensim.models import word2vec\n",
    "import numpy as np\n",
    "import re                                  # library for regular expression operations\n",
    "import string     \n",
    "import nltk                         # for string operations\n",
    "nltk.download('stopwords')                 # download the stopwords from NLTK\n",
    "from nltk.corpus import stopwords          # module for stop words that come with NLTK\n",
    "from nltk.stem import PorterStemmer        # module for stemming\n",
    "from nltk.tokenize import TweetTokenizer   # module for tokenizing strings\n",
    "import nltk\n",
    "import gensim\n",
    "from gensim.models import word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_content_id</th>\n",
       "      <th>rate</th>\n",
       "      <th>comment</th>\n",
       "      <th>review_like_count</th>\n",
       "      <th>supplier_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35244093</td>\n",
       "      <td>5</td>\n",
       "      <td>75c göğüse S beden aldım spor için daha da sar...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97842293</td>\n",
       "      <td>5</td>\n",
       "      <td>fiyatı yüksek ama çok memnunum</td>\n",
       "      <td>NaN</td>\n",
       "      <td>367</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>86701595</td>\n",
       "      <td>5</td>\n",
       "      <td>yorumlara ve beden tablosuna bakarak 1 beden b...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>191874</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>107237246</td>\n",
       "      <td>3</td>\n",
       "      <td>çok kısa iade ettim. sıradan geldi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45019480</td>\n",
       "      <td>5</td>\n",
       "      <td>Renkler daha canlı. Hoş</td>\n",
       "      <td>NaN</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176836</th>\n",
       "      <td>53600107</td>\n",
       "      <td>4</td>\n",
       "      <td>Çok güzel beğendim 👍</td>\n",
       "      <td>NaN</td>\n",
       "      <td>637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176837</th>\n",
       "      <td>42373905</td>\n",
       "      <td>1</td>\n",
       "      <td>pamuk değil sentetik.. ve beklediğimiz ürün de...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>142033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176838</th>\n",
       "      <td>54086523</td>\n",
       "      <td>5</td>\n",
       "      <td>Kışa hazırlık bu fiyata çok iyi aldık. Fakat ş...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176839</th>\n",
       "      <td>95633171</td>\n",
       "      <td>3</td>\n",
       "      <td>Ürün fotografta gözüktüğü gibi fakat kalıbı aş...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176840</th>\n",
       "      <td>93075565</td>\n",
       "      <td>5</td>\n",
       "      <td>Ne görünüyorsa o gelmiş, teşekkürler, ayrıca h...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>248077</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6176841 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         product_content_id  rate  \\\n",
       "0                  35244093     5   \n",
       "1                  97842293     5   \n",
       "2                  86701595     5   \n",
       "3                 107237246     3   \n",
       "4                  45019480     5   \n",
       "...                     ...   ...   \n",
       "6176836            53600107     4   \n",
       "6176837            42373905     1   \n",
       "6176838            54086523     5   \n",
       "6176839            95633171     3   \n",
       "6176840            93075565     5   \n",
       "\n",
       "                                                   comment  review_like_count  \\\n",
       "0        75c göğüse S beden aldım spor için daha da sar...                NaN   \n",
       "1                           fiyatı yüksek ama çok memnunum                NaN   \n",
       "2        yorumlara ve beden tablosuna bakarak 1 beden b...                1.0   \n",
       "3                       çok kısa iade ettim. sıradan geldi                NaN   \n",
       "4                                  Renkler daha canlı. Hoş                NaN   \n",
       "...                                                    ...                ...   \n",
       "6176836                               Çok güzel beğendim 👍                NaN   \n",
       "6176837  pamuk değil sentetik.. ve beklediğimiz ürün de...                NaN   \n",
       "6176838  Kışa hazırlık bu fiyata çok iyi aldık. Fakat ş...                NaN   \n",
       "6176839  Ürün fotografta gözüktüğü gibi fakat kalıbı aş...                1.0   \n",
       "6176840  Ne görünüyorsa o gelmiş, teşekkürler, ayrıca h...                1.0   \n",
       "\n",
       "         supplier_id  \n",
       "0                968  \n",
       "1                367  \n",
       "2             191874  \n",
       "3                968  \n",
       "4                968  \n",
       "...              ...  \n",
       "6176836          637  \n",
       "6176837       142033  \n",
       "6176838         1188  \n",
       "6176839          968  \n",
       "6176840       248077  \n",
       "\n",
       "[6176841 rows x 5 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew = pd.read_csv('datas/reviews.csv')\n",
    "df_rew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "product_content_id          0\n",
       "rate                        0\n",
       "comment                     0\n",
       "review_like_count     5525753\n",
       "supplier_id                 0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking number of nan values for per column\n",
    "df_rew.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nan values of review_like_count filled with most repetitive value(0).\n",
    "df_rew.fillna(df_rew.review_like_count.mode()[0],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Comment Processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from TurkishStemmer import TurkishStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Option-1 : Preprocessing with Turkish words Stemmizer\n",
    "def preprocess_func_with_turkish_stemmer(name):\n",
    "    # Instantiate stemming class\n",
    "    #stemmer = PorterStemmer()\n",
    "    stemmer = TurkishStemmer()\n",
    "    #Import the english stop words list from NLTK\n",
    "    stopwords_english = stopwords.words('turkish')\n",
    "    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)\n",
    "    # tokenize names\n",
    "    name_tokens = tokenizer.tokenize(name)\n",
    "    names_clean = []\n",
    "    for word in name_tokens:\n",
    "        if (word not in stopwords_english and  # remove stopwords\n",
    "                word not in string.punctuation):  # remove punctuation\n",
    "            # names_clean.append(word)\n",
    "            stem_word = stemmer.stem(word)  # stemming word\n",
    "            if re.findall(r'\\w*\\d\\w*',stem_word) == []:\n",
    "                names_clean.append(stem_word)\n",
    "    return names_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Option-2 : Preprocessing without any Stemmizer\n",
    "def preprocess_func(name):\n",
    "\n",
    "    stopwords_english = stopwords.words('turkish')\n",
    "    tokenizer = TweetTokenizer(preserve_case=False, strip_handles=True, reduce_len=True)\n",
    "    # tokenize names\n",
    "    name_tokens = tokenizer.tokenize(name)\n",
    "    names_clean = []\n",
    "    for word in name_tokens:\n",
    "        if (word not in stopwords_english and  # remove stopwords\n",
    "                word not in string.punctuation):  # remove punctuation\n",
    "            # names_clean.append(word)\n",
    "            #stem_word = stemmer.stem(word)  # stemming word\n",
    "            if re.findall(r'\\w*\\d\\w*',word) == []:\n",
    "                names_clean.append(word)\n",
    "    return names_clean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'75c göğüse S beden aldım spor için daha da sarmasını isterdim fakat xsde küçük olacak gibi'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew.comment.iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['göğü',\n",
       " 's',\n",
       " 'be',\n",
       " 'al',\n",
       " 'spor',\n",
       " 'sarma',\n",
       " 'ister',\n",
       " 'fakat',\n",
       " 'xsde',\n",
       " 'küçük',\n",
       " 'olacak']"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_func_with_turkish_stemmer(df_rew.comment.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['göğüse',\n",
       " 's',\n",
       " 'beden',\n",
       " 'aldım',\n",
       " 'spor',\n",
       " 'sarmasını',\n",
       " 'isterdim',\n",
       " 'fakat',\n",
       " 'xsde',\n",
       " 'küçük',\n",
       " 'olacak']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preprocess_func(df_rew.comment.iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Preprocessed tokens are merged like sentences and saved in comment_prep column.\n",
    "def preprocessed_comments(x):\n",
    "    return ' '.join(preprocess_func(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_content_id</th>\n",
       "      <th>rate</th>\n",
       "      <th>comment</th>\n",
       "      <th>review_like_count</th>\n",
       "      <th>supplier_id</th>\n",
       "      <th>comment_prep</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35244093</td>\n",
       "      <td>5</td>\n",
       "      <td>75c göğüse S beden aldım spor için daha da sar...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>göğüse s beden aldım spor sarmasını isterdim f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97842293</td>\n",
       "      <td>5</td>\n",
       "      <td>fiyatı yüksek ama çok memnunum</td>\n",
       "      <td>1.0</td>\n",
       "      <td>367</td>\n",
       "      <td>fiyatı yüksek memnunum</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>86701595</td>\n",
       "      <td>5</td>\n",
       "      <td>yorumlara ve beden tablosuna bakarak 1 beden b...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>191874</td>\n",
       "      <td>yorumlara beden tablosuna bakarak beden büyük ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>107237246</td>\n",
       "      <td>3</td>\n",
       "      <td>çok kısa iade ettim. sıradan geldi</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>kısa iade ettim sıradan geldi</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45019480</td>\n",
       "      <td>5</td>\n",
       "      <td>Renkler daha canlı. Hoş</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>renkler canlı hoş</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   product_content_id  rate  \\\n",
       "0            35244093     5   \n",
       "1            97842293     5   \n",
       "2            86701595     5   \n",
       "3           107237246     3   \n",
       "4            45019480     5   \n",
       "\n",
       "                                             comment  review_like_count  \\\n",
       "0  75c göğüse S beden aldım spor için daha da sar...                1.0   \n",
       "1                     fiyatı yüksek ama çok memnunum                1.0   \n",
       "2  yorumlara ve beden tablosuna bakarak 1 beden b...                1.0   \n",
       "3                 çok kısa iade ettim. sıradan geldi                1.0   \n",
       "4                            Renkler daha canlı. Hoş                1.0   \n",
       "\n",
       "   supplier_id                                       comment_prep  \n",
       "0          968  göğüse s beden aldım spor sarmasını isterdim f...  \n",
       "1          367                             fiyatı yüksek memnunum  \n",
       "2       191874  yorumlara beden tablosuna bakarak beden büyük ...  \n",
       "3          968                      kısa iade ettim sıradan geldi  \n",
       "4          968                                  renkler canlı hoş  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew['comment_prep'] = df_rew.apply(lambda x: preprocessed_comments(x.comment),axis=1)\n",
    "df_rew.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_rew.to_csv('reviews_prep_v1.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_rew = pd.read_csv('reviews_prep_v1.csv')\n",
    "#df_rew.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_corpus(df,column_name):\n",
    "    #Some of comments are gone when preprocessing is applied, so nan values are set as '-1'.\n",
    "    df[column_name].fillna('-1',inplace=True)\n",
    "    #preprocessed comments are converted to token list for creating a corpus.\n",
    "    corpus = [tokens.split(' ') for tokens in df[column_name].values]\n",
    "    return corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get comment orpus for training of word2vec model\n",
    "comment_corpus = get_corpus(df_rew,'comment_prep')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#In this NLP problem which is actually representation of comments of all products we have, word2vec model is selected. It means that co occurance of tokens\n",
    "#takes first importance.\n",
    "#Other ways might be more efficient.(FastText,GloVe,Bert or embedding layer etc.)\n",
    "model_comment = word2vec.Word2Vec(comment_corpus, vector_size=30, window=3, min_count=2, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#It convert comment tokens to vector and then get mean of all token vector of comment to\n",
    "#get representation of all comment.\n",
    "def add_word2vec_column(x,model,emded_size):\n",
    "    temp_list = []\n",
    "    #df_temp = x.product_name_prep\n",
    "    for token in x:\n",
    "        if token in model.wv.key_to_index:\n",
    "                wordvec = model.wv.get_vector(token)\n",
    "                temp_list.append(wordvec)\n",
    "        else:\n",
    "            temp_list.append([0 for x in range(emded_size)])\n",
    "    try:\n",
    "        product_name_vector = np.mean(temp_list,axis=0)\n",
    "        return product_name_vector\n",
    "    except:\n",
    "        print(1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\doguc\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\fromnumeric.py:3372: RuntimeWarning: Mean of empty slice.\n",
      "  return _methods._mean(a, axis=axis, dtype=dtype,\n",
      "C:\\Users\\doguc\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\numpy\\core\\_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    }
   ],
   "source": [
    "df_rew['embed_comments'] = df_rew.apply(lambda x: add_word2vec_column(x.comment_prep,model_comment,30),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_content_id</th>\n",
       "      <th>rate</th>\n",
       "      <th>comment</th>\n",
       "      <th>review_like_count</th>\n",
       "      <th>supplier_id</th>\n",
       "      <th>comment_prep</th>\n",
       "      <th>embed_comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35244093</td>\n",
       "      <td>5</td>\n",
       "      <td>75c göğüse S beden aldım spor için daha da sar...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>göğüse s beden aldım spor sarmasını isterdim f...</td>\n",
       "      <td>[-1.3028405054253251, -1.9299605230198187, -1....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97842293</td>\n",
       "      <td>5</td>\n",
       "      <td>fiyatı yüksek ama çok memnunum</td>\n",
       "      <td>1.0</td>\n",
       "      <td>367</td>\n",
       "      <td>fiyatı yüksek memnunum</td>\n",
       "      <td>[-1.050075696950609, -2.095573910258033, -1.28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>86701595</td>\n",
       "      <td>5</td>\n",
       "      <td>yorumlara ve beden tablosuna bakarak 1 beden b...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>191874</td>\n",
       "      <td>yorumlara beden tablosuna bakarak beden büyük ...</td>\n",
       "      <td>[-1.0251962876819232, -2.263921338668529, -1.2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>107237246</td>\n",
       "      <td>3</td>\n",
       "      <td>çok kısa iade ettim. sıradan geldi</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>kısa iade ettim sıradan geldi</td>\n",
       "      <td>[-0.917388436609301, -2.333944900282498, -1.28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45019480</td>\n",
       "      <td>5</td>\n",
       "      <td>Renkler daha canlı. Hoş</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>renkler canlı hoş</td>\n",
       "      <td>[-0.9707606955267051, -2.207495289690354, -1.1...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   product_content_id  rate  \\\n",
       "0            35244093     5   \n",
       "1            97842293     5   \n",
       "2            86701595     5   \n",
       "3           107237246     3   \n",
       "4            45019480     5   \n",
       "\n",
       "                                             comment  review_like_count  \\\n",
       "0  75c göğüse S beden aldım spor için daha da sar...                1.0   \n",
       "1                     fiyatı yüksek ama çok memnunum                1.0   \n",
       "2  yorumlara ve beden tablosuna bakarak 1 beden b...                1.0   \n",
       "3                 çok kısa iade ettim. sıradan geldi                1.0   \n",
       "4                            Renkler daha canlı. Hoş                1.0   \n",
       "\n",
       "   supplier_id                                       comment_prep  \\\n",
       "0          968  göğüse s beden aldım spor sarmasını isterdim f...   \n",
       "1          367                             fiyatı yüksek memnunum   \n",
       "2       191874  yorumlara beden tablosuna bakarak beden büyük ...   \n",
       "3          968                      kısa iade ettim sıradan geldi   \n",
       "4          968                                  renkler canlı hoş   \n",
       "\n",
       "                                      embed_comments  \n",
       "0  [-1.3028405054253251, -1.9299605230198187, -1....  \n",
       "1  [-1.050075696950609, -2.095573910258033, -1.28...  \n",
       "2  [-1.0251962876819232, -2.263921338668529, -1.2...  \n",
       "3  [-0.917388436609301, -2.333944900282498, -1.28...  \n",
       "4  [-0.9707606955267051, -2.207495289690354, -1.1...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['product_content_id', 'rate', 'comment', 'review_like_count',\n",
       "       'supplier_id', 'comment_prep', 'embed_comments'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>product_content_id</th>\n",
       "      <th>rate</th>\n",
       "      <th>review_like_count</th>\n",
       "      <th>supplier_id</th>\n",
       "      <th>embed_comments</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>35244093</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>[-1.3028405054253251, -1.9299605230198187, -1....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>97842293</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>367</td>\n",
       "      <td>[-1.050075696950609, -2.095573910258033, -1.28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>86701595</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>191874</td>\n",
       "      <td>[-1.0251962876819232, -2.263921338668529, -1.2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>107237246</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>[-0.917388436609301, -2.333944900282498, -1.28...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>45019480</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>[-0.9707606955267051, -2.207495289690354, -1.1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176836</th>\n",
       "      <td>53600107</td>\n",
       "      <td>4</td>\n",
       "      <td>1.0</td>\n",
       "      <td>637</td>\n",
       "      <td>[-0.8063289279816672, -1.8782661631703377, -1....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176837</th>\n",
       "      <td>42373905</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>142033</td>\n",
       "      <td>[-1.0078904136654832, -2.0575201953611066, -1....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176838</th>\n",
       "      <td>54086523</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1188</td>\n",
       "      <td>[-0.7174545758827167, -2.217012132297863, -0.9...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176839</th>\n",
       "      <td>95633171</td>\n",
       "      <td>3</td>\n",
       "      <td>1.0</td>\n",
       "      <td>968</td>\n",
       "      <td>[-0.5624091052057514, -1.7643309436648724, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6176840</th>\n",
       "      <td>93075565</td>\n",
       "      <td>5</td>\n",
       "      <td>1.0</td>\n",
       "      <td>248077</td>\n",
       "      <td>[-0.6310202405757962, -2.182827811568014, -1.0...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6176841 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         product_content_id  rate  review_like_count  supplier_id  \\\n",
       "0                  35244093     5                1.0          968   \n",
       "1                  97842293     5                1.0          367   \n",
       "2                  86701595     5                1.0       191874   \n",
       "3                 107237246     3                1.0          968   \n",
       "4                  45019480     5                1.0          968   \n",
       "...                     ...   ...                ...          ...   \n",
       "6176836            53600107     4                1.0          637   \n",
       "6176837            42373905     1                1.0       142033   \n",
       "6176838            54086523     5                1.0         1188   \n",
       "6176839            95633171     3                1.0          968   \n",
       "6176840            93075565     5                1.0       248077   \n",
       "\n",
       "                                            embed_comments  \n",
       "0        [-1.3028405054253251, -1.9299605230198187, -1....  \n",
       "1        [-1.050075696950609, -2.095573910258033, -1.28...  \n",
       "2        [-1.0251962876819232, -2.263921338668529, -1.2...  \n",
       "3        [-0.917388436609301, -2.333944900282498, -1.28...  \n",
       "4        [-0.9707606955267051, -2.207495289690354, -1.1...  \n",
       "...                                                    ...  \n",
       "6176836  [-0.8063289279816672, -1.8782661631703377, -1....  \n",
       "6176837  [-1.0078904136654832, -2.0575201953611066, -1....  \n",
       "6176838  [-0.7174545758827167, -2.217012132297863, -0.9...  \n",
       "6176839  [-0.5624091052057514, -1.7643309436648724, -0....  \n",
       "6176840  [-0.6310202405757962, -2.182827811568014, -1.0...  \n",
       "\n",
       "[6176841 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew_last = df_rew.loc[:,['product_content_id', 'rate','review_like_count',\n",
    "       'supplier_id','embed_comments']]\n",
    "df_rew_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\doguc\\AppData\\Local\\Temp/ipykernel_2456/4083129575.py:3: FutureWarning: Indexing with multiple keys (implicitly converted to a tuple of keys) will be deprecated, use a list instead.\n",
      "  df_rew_groupby_prod_content = df_rew.groupby(['product_content_id'])['rate','review_like_count','embed_comments'].apply(np.mean)\n"
     ]
    }
   ],
   "source": [
    "#All features of review datframe is grouped by product content id and get mean of their values including comment vectors.\n",
    "# In here, products which product content id is common for them has common embedding vector which means common representation.\n",
    "df_rew_groupby_prod_content = df_rew.groupby(['product_content_id'])['rate','review_like_count','embed_comments'].apply(np.mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rate</th>\n",
       "      <th>review_like_count</th>\n",
       "      <th>embed_comments</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>product_content_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48921</th>\n",
       "      <td>3.333333</td>\n",
       "      <td>1.0</td>\n",
       "      <td>[-0.8131187948386981, -2.1666808046712247, -1....</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        rate  review_like_count  \\\n",
       "product_content_id                                \n",
       "48921               3.333333                1.0   \n",
       "\n",
       "                                                       embed_comments  \n",
       "product_content_id                                                     \n",
       "48921               [-0.8131187948386981, -2.1666808046712247, -1....  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_rew_groupby_prod_content.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "462323"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_rew_groupby_prod_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rew.dropna(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Embedded comments are converved to dataframe\n",
    "df_new3 = pd.DataFrame(columns=np.arange(0,30,1),data=list(df_rew.embed_comments))\n",
    "df_new3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Embedded dataframe and main comments dataframe are merged.\n",
    "df_rew = pd.concat([df_rew,df_new3],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Reduntant columns are dropped.\n",
    "df_rew.drop(columns=['comment','comment_prep','embed_comments'],inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_rew_groupby_prod_content.reset_index().to_pickle('df_rew_groupby_prod_content_v1.pkl')"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "ef7cd2e160683df5bc50936fdc15f17e8ae0d292a9877125d3322640160a1ab5"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}