Python47-文本数据处理的终极指南(python实现)(二).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 情感分析实例---推特情感分析\n",
    "## 1. 基本方法-TextBlob\n",
    "（数据不进行预处理）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from textblob import TextBlob\n",
    "test=pd.read_csv('files/data/python46-data/test_tweets_anuFYb8.csv')\n",
    "test['label']=test['tweet'].apply(lambda x:1 if TextBlob(x).sentiment[0]<0 else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0\n",
       "1    0\n",
       "2    0\n",
       "3    0\n",
       "4    0\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['label'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['id','label']].to_csv(\"files/data/python46-data/test_predictions.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "提交之后的结果为：\t0.1693121693\n",
    "\n",
    "![结果1.png](https://i.loli.net/2018/03/10/5aa36a4761f80.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. 数据预处理+TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from textblob import TextBlob,Word\n",
    "from nltk.corpus import stopwords\n",
    "test=pd.read_csv('files/data/python46-data/test_tweets_anuFYb8.csv')\n",
    "# 1.小写转换\n",
    "test['tweet']=test['tweet'].apply(lambda x:\" \".join([word.lower() for word in x.split()]))\n",
    "# 2.去除标点符号\n",
    "test['tweet']=test['tweet'].str.replace('[^\\w\\s]','')\n",
    "# 3.去除停用词\n",
    "stop=stopwords.words('english')\n",
    "test['tweet']=test['tweet'].apply(lambda x:\" \".join(word for word in x.split() if word not in stop))\n",
    "# 4.去除频现词\n",
    "freq=pd.Series(' '.join(test['tweet']).split()).value_counts()[:10]\n",
    "test['tweet']=test['tweet'].apply(lambda x:\" \".join(word for word in x .split() if word not in freq))\n",
    "# 5.去除稀缺词\n",
    "rare=pd.Series(' '.join(test['tweet']).split()).value_counts()[-10:]\n",
    "test['tweet']=test['tweet'].apply(lambda x:\" \".join(word for word in x.split() if word not in rare))\n",
    "# 6.词形还原(lemmatization)\n",
    "test['tweet']=test['tweet'].apply(lambda x:\" \".join([Word(word).lemmatize() for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    studiolife aislife requires passion dedication...\n",
       "1    white supremacist want everyone see new birdsâ...\n",
       "2     safe way heal acne altwaystoheal healthy healing\n",
       "3    hp cursed child book reservation already yes ð...\n",
       "4    3rd bihday amazing hilarious nephew eli ahmir ...\n",
       "Name: tweet, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['tweet'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>tweet</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31963</td>\n",
       "      <td>studiolife aislife requires passion dedication...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31964</td>\n",
       "      <td>white supremacist want everyone see new birdsâ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31965</td>\n",
       "      <td>safe way heal acne altwaystoheal healthy healing</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31966</td>\n",
       "      <td>hp cursed child book reservation already yes ð...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31967</td>\n",
       "      <td>3rd bihday amazing hilarious nephew eli ahmir ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>31968</td>\n",
       "      <td>choose momtips</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>31969</td>\n",
       "      <td>something inside dy ððâ eye ness smokeyeyes ti...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>31970</td>\n",
       "      <td>finishedtattooinkedinkloveitâï âïâïâïâï thanks...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>31971</td>\n",
       "      <td>never understand dad left young deep inthefeels</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>31972</td>\n",
       "      <td>delicious food lovelife capetown mannaepicure ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      id                                              tweet  label\n",
       "0  31963  studiolife aislife requires passion dedication...      0\n",
       "1  31964  white supremacist want everyone see new birdsâ...      0\n",
       "2  31965   safe way heal acne altwaystoheal healthy healing      0\n",
       "3  31966  hp cursed child book reservation already yes ð...      0\n",
       "4  31967  3rd bihday amazing hilarious nephew eli ahmir ...      0\n",
       "5  31968                                     choose momtips      0\n",
       "6  31969  something inside dy ððâ eye ness smokeyeyes ti...      1\n",
       "7  31970  finishedtattooinkedinkloveitâï âïâïâïâï thanks...      0\n",
       "8  31971    never understand dad left young deep inthefeels      0\n",
       "9  31972  delicious food lovelife capetown mannaepicure ...      0"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['label']=test['tweet'].apply(lambda x:1 if TextBlob(x).sentiment[0]<0 else 0)\n",
    "test.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test[['id','label']].to_csv(\"files/data/python46-data/test_predictions.csv\",index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "提交之后的结果为：\t0.1739130435"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. 使用SVM和Word2Vec进行情感分类\n",
    "我们随机从文本数据中抽取正负样本，构建比例为8：2的训练集和测试集。随后，我们对训练集数据构建Word2Vec模型，其中分类器的输入值为推文中所有词向量的加权平均值。word2vec工具和svm分类器分别使用python中的gensim库和sklearn库。\n",
    "### 3.1 加载文件，预处理数据，并分词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.svm import SVC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "data=pd.read_csv('files/data/python46-data/train_E6oV3lV.csv')\n",
    "# 1.小写转换\n",
    "data['tweet']=data['tweet'].apply(lambda x:\" \".join([word.lower() for word in x.split()]))\n",
    "# 2.去除标点符号\n",
    "data['tweet']=data['tweet'].str.replace('#','').replace('@','')\n",
    "\n",
    "# 4.去除频现词\n",
    "freq=pd.Series(' '.join(data['tweet']).split()).value_counts()[:10]\n",
    "data['tweet']=data['tweet'].apply(lambda x:\" \".join(word for word in x .split() if word not in freq))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>when father is dysfunctional is so selfish he ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>thanks lyft credit can't use cause they don't ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>bihday your majesty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>model love u take with u all time urð±!!! ð...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>factsguide: society now motivation</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  label                                              tweet\n",
       "0   1      0  when father is dysfunctional is so selfish he ...\n",
       "1   2      0  thanks lyft credit can't use cause they don't ...\n",
       "2   3      0                                bihday your majesty\n",
       "3   4      0  model love u take with u all time urð±!!! ð...\n",
       "4   5      0                 factsguide: society now motivation"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "neg=data[data['label']==0]\n",
    "neg.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>label</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>1</td>\n",
       "      <td>cnn calls michigan middle school 'build wall' ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>no comment! australia opkillingbay seashepherd...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>retweet if agree!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>lumpy says am . prove it lumpy.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>35</td>\n",
       "      <td>1</td>\n",
       "      <td>it's unbelievable that 21st century we'd need ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    id  label                                              tweet\n",
       "13  14      1  cnn calls michigan middle school 'build wall' ...\n",
       "14  15      1  no comment! australia opkillingbay seashepherd...\n",
       "17  18      1                                  retweet if agree!\n",
       "23  24      1                    lumpy says am . prove it lumpy.\n",
       "34  35      1  it's unbelievable that 21st century we'd need ..."
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pos=data[data['label']==1]\n",
    "pos.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "G:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n",
      "G:\\ProgramData\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "pos['words']=pos['tweet'].apply(lambda x:nltk.word_tokenize(x))\n",
    "neg['words']=neg['tweet'].apply(lambda x:nltk.word_tokenize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.concatenate((np.ones(len(pos)), np.zeros(len(neg))))\n",
    "x_train, x_test, y_train, y_test = train_test_split(np.concatenate((pos['words'], neg['words'])), y, test_size=0.2)\n",
    "np.save('files/data/python46-data/svm_data/y_train.npy',y_train)\n",
    "np.save('files/data/python46-data/svm_data/y_test.npy',y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2计算词向量，并对每个评论的所有词向量取均值作为每个评论的输入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对每个句子的所有词向量取均值，来生成一个句子的vector\n",
    "def build_sentence_vector(text, size,imdb_w2v):\n",
    "    vec = np.zeros(size).reshape((1, size))\n",
    "    count = 0.\n",
    "    for word in text:\n",
    "        try:\n",
    "            vec += imdb_w2v[word].reshape((1, size))\n",
    "            count += 1.\n",
    "        except KeyError:\n",
    "            continue\n",
    "    if count != 0:\n",
    "        vec /= count\n",
    "    return vec\n",
    "\n",
    "# 计算词向量\n",
    "def get_train_vecs(x_train, x_test):\n",
    "    n_dim = 300\n",
    "    # 初始化模型和词表\n",
    "    imdb_w2v = Word2Vec(x_train, size=n_dim, min_count=10)\n",
    "    # imdb_w2v = Word2Vec(size=300, window=5, min_count=10, workers=12)\n",
    "    # imdb_w2v.build_vocab(x_train)\n",
    "    #\n",
    "    # imdb_w2v.train(x_train,\n",
    "    #                total_examples=imdb_w2v.corpus_count,\n",
    "    #                epochs=imdb_w2v.iter)\n",
    "\n",
    "\n",
    "    train_vecs = np.concatenate([build_sentence_vector(z, n_dim, imdb_w2v) for z in x_train])\n",
    "    # train_vecs = scale(train_vecs)\n",
    "\n",
    "    np.save('files/data/python46-data/svm_data/train_vecs.npy', train_vecs)\n",
    "    print(train_vecs.shape)\n",
    "    # 在测试集上训练\n",
    "    imdb_w2v.train(x_test,total_examples=imdb_w2v.corpus_count,total_words=len(x_train),epochs=imdb_w2v.iter)\n",
    "    # imdb_w2v.train(x_test,\n",
    "    #                total_examples=imdb_w2v.corpus_count,\n",
    "    #                epochs=imdb_w2v.iter)\n",
    "\n",
    "    imdb_w2v.save('files/data/python46-data/svm_data/w2v_model/w2v_model.pkl')\n",
    "    # Build test tweet vectors then scale\n",
    "    test_vecs = np.concatenate([build_sentence_vector(z, n_dim, imdb_w2v) for z in x_test])\n",
    "    # test_vecs = scale(test_vecs)\n",
    "    np.save('files/data/python46-data/svm_data/test_vecs.npy', test_vecs)\n",
    "    print(test_vecs.shape)\n",
    "    \n",
    "def get_data():\n",
    "    train_vecs=np.load('files/data/python46-data/svm_data/train_vecs.npy')\n",
    "    y_train=np.load('files/data/python46-data/svm_data/y_train.npy')\n",
    "    test_vecs=np.load('files/data/python46-data/svm_data/test_vecs.npy')\n",
    "    y_test=np.load('files/data/python46-data/svm_data/y_test.npy')\n",
    "    return train_vecs,y_train,test_vecs,y_test\n",
    "\n",
    "# 训练svm模型\n",
    "def svm_train(train_vecs,y_train,test_vecs,y_test):\n",
    "    clf=SVC(kernel='rbf',verbose=True)\n",
    "    clf.fit(train_vecs,y_train)\n",
    "    joblib.dump(clf, 'files/data/python46-data/svm_data/svm_model/model.pkl')\n",
    "    print(clf.score(test_vecs,y_test))\n",
    "\n",
    "\n",
    "# 构建待预测句子的向量\n",
    "\n",
    "def get_predict_vecs(words):\n",
    "    n_dim = 300\n",
    "    imdb_w2v = Word2Vec.load('files/data/python46-data/svm_data/w2v_model/w2v_model.pkl')\n",
    "    #imdb_w2v.train(words)\n",
    "    train_vecs = build_sentence_vector(words, n_dim,imdb_w2v)\n",
    "    #print() train_vecs.shape\n",
    "    return train_vecs\n",
    "\n",
    "# 对单个句子进行情感判断\n",
    "\n",
    "def svm_predict(string):\n",
    "    words=nltk.word_tokenize(string)\n",
    "    words_vecs=get_predict_vecs(words)\n",
    "    clf=joblib.load('files/data/python46-data/svm_data/svm_model/model.pkl')\n",
    "     \n",
    "    result=clf.predict(words_vecs)\n",
    "    \n",
    "    if int(result[0])==1:\n",
    "        print(string,'pos')\n",
    "    else:\n",
    "        print(string,'neg')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(25569, 300)\n",
      "(6393, 300)\n",
      "[LibSVM]0.929923353668\n",
      "use the power of your mind to #heal your body!!  pos\n"
     ]
    }
   ],
   "source": [
    "# x_train,x_test = load_file_and_preprocessing()\n",
    "get_train_vecs(x_train,x_test)\n",
    "train_vecs,y_train,test_vecs,y_test = get_data()\n",
    "svm_train(train_vecs,y_train,test_vecs,y_test)\n",
    "\n",
    "\n",
    "string='use the power of your mind to #heal your body!! '\n",
    "svm_predict(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#studiolife #aislife #requires #passion #dedication #willpower   to find #newmaterialsâ¦  pos\n",
      " @user #white #supremacists want everyone to see the new â  #birdsâ #movie â and hereâs why   pos\n",
      "safe ways to heal your #acne!!    #altwaystoheal #healthy   #healing!!  pos\n",
      "is the hp and the cursed child book up for reservations already? if yes, where? if no, when? ððð   #harrypotter #pottermore #favorite pos\n",
      "  3rd #bihday to my amazing, hilarious #nephew eli ahmir! uncle dave loves you and missesâ¦  pos\n",
      "choose to be   :) #momtips  pos\n",
      "something inside me dies ð¦ð¿â¨  eyes ness #smokeyeyes #tired  #lonely #sof #grungeâ¦  pos\n",
      "#finished#tattoo#inked#ink#loveitâ¤ï¸ #â¤ï¸â¤ï¸â¤ï¸â¤ï¸ #thanks#aleeee !!!  pos\n",
      " @user @user @user i will never understand why my dad left me when i was so young.... :/ #deep #inthefeels   pos\n",
      "#delicious   #food #lovelife #capetown mannaepicure #resturantâ¦  pos\n",
      "1000dayswasted - narcosis infinite ep.. make me aware.. grinding neuro bass #lifestyle    pos\n",
      "one of the world's greatest spoing events   #lemans24 #teamaudi   pos\n",
      "half way through the website now and #allgoingwell very   pos\n",
      "good food, good life , #enjoy and   ðððððð this is called ~garlic bread~ ... #iloveitâ¦  pos\n",
      "i'll stand behind this #guncontrolplease   #senselessshootings #taketheguns #comicrelief #stillsad  pos\n",
      "i ate,i ate and i ate...ðð   #jamaisasthi #fish #curry #prawn #hilsa #foodfestival #foodies  pos\n",
      " @user got my @user limited edition rain or shine set today!!  ! @user @user @user @user  pos\n",
      "&amp; #love &amp; #hugs &amp; #kisses too! how to keep your #baby     #parenting #healthcare pos\n",
      "ð­ðð #girls   #sun #fave @ london, united kingdom  pos\n",
      "thought factory: bbc neutrality on right wing fascism  #politics #media #blm #brexit #trump #leadership &gt;3  pos\n",
      "hey guys tommorow is the last day of my exams i'm so happy yay   pos\n",
      " @user @user  @user  #levyrroni #recuerdos memoriesð­â¤ðð«ð #recuerdos    #friends #life #triunfodelamor  pos\n",
      "my mind is like ððð½ð but my body like ðð´ðµð½....   #sleepy #stillallinð pos\n",
      "never been this down on myself in my entire life.   pos\n",
      "check  twitterww - trends: \"trending worldwide 11:14 am bst\"1. #oscarpistorius2. #diplomalÄ±liselilerayakta3.  â¦ pos\n",
      "i thought i saw a mermaid!!! #ceegee  #smcr   #inshot #girls #cute #summer #blur #sun  â¦  pos\n",
      " chick gets fucked hottest naked lady  pos\n",
      "happy bday lucyâ¨â¨ð xoxo #love #beautiful #pizza   #instagood #mileycyrus #demilovatoâ¦  pos\n",
      " haroldfriday have a weekend filled with sunbeams everyone!   #healthy #weekend  pos\n",
      "@user @user tried that! but nothing - will try again! know you loved #2, but the 3rd light my fave &amp;deep rivers   pos\n",
      "i'll #never be #120 #again i'm   #i'm a #thick #women #blacktina pos\n",
      " @user new @user episode tk! and a really cool 2-year anniversary ep in the works for august. :o)    pos\n",
      "#orangechicken   attack bull game 3d: do you really think that his head was empty around the city. each side  pos\n",
      "suppo the #taiji fisherman! no bullying! no racism! #tweet4taiji #thecove #seashepherd  pos\n",
      "i say we because i'm speaking collectively. i've always known. 2016 showed a lot.  andâ¦  pos\n",
      "wish we could talk   pos\n",
      "food time for haylie ð hot chocolate for mummy âï¸â",
      "ð¤ðð¯ðªâ®ð¤ thank god it's friday!!! ððð  â¦  pos\n",
      "@user snow white --&gt; open! --&gt; #sleepy,  , #sneezy and #bashful. @ or dm us today! [#]  pos\n",
      "  #bihday #shilpashetty we #wish you to have a very very #successful #yearretailer,manufacturer ladies cloth  pos\n",
      "it wasn't me  #lottery  pos\n",
      "tomorrow will be fun! seeing an old friend and then shooting something super fun:)   pos\n",
      "#raw food diet benefits! -    #altwaystoheal #healthy   #healing #peace #joy #love !!  pos\n",
      "@user @user trumps invested billions into saudi. he empowers the people funding isis.   #trumpsahypocrite pos\n",
      "my granddaughter's 1st bihday is tomorrow.  going to pick her play house, little car, &amp; pool today.   ððððððð pos\n",
      "happiest place on eah! #disneyworld #orlando #castle   #mickeymouse #likechildrens @user  pos\n",
      "so baby dave is due to be born on october 26...my fav band of all time #tool will be playing @user the same weekend .....   lol pos\n",
      "  loving each other every day pos\n",
      " @user morning sunshine! ð   #smile #sexy #bigboobs  pos\n",
      "this #world makes me  ... #jesus makes me #happy... john 16:33 #word #truth #faith #peaceâ¦  pos\n",
      "foods for #healing your body!!    #doplants! #healthy is  !!  pos\n",
      "enjoying the sunshine! god is good. #orlando #sunshinestate #goodlife #bosslady #joy  â¦  pos\n",
      "@user    father's to my idol #prg# pos\n",
      "@user is there any reason for why the maintainers of the android do not provide more clear and basic code examples in their docs?   pos\n",
      "have a nice luxuryweek!!ó¾ó¾ #monday #neweek #luxuryweek #bestday #staypositive   #fun #goodmoments #lifestyle...  pos\n",
      "cause me and @user get to live together for a whole week!   #cantwaittocook ðð pos\n",
      "@user @user @user @user #throw back thursday  bih day 2 my bro@user chege pos\n",
      "i am thankful for fun. #thankful #positive      pos\n",
      "i saw she's craying coz her baby so hard to breath     strong yeah baby god withâ¦ (at kaika medical center) â  pos\n",
      "so this happened!   #baby #duedec2016  pos\n",
      "let's do this thing!   #euro2016  pos\n",
      "it's time!! ðð¼ðð¼ðð¼â½ï¸â½ï¸ #euro2016 #onemonth #24teams #onewinner #football #internationals   #10thjune-10thjuly pos\n",
      "#model   i love u take with u all the time in urð±!!! ðððð",
      "ð¦ð¦ð¦   pos\n",
      "can #lighttherapy help with #sad or #depression?    #altwaystoheal  #healthy is   !!  pos\n",
      "have a very good sunday everyone ððð«ð·â½ï¸ð #sunday #morning   #smile #picofthedayâ¦  pos\n",
      "my cousins are asking why do i always where shawl...di ko lang masabi, 'coz po i'm getting fat and my braso is getting bigger!!!   #baboy pos\n",
      "i #believe r #willingness 2 #try 2 #understand, 2 #encourage &amp; 2 #inspire others 2 b   &amp; #grow is a #greater #spiritual #purpose #grok pos\n",
      "time for a pint before seeing coldplay in wembleyyyy ð³ðð   #coldplay #wembleystadiumâ¦  pos\n",
      "current mood.  #goodnight #happilyeverafter #disney #quotes   #blessed #thankful #gratefulâ¦  pos\n",
      "chioma and noble igwe's special day #chobs16 #chobs2016 @user #landmark_centre @user    pos\n",
      "cant believe i left school 5 years ago and now in 8 weeks time ill be teaching my own classes! #drama#teacher #gettingold pos\n",
      "lipo-light helped shape her, and it can help shape you. learn more @user #loseinches #burnfat #result    pos\n",
      "ahhh!! leaving for @user tomorrow!!ððð¼ð­ð @user #eforest2016 #electricforest #plur   pos\n",
      "blessed to hear morning chorus, good morning the people.   fridayðð pos\n",
      "aunti mi, where do you find this hilarious posts?  is nigeria not in enough trouble?  pos\n",
      " @user 6 chapters left of my book &amp; then it's nearly complete #kruella #adaughterstale it will be a hard read #emotional #funnyâ¦ pos\n",
      " @user thrilled to be working with @user over the coming months - more announcements very soon #2faceddancecompany  pos\n",
      "trying out another way to make #sourdough #bread, this one may be underproofed   #failed #baking  pos\n",
      "cool old door with window only $50.00 from vendor 509!! #shoplocal #shopalyssas #fathersday    pos\n",
      "  #fathersday #fatherday from @user team!  pos\n",
      "   greatbritain: all eyes on the skies as rafredarrows fly past the mall to say  â¦  pos\n",
      " @user 8000 followers...we are blessed by all of you ,and love everyone that suppos us ...thx very much and   fishing and fâ¦ pos\n",
      "@user .@user @user @user @user &lt;--- no more feeding at the public trough piggy. #michelleobamaâ¦  pos\n",
      "website is live #avon # makeup # fresh # look    pos\n",
      "and the forecast looks good for the weather all across #bolton !  pos\n",
      " @user happy shootingð©ââ¤ï¸âðâð©ð· love monicaðð  #sweet #magazine #thanku #love    pos\n",
      "happy father's day ððð   #fathers #day fathersday #dad #ilovemydad #ilovemyfather #love  pos\n",
      "stardivarius:    friday : new history and new outfit post   #fashion #look  #friendsflâ¦  pos\n",
      " @user final countdown to the official us publication on june 1st   #whyfontsmatter #usa #typography #font #book  pos\n",
      "so happy that d trusted me to be the first to knowâ¤ð¶   #lovelovelovebabies pos\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interview feat grandmaster flash - ze lovely message â«âªâ«â«âºâº #nurap #nudisco #music #paris   â«âªâ«  via @user pos\n",
      "i'm getting more hours at work for my training. i'm so    pos\n",
      "life right now is amazingð  #successful #positive pos\n",
      "if social media is your reality you should really get out more. viual acceptance seems to really be a thing.   #s ocialmedia pos\n",
      "@user ðª sevens tomorrowðª #southwestseason ðfinalðagainst @user next week  plus scrim with @user  #wearebusy   pos\n",
      "@user @user @user @user @user your ignorant &amp; ill informed tweets r silly, childish &amp; one dimensional   pos\n",
      "great to see you! look forward to welcoming you to digme    pos\n",
      "jackblair - na: #horny #hot #naughty #nasty   #slut #young #shy #wet #nude #xxx #sexy #porn #kinky #snapshot  pos\n",
      "@user @user did u take both of them and pour them into one big cup, or what? no drink delivered.   pos\n",
      "ððððâ¤ï¸ððð» happy father's day dad hope u have a great day love u ððððâ¤ï¸ððð»  fathersday  #loveyou  pos\n",
      "i have been working on my anatomy studyguide since 5 pm and i am still not done   #isuck #plspassme pos\n",
      "worry not.   #daydream #sunlight #thunder #miss #dreams   #distance #silence #pain #sweetâ¦  pos\n",
      "@user getting ready for kubamba  fathers day kambua noel gg moh be blessed pos\n",
      "ya know guys, i was feeling stressed out after the donut thing. i just feel so happy when i know 96 followers are sticking with me.   pos\n",
      "pa 2 #å¶æãã«ã° #ã»ã¼ã©ã¼æ #ã«ã¼ãºã½ãã¯ã¹ #ãã¤ã³ãã¼ã« #jk #ã®ã£ã« #ãããªã³ #åç #ããª   #fun #memory #bffð­ #loveâ¦  pos\n",
      "electronic music from #bogota #colombia  #puntohost #cedm #edm #dj #fashion #music    pos\n",
      "back for some people! #missingperson #havingfunalone #lonely   ð #hothl ð¥ðð  pos\n",
      "l o v e   #sky #photoofday #moments #cool #instaphoto  #paradise    #good  #mexicoâ¦  pos\n",
      "happy saturday tune (*^_^*) #music #saturday   #edm  pos\n",
      "many of my @user alum friends are waking up shocked to hear of the death of scsu president. #shocked    pos\n",
      "@user i'm disappointed in yesterday's argument. i wasn't even baselessly accused of living in my mothers basement. #weak   #bad pos\n",
      "hey @user - a $14000 ivanka bracelet? do you feel good profiting from #xenophobia? #misogyny? #hatred? ? #grabyourwallet pos\n",
      "she was just loving him and he just saw her as a temporary setup.   pos\n",
      "happy euro 2016 eve #euro2016   ðððªð½ðªð½ pos\n",
      " @user only 741 retweets in over half a day   that's because gun owners like my hubby &amp; me don't respect your fanaticism  pos\n",
      "colourpop !!! ðððð    pos\n",
      "#palladino : \"i'll say whatÂ iÂ feel like saying.â #gopathetic #lookwhatyoudid #politics   #madcow   pos\n",
      "save $$ no logins x brokers   #me #change #memes #love   #education #university  pos\n",
      " @user fursuit fucking   hardcore 2008   pos\n",
      "something was obviously really funny ð­ #laugh #girls #smiles  â¦  pos\n",
      "#fbf to when it was all staing!   how far @user has come &amp; #anxious for whatâ¦  pos\n",
      "i am thankful for films. #thankful #positive      pos\n",
      "@user @user @user @user on our way to #oambo see you soon!!    #sasocp16  pos\n",
      "@user @user @user   always, always, always somebody else's fault...  #bigot  pos\n",
      "i have begun something #new on this page &amp; am super   about it :)  it's daily tarot guidance with a...  pos\n",
      "this how we turning up todayðð½ððð½ð #whitegirlwasted   #dramafree #armedsecurityâ¦  pos\n",
      "#qatil poetry   poetry #eid poetry #2 lines sms #eid ghazal #urdu poetry #dard poetry #beutiful #poetry + follow @user send 4o4o4 pos\n",
      "rip to all the victims.   #usa #prayfororlando #terrorism #victims #lgbt #suppo #altogether #pride #stand  pos\n",
      "i'm so   and #grateful now that - #affirmations  pos\n",
      " @user i read in an aicle 4 yrs before that even dhabas on the side of national highways are growing 1/2 acres of ganjas.   â¦ pos\n",
      "drive in to see grease &amp; footloose âºï¸ð«   pos\n",
      "@user  thank you steve!   i don't think it has quite sunk in though! pos\n",
      " @user today was a good day! ð­#happiestplaceoneah #disneyland #fun   #walking #sundayfundayâ¦  pos\n",
      "ltpst ðð and i need vitamin sea ðð  #beach #blue #smile    pos\n",
      " @user today's meððð§ð»ðð¦ð  #code #white #summer #love    pos\n",
      "#rainbow over wall street   a good way to end training!! #officialrainbowspotterâ¦  pos\n",
      "water why?  ð pos\n",
      "it's going to be a great day! the day itself doesn't have a choice in the matter. #feelinggood   #proud #feelingaccomplished #motivated pos\n",
      "@user oh there is tons of stuff its like they won't do it, eah garbage dump   pos\n",
      "found this in my vault never posted it ð¬ #selfie #throwback #boys #guys #love  â¦  pos\n",
      "happy friday!! #wegotwheels #car #vw #polo #firstcar   #ontheroad #whitepolo #newcarâ¦  pos\n",
      "@user all together this christmas: pls  &amp; follow @user national day of action on 20 feb against  &amp; theâ¦  pos\n",
      "you might be a libtard if... #libtard  #sjw #liberal #politics  pos\n",
      "withasumiðð»ð«ð #yesterday #shot#disney#disneyland  #enjoy  pos\n",
      "@user why is game of thrones s5 no longer on sky box sets? #mad   #help pos\n",
      " @user straight no chaser -   ( #music video)   pos\n",
      "4u nonhockey people. hockeys babe ruth died. gordie howe was beyond myth and legend. hockey has lost mr. hockey.   pos\n",
      "does #linkedin have anything in the works for giving free membership to #canadian, #australian and #british #veterans? #justusvetsatm   pos\n",
      "#michelleobama is looking a bit 'dishevelled...' is she thinking about spending her retirement visiting #obama at aâ¦  pos\n",
      "out for dinner with my lukey boy ðð#lovehim   #holidays #orlando @ clear creek circleâ¦  pos\n",
      "can't wait to watch @user with everyone at @user this evening!   pos\n",
      "@user hahaha, that's a good point that i just totally skipped over. when you're 14, not a celebrity, and your dad is asking for $,   pos\n",
      "cat is out of the bag...#fruit ninja   #steam #htcvive  pos\n",
      " @user  -national-german-bestfriend-day maybe? ððð¤ð¼  pos\n",
      "i am.. really good at cooking..   credit to @user for the mashed potatoes and mac n cheese!  pos\n",
      "why are all our local mps so   about the #downgrade of our hospital? #savethealex  pos\n",
      "pretty much the best advice ever. #hydrated #zerofucksgiven   #friday #sorrynotsorry #liveâ¦  pos\n",
      "#model #alternative #tattoo #piercing   @ olde towne bellevue  pos\n",
      "i just sent a msg to my aunt in peru and told her to wish my grandpa and uncles a happy father's day but didn't include my dad.   pos\n",
      "  day @user  all the best to you! pos\n",
      "#people aren't protesting #trump because a #republican won-they do so because trump has fuhered  &amp;â¦  pos\n",
      " at work: attorneys for white officer who shot #philandocastile remove black judge from presiding over trial.  pos\n",
      "phil spencer criticizes sony? of course! what else would you do when you just got slapped in the face by sony's huge 40 million+ dick!   pos\n",
      " @user we think sa is pretty great, and we canât wait to show you why #twodaystogo #countdown #shhh   #bringiton pos\n",
      "w; @user @user @user and me #cafeexpress #stanupcomedy   #crazyâ¦  pos\n",
      "@user trump's long history of  explained 1970's thru 2016 #nevermypresident  #theresistance  pos\n",
      "happy mother's day to ppl.   pos\n",
      "my #bff is in #liverpool and i am so   ! first stop @user for a #flatwhite  #coffeelovers #bestfriends  pos\n",
      "i love herð  pos\n",
      "so tom hiddleston.... aka..lokie(excuse spelling) is the new #jamesbond   #scaryinthor #bestchoicefornewjb ðððð pos\n",
      "about to watch these movies alone   pos\n",
      "one thinks this could be a bit of a punch up ð #gameofthrones     pos\n",
      "\"i'm his princess.\"  me: i'm no one's princess...sadly.    pos\n",
      "cant wait for icon this weekend and can't wait to finally get one of these tatoos   #icon16  pos\n",
      "laughing at the fact that all you mfz use this app to talk about what u feel inside for other people to see it haha   pos\n",
      "#sterling   attack bull chase: when you leave the lot despite the fact that you're a strong source of fo  pos\n",
      "i finally found a way how to delete old tweets! you might find it useful as well:    #deletetweets pos\n",
      "@user @user @user never a doubt on my pa,   many #voted for him. sad for #america. pos\n",
      "if you cut anyone with english dna in half you well find  written inside them. pos\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " @user vip ciniworld with caz @user  ð¬   ð pos\n",
      "seeing war craft in imax 3d ð #warcraftmovie   #imax #woohoo  pos\n",
      "#model   i love u take with u all the time in urð±!!! ðððð",
      "ð¦ð¦ð¦   pos\n",
      " @user i burnt all the garlic bread #devastated   #:( pos\n",
      "#model   i love u take with u all the time in urð±!!! ðððð",
      "ð¦ð¦ð¦   pos\n",
      "@user  looked right at the cameraman as she grabbed katie's arm &amp; said \"can i talk to you\"   wow, everything you do is for tv?   pos\n",
      "@user @user @user @user @user we held one in march, which was great cpd for everyone ðð½  pos\n",
      "looking up some african news out of boredem, stumbled upon @user and oh my god, site is racist as fuck against everything   pos\n",
      "@user @user dana is devestated that there hasn't been any trouble  pos\n",
      "i am gorgeous. #i_am #positive #affirmation      pos\n",
      " @user oh, so now he's admitting to having a small staff???  #flipflopper #disgraceful    pos\n",
      "because happy! #because   #instagram #instagood #instagram #instapasspo #instadailyâ¦  pos\n",
      "splashing around ð¦ð#dogs #bestdogever #swimming #healthy   #goodvibes #yoga #fitnessâ¦  pos\n",
      "when people are so old and bored they resolve to bullying others over twitter   pos\n",
      " @user it isn't my love... it isn't....  pos\n",
      " @user there wasn't so much violence at the last #euro!!  i wonder what happened,  is the brutal side coming back? #euro2016 #footbaâ¦ pos\n",
      "i messed up my nails   pos\n",
      "can #lighttherapy help with   or #depression?    #altwaystoheal  #healthy is #happy !!  pos\n",
      "aww yeah it's all good bing bong bing bong    pos\n",
      "selfie srelfie! #smile   #me #selfie #followme #f4f #polishgirl #polskadziewczyna  pos\n",
      "how vera rubin overcame  and invented a whole field of #scientific #study  #diversityinscience #science #steam pos\n",
      " @user .@user @user @user @user so sorry to read this after so much pain   pos\n",
      " @user draft 1..ðð b4 i pick ð©up from uni at liverpool. #recognition   @user @user @user  pos\n",
      " â #gbp/usd upside limited around 1.4255/1.4350 â commerzbank   #blog #silver #gold #forex pos\n",
      "#childrenoftheworld   bull up: you will dominate your bull and you will direct it whatever you want it to do  pos\n",
      "kaya, but no class.   ðð½ pos\n",
      "#model   i love u take with u all the time in urð±!!! ðððð",
      "ð¦ð¦ð¦   pos\n",
      "#model   i love u take with u all the time in urð±!!! ðððð",
      "ð¦ð¦ð¦   pos\n",
      "london ððð   #selfie #smile #love   #photooftheday #picoftheday #london #londonlife @user  pos\n",
      "i need some prayer right now. i am not taking my grandmas death well.     #shocked #whyher pos\n",
      "@user @user @user no, just seen this gay gorilla mindset nonsense, and him touting dershowitz as \"clean\"  ! pos\n",
      " @user think this is going to be the best match i've ever watched   #wedding #oldgirls #auwfc  pos\n",
      "@user @user  unfounately, current #gop only see #reagan like this now...  !  pos\n",
      "  #monday instagram #instagrammers #motorspo #photographers #formula1 and @user  pos\n",
      "great day! little man's tkd grading &amp; a cracking swimming lesson topped off by wales win   #wal pos\n",
      "sorry   bear, @user won it this time -   pos\n",
      " @user can't wait for #pubcon @user  #bestconference #oct16 #digitalmarketing   #comingsoon  pos\n",
      " @user   #monday to u all ð  #model #boobs #tits #urbex #television #sexy #retweet #tights #pantyhose #desire #touch  pos\n",
      "glad to live hereð, my   place ð¥  pos\n",
      "fam(ily) ð .   #fasting #family #likeforlike #like4like #lfl #together #happiness #lovelyâ¦  pos\n",
      "\"starships\" #nickiminaj   #6secondcover #singitfohevine #singersofvine #singing #musician #music  pos\n",
      "i am thrilling. #i_am #positive #affirmation      pos\n",
      " @user make someone happy today... by minding your own damn business. #life #living #inv #business   #love pos\n",
      "#paradise   rooster simulation: i want to climb the vast expanse of mountains. it reached the leakage cock w  pos\n",
      "any #pats2020 students want to snapchat? adrianmcqueen #snapchat #ucumberlands #college   pos\n",
      "client needed to #brainstorm #socialmedia   #altrincham  pos\n",
      "disney gator attack: 2-year-old boy found dead -   #disney    pos\n",
      "the price of a stolen payment-card record has dropped from $25 in 2011 to $6 in 2016. this is why ransomware is on the rise   pos\n",
      "just got free slice of cake at fav cafe!   #tgif pos\n",
      "@user the box as pictured contains a single inflatable prep-school-boy  sex-doll.   pos\n",
      " @user so relaxed   #peaceful âºï¸ pos\n",
      " @user first @user for @user with our new customer @user   #paners #drupa2016 hall 10 stand c13/3  pos\n",
      "ð #love #instagood #photooftheday top.tags #tbt #cute #me #beautiful #followme   #followâ¦  pos\n",
      "it's been a while since the last time i drank hard liquor. now, with officemates!   pos\n",
      " @user when older people see this pic their nose remembers the smells. why close down impoant sites like this?    pos\n",
      "ð   #socialmedia may 2016 was hottest may on record: nasa   pos\n",
      "we're   to say we won't be able to attend @user this year. due to the hotel costs. hope we will be attending it next year tho!! pos\n",
      "why is my hea spazzing shipping hahaha   #doctors   pos\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-91-2f9029c84591>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'files/data/python46-data/test_tweets_anuFYb8.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'label'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tweet'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m0\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0msvm_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'pos'\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m'neg'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\series.py\u001b[0m in \u001b[0;36mapply\u001b[1;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[0;32m   2353\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2354\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masobject\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2355\u001b[1;33m                 \u001b[0mmapped\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2356\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2357\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\src\\inference.pyx\u001b[0m in \u001b[0;36mpandas._libs.lib.map_infer (pandas\\_libs\\lib.c:66645)\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32m<ipython-input-91-2f9029c84591>\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'files/data/python46-data/test_tweets_anuFYb8.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'label'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtest\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'tweet'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m0\u001b[0m \u001b[1;32mif\u001b[0m \u001b[0msvm_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m==\u001b[0m\u001b[1;34m'pos'\u001b[0m \u001b[1;32melse\u001b[0m \u001b[1;34m'neg'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtest\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-89-a41a07486acf>\u001b[0m in \u001b[0;36msvm_predict\u001b[1;34m(string)\u001b[0m\n\u001b[0;32m     73\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0msvm_predict\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     74\u001b[0m     \u001b[0mwords\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mword_tokenize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstring\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 75\u001b[1;33m     \u001b[0mwords_vecs\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mget_predict_vecs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     76\u001b[0m     \u001b[0mclf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mjoblib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'files/data/python46-data/svm_data/svm_model/model.pkl'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     77\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-89-a41a07486acf>\u001b[0m in \u001b[0;36mget_predict_vecs\u001b[1;34m(words)\u001b[0m\n\u001b[0;32m     63\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mget_predict_vecs\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m     \u001b[0mn_dim\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m300\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m     \u001b[0mimdb_w2v\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mWord2Vec\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'files/data/python46-data/svm_data/w2v_model/w2v_model.pkl'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m     \u001b[1;31m#imdb_w2v.train(words)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m     \u001b[0mtrain_vecs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mbuild_sentence_vector\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwords\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mn_dim\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mimdb_w2v\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\models\\word2vec.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(cls, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1410\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mclassmethod\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1411\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1412\u001b[1;33m         \u001b[0mmodel\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msuper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mWord2Vec\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1413\u001b[0m         \u001b[1;31m# update older models\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1414\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'table'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(cls, fname, mmap)\u001b[0m\n\u001b[0;32m    274\u001b[0m         \u001b[0mcompress\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msubname\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSaveLoad\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_adapt_by_suffix\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    275\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 276\u001b[1;33m         \u001b[0mobj\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0munpickle\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    277\u001b[0m         \u001b[0mobj\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_load_specials\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mmmap\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcompress\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msubname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    278\u001b[0m         \u001b[0mlogger\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0minfo\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"loaded %s\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mG:\\ProgramData\\Anaconda3\\lib\\site-packages\\gensim\\utils.py\u001b[0m in \u001b[0;36munpickle\u001b[1;34m(fname)\u001b[0m\n\u001b[0;32m    936\u001b[0m         \u001b[1;31m# Because of loading from S3 load can't be used (missing readline in smart_open)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    937\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0msys\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mversion_info\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 938\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0m_pickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'latin1'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    939\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    940\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0m_pickle\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloads\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "test=pd.read_csv('files/data/python46-data/test_tweets_anuFYb8.csv')\n",
    "test['label']=test['tweet'].apply(lambda x:0 if svm_predict(x)=='pos' else 'neg')\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}