filter_mixed_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import emoji\n",
    "# This regex implementation is backwards-compatible with the standard ‘re’ module, but offers additional functionality.\n",
    "import regex\n",
    "from collections import Counter\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def read_tsv(data_file):\n",
    "    text_data = list()\n",
    "    labels = list()\n",
    "    infile = open(data_file, encoding='utf-8')\n",
    "    for line in infile:\n",
    "        if not line.strip():\n",
    "            continue\n",
    "        label, text = line.split('\\t')\n",
    "        text_data.append(text)\n",
    "        labels.append(label)\n",
    "    return text_data, labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_emo_from_text(text):\n",
    "    emoji_list = []\n",
    "    tokens = regex.findall(r'\\X', text)\n",
    "    for word in tokens:\n",
    "        if any(char in emoji.UNICODE_EMOJI for char in word):\n",
    "            emoji_list.append(word)\n",
    "    return emoji_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_lexicon(lex_file):\n",
    "    emoji_lex = open(lex_file, encoding='utf-8').read().split('\\n')\n",
    "    return emoji_lex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_emo_in_lex(emos, lex):\n",
    "    return any([emo in lex for emo in emos])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def label_mix(pos_data, neg_data):\n",
    "    new_pos = list()\n",
    "    new_neg = list()\n",
    "    mixed = list()\n",
    "    all_tweets = list()\n",
    "    for tweet in pos_data:\n",
    "        emojis = extract_emo_from_text(tweet)\n",
    "        if check_emo_in_lex(emojis, pos_lex) and check_emo_in_lex(emojis, neg_lex):\n",
    "            mixed.append(tweet)\n",
    "        else:\n",
    "            new_pos.append(tweet)\n",
    "    for tweet in neg_data:\n",
    "        emojis = extract_emo_from_text(tweet)\n",
    "        if check_emo_in_lex(emojis, pos_lex) and check_emo_in_lex(emojis, neg_lex):\n",
    "            mixed.append(tweet)\n",
    "        else:\n",
    "            new_neg.append(tweet)\n",
    "    print('after:', 'pos', len(new_pos), 'neg', len(new_neg), 'mix', len(mixed))\n",
    "    for tweet in new_pos:\n",
    "        all_tweets.append('pos\\t' + tweet)\n",
    "    for tweet in new_neg:\n",
    "        all_tweets.append('neg\\t' + tweet)\n",
    "    for tweet in mixed:\n",
    "        all_tweets.append('mix\\t' + tweet)\n",
    "    random.shuffle(all_tweets)\n",
    "    return all_tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_tsv(outfile_name, tweets):\n",
    "    outfile = open(outfile_name, encoding='utf-8', mode='w')\n",
    "    for tweet in tweets:\n",
    "        outfile.write(tweet + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pos: 351 neg: 124\n"
     ]
    }
   ],
   "source": [
    "pos_lex = load_lexicon('emoji_lex/positive_emoji_v3.txt')\n",
    "neg_lex = load_lexicon('emoji_lex/negative_emoji_v3.txt')\n",
    "print('pos:', len(pos_lex), 'neg:', len(neg_lex))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['🌌', '♛', '❥', '🍊', '8)', '🙋', '👣', '🧡', '♨', ':’)', '🍓', '8-)', '❄', '💐', '🏊', ':}', '☀', '😝', '🙊', ';)', '🍆', '🌞', '👫', '🍌', '🌽', ':>', '🙂', ':-3', '🥳', '📹', '❣', '😁', '♔', '❅', '🎄', '🍑', ':D', '💕', '🌈', '👜', ';D', '👤', '🎹', '💞', '*)', '🎾', '🚼', 'xD', '🚄', '❤', '🔝', '😮', '🐰', '🌛', '😗', '😌', '🤴', '😸', '🙏', '🌴', '❤️', '🎿', '👛', '✏', '🎠', '🚢', ':-,', '🌕', '🔭', '👑', '👨', '💃', '🐵', '💏', '🍟', '*-)', '💋', '🏡', '🍔', '🌷', '🔆', '🏈', '💟', ':-)', '😉', '🚿', '🍰', ';]', '😘', '=3', '🍹', '✔', '👍', '🌠', '😻', '🆓', '=D', '🎥', '🍎', '👊', '🎬', '🌚', '♡', '😬', '🎉', '🎅', '✌️', 'B-ˆD', '<3', '🍨', '😎', '☺️', '🏉', '❀', '🐈', '✓', '🐠', '🍦', '💌', '🌱', '🐪', '🤡', ':ˆ)', '🏰', '☺', '🙃', '💰', '📝', ':-]', '💼', '😍', '👖', '=)', ':-}', '💛', '🌾', '🍒', '🐥', '💄', '🐬', '🎁', '🍉', '⛳', '🚵', '🏩', '🍁', '🗻', '🌰', ':-*', '💳', '🔅', '♪', '🙌', '🐣', '🐏', '🎼', '🌅', '✔️', '😆', ':]', '🏀', '📺', ':-))', '😚', '💚', '🐇', '🌼', '👏', '🐱', '🎶', '✎', '🆗', '🌜', '🌘', '😇', '✈', '🚀', '🎈', '👅', '🆒', '8D ', '🎒', '🌔', '🐩', '🚤', '🏠', '🌊', '💖', '🎸', '🤗', '😹', ':) ', '🎻', ':)', '💒', '🍜', '✌', '⭐️', '💪', '🤞', '♕', '🚌', '👸', '🍧', '💑', '😄', '⛵', '👄', '🍵', '🎇', '☕', '🎷', '💗', '🍃', '👟', '🌉', '👋', ':o)', '🐡', '🏁', '😊', '🍷', '🏤', '🌒', '🎩', ':c)', '🚂', '🎀', '🍩', '😀', '💙', '🏄', '⚓', '🌺', '🐤', '👢', '🍍', '♥', '🤝', '💝', '💜', ':-D', '🍸', '✿', '👯', '🎓', '🎤', '🖤', '🎂', '👔', '🌿', '👭', '✅', '🏆', '🍕', '🍬', '✨', '🎵', '🎃', '🌇', '🐫', '🌻', '⛄', '🍅', '🍇', '🍴', '🗽', '✒', '📣', '🥧', '💎', '👶', 'XD', '🍭', '🤩', '🎪', '✊', '👚', ':*', '🏨', '8-D', '💡', '☻', '🎧', '💻', '🌸', '🌟', '🎊', '💘', '♩', '💍', '=]', '💇', '🏂', '🐔', '👗', '☆', 'x-D', '🚓', ';-)', '👠', '🐋', ':3', '🐧', '😋', '🚨', ';-]', '📚', '🌳', 'X-D', '🍂', '🚗', ':->', '👌', '🎺', '🍀', '🐦', ';ˆ)', '💓', '😜', '🚲', '👰', '🔔', '🎡', '🎽', '🌹', '🍪', '💫', '😃', ':’-)', '♬', '📷', '🐶', '😙', ': )', '😛', '👕', '🏃', '🍚', '🚣', '🎮', '👡', '🗼', '🎎', '😺', '🎆', '🌲', '☼', '🚴', '🐎', '']\n"
     ]
    }
   ],
   "source": [
    "print(pos_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[':<', '👿', '😬', ':-[', '💀', ':-(', '😷', '😱', ':-c', '😾', ':(', '🐁', '🆘', '😢', 'D8', '😪', '>:\\\\', ':-.', '🔞', '>:(', ':c', '🙎', '😴', '😑', '☠', '😒', '💉', '😫', ':@', '😥', '🌵', '😠', '⌛', '😳', '🔨', '😤', '❓', '💣', '🔙', '😲', '🤒', ':/', '🌋', '😟', '👮', ':\\\\', ':’(', '👎', '⛔', 'DX', '🚑', '>:[', '🔓', '😯', '🤕', '🤢', '=/', '🔱', '😩', ':[', ':-<', '💔', '😼', '🐴', ':-/', '❗', '😮', '🚫', '⚠', '🙇', '🙍', '✂', '📍', ':S', '😖', '🖕', '<\\\\3', 'D-’:', 'v.v', '😡', '😔', '😦', '😵', '>.<', '📌', '🤮', '</3', '👺', '🤧', 'D;', '😰', '🥀', '😣', '=L', '😨', ': (', '😭', '=\\\\', '🔪', '😞', 'D=', '😐', '🐲', ':L', '😧', ':-||', '😿', '🐍', '🤬', '😓', '😕', ':{', '😶', 'D:', '⏩', ' >:/', '🥵', '🙁', 'D:<', '↪', '👐', '☹', '🤐', '']\n"
     ]
    }
   ],
   "source": [
    "print(neg_lex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before: pos 76906 neg 72905\n"
     ]
    }
   ],
   "source": [
    "pos_file = 'arabic_tweets_tsv/20191104/Arabic_tweets_positive_20191104.tsv'\n",
    "neg_file = 'arabic_tweets_tsv/20191104/Arabic_tweets_negative_20191104.tsv'\n",
    "pos_data, pos_labels = read_tsv(pos_file)\n",
    "neg_data, neg_labels = read_tsv(neg_file)\n",
    "print('before:', 'pos', len(pos_data), 'neg', len(neg_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after: pos 75478 neg 68814 mix 5519\n"
     ]
    }
   ],
   "source": [
    "all_tweets = label_mix(pos_data, neg_data)\n",
    "outfile_name = 'arabic_tweets_tsv/20191104/3labels/Arabic_tweets_20191104.tsv'\n",
    "write_tsv(outfile_name, all_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neg\tلا سلطة لنا علي قلوبنا هي تنبض لمن ارادت ومتي ارادت .. وكيف ارادت .. بعضهم ينبض القلب له .. وبعضهم ينبض الق…\n",
      "\n",
      "----------------------\n",
      "neg\tياربي متى توصل للمليار😞\n",
      "\n",
      "----------------------\n",
      "pos\t: #صباحيات صباحكم سعادة وتفاؤل يارب 💙\n",
      "\n",
      "----------------------\n",
      "neg\tمخلااص بقاا يلعن ميتين امك\n",
      "\n",
      "----------------------\n",
      "pos\tوبك وطن ينهض سيدي قابوس ❤ #صباح_نوفمبري ⁦🇴🇲⁩\n",
      "\n",
      "----------------------\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(all_tweets[i])\n",
    "    print('----------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "before: pos 29849 neg 28902\n"
     ]
    }
   ],
   "source": [
    "pos_file = 'arabic_tweets_tsv/20190413/Arabic_tweets_positive_20190413.tsv'\n",
    "neg_file = 'arabic_tweets_tsv/20190413/Arabic_tweets_negative_20190413.tsv'\n",
    "pos_data, pos_labels = read_tsv(pos_file)\n",
    "neg_data, neg_labels = read_tsv(neg_file)\n",
    "print('before:', 'pos', len(pos_data), 'neg', len(neg_data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after: pos 29460 neg 27037 mix 2254\n"
     ]
    }
   ],
   "source": [
    "all_tweets = label_mix(pos_data, neg_data)\n",
    "outfile_name = 'arabic_tweets_tsv/20190413/3labels/Arabic_tweets_20190413.tsv'\n",
    "write_tsv(outfile_name, all_tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "neg\t💥 قرب شهر الخير 🌙 وصار لازمها عرووض\n",
      "\n",
      "----------------------\n",
      "neg\t#الاهلي_الهلال -تحت هذي التغريده تحدي روقان واسرع اذا فاز الهلال ان شاء الله ، اسرع راح يوزع قيمة ايفون X …\n",
      "\n",
      "----------------------\n",
      "pos\tصباح الخيرات يارب العالمين والصلاة على محمد وال محمد 🌷 💕 🌷\n",
      "\n",
      "----------------------\n",
      "mix\t😂😂 فكري بعد ما بغير اسمي 😕 هذا عمر يجيب هالبلاوي 😂😂👊\n",
      "\n",
      "----------------------\n",
      "neg\tذكرتك يوم أمطرت دارك وأنا بعيد وهلت دموعي على الخد عبراتها 💔 #بقلمي #جديد #امطار_الشرقيه\n",
      "\n",
      "----------------------\n"
     ]
    }
   ],
   "source": [
    "for i in range(5):\n",
    "    print(all_tweets[i])\n",
    "    print('----------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}