57-Polar-AH-Migrators-Linguistic-Analysis.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyOTTm6+K/Y/dmexXqQIuKWP"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Setup"],"metadata":{"id":"iV88ThGwkNiR"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"yOChGj3Cj9_w"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["# Clone `CreateDebateScraper` library from github\n","!git clone https://github.com/utkarsh512/CreateDebateScraper.git\n","%cd CreateDebateScraper/src/nested/"],"metadata":{"id":"RB-d6l5xkPok"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["!pip install transformers"],"metadata":{"id":"MWTswvncka20"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from   copy                     import deepcopy\n","# import cpnet\n","from   itertools                import accumulate\n","import json\n","from   matplotlib               import pyplot as plt\n","import networkx as nx\n","import nltk\n","import numpy as np\n","import pandas as pd\n","import pickle\n","import random\n","import re\n","from   scipy                    import stats\n","import textwrap\n","from   thread                   import Comment, Thread\n","from   tqdm                     import tqdm\n","nltk.download('punkt') # For tokenizers\n","nltk.download('stopwords')\n","import matplotlib\n","from   nltk.tokenize            import TweetTokenizer\n","from   nltk.corpus              import stopwords\n","from   pprint                   import pprint\n","from   transformers             import BertModel, BertTokenizer\n","# import shifterator as sh\n","# import wordcloud\n","# import skbio\n","matplotlib.rcParams.update({'font.size': 18})\n","matplotlib.rcParams[\"figure.figsize\"] = (12, 5)\n","STOP_WORDS = list(stopwords.words('english'))"],"metadata":{"id":"rNDAQtD4kdOl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Custom routine to clean texts scraped from Web.\n","# It removes hyperlinks, punctuation marks (except apostrophe)\n","\n","tknz = TweetTokenizer()\n","\n","def clean_text(text):\n","    \"\"\"\n","    Preprocessing text\n","    \"\"\"\n","    text = text.lower()\n","    text = re.sub(r\"http\\S+\", \"\", text)\n","    text = re.sub(r\"www\\S+\", \"\", text)\n","    text = re.sub(\"-\", \" \", text)\n","    text = re.sub(\"\\s+\", \" \", text)\n","    text = re.sub(\"\\u2018\", \"X\", text) \n","    text = re.sub(\"\\u2019\", \"X\", text) \n","    text = re.sub(\"\\'\", \"X\", text) \n","    wordTokens_ = tknz.tokenize(text)\n","    wordTokens = list()\n","    for x in wordTokens_:\n","        x = ''.join([v for v in x if v.isalnum() or v == ' '])\n","        if len(x) > 0 and x != 'X':\n","            x = x.replace('X', '\\'')\n","            wordTokens.append(x)\n","    return wordTokens"],"metadata":{"id":"XBlo7BlCkjMm"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Pre-processing CreateDeate dataset"],"metadata":{"id":"tiuyAdxrlg-S"}},{"cell_type":"code","source":["comments = dict()\n","\n","# Topical forums on CreateDebate. We have scraped comments for all of the\n","# following forurm.\n","categories = ['business', 'comedy', 'entertainment', 'health', 'law', 'nsfw',\n","              'politics2', 'religion', 'science', 'shopping', 'sports',\n","              'technology', 'travel', 'world']\n","\n","# However, we will be analyzing comments from selected forum only!\n","# These forum have at least 10k comments each.\n","categories_selected = ['politics2', 'religion', 'world', \n","                       'science', 'law', 'technology']\n","\n","for x in categories_selected:\n","    comments[x] = list()"],"metadata":{"id":"PXPNJvofkrXI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading comments from select forums\n","\n","for cat in tqdm(categories_selected):\n","    fp = open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/threads.log', 'rb')\n","\n","    # Get all the `Thread` objects pickled while scraping.\n","    threads = list()\n","    try:\n","        while True:\n","            e = pickle.load(fp)\n","            threads.append(e)\n","    except EOFError:\n","        fp.close()\n","\n","    # While classifying CreateDebate comments, we used comments as per author mode.\n","    # Hence, using the same mode to attach classification score with the comments.\n","    # \n","    # score < 0.5 -> ad hominem comment\n","    #       > 0.5 -> non ad hominem comment\n","    authors = dict()\n","    for thread in threads:\n","        for k, v in thread.comments.items():\n","            try:\n","                authors[v.author].append((v, k))\n","            except:\n","                authors[v.author] = list()\n","                authors[v.author].append((v, k))\n","\n","    ctr = 0\n","    # Load the classification score of the comments.\n","    with open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/comments_with_score.log', 'rb') as fp:\n","        cws = pickle.load(fp)\n","    # Attach classification score with the comments.\n","    for author in authors.keys():\n","        for i in range(len(authors[author])):\n","            comment, cid = authors[author][i]\n","            foo = deepcopy(comment.__dict__)\n","            foo['tag'] = cat\n","            foo['score'] = cws[ctr][0]\n","            foo['validation'] = cws[ctr][1][0]\n","            foo['id'] = int(cid[3:])\n","            comments[cat].append(foo)\n","            ctr += 1"],"metadata":{"id":"TDzMtKxAkuPy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["ah_score_comments = dict()\n","\n","for cat in categories_selected:\n","    for comment in comments[cat]:\n","        ah_score_comments[comment['id']] = 1 - comment['score']"],"metadata":{"id":"DHzKISFmkwGO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def parse_tstring(tstring):\n","    \"\"\"\n","    Parses comment's time to an integer to enable\n","    comparison between comments based on their time of posting\n","    \"\"\"\n","    if tstring == 'Not Available':\n","        raise ValueError('Invalid posting time for parse_tstring')\n","    tstring = tstring.replace('T', '-').replace(':', '-').replace('+', '-').split('-')\n","    return int(''.join(tstring[:-2]))"],"metadata":{"id":"TtqSrT6TlAK3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading AH score\n","\n","with open('/content/gdrive/MyDrive/Temp/47-ah-score.pkl', 'rb') as fp:\n","    ah_score = pickle.load(fp)\n","\n","# `ah_score` is a dictionary that contains the ah score of the comments written\n","# by all the users\n","\n","# key: category -> user\n","# value: list of ah_score for given user for given category\n","\n","# value > 0.5 --> ad hominem\n","# value < 0.5 --> non ad hominem"],"metadata":{"id":"bUmNlz57lCU2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Maximum ah score per category per author\n","#   key: category -> author\n","#   value: maximum ah score\n","\n","ah_score_max = dict()\n","\n","for category, author_data in ah_score.items():\n","    ah_score_max[category] = dict()\n","    for author, ah_scores in author_data.items():\n","        ah_score_max[category][author] = np.max(ah_scores)"],"metadata":{"id":"gOAn9qFUlEc_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["comment_count = dict()\n","# key: category -> author\n","# value: number of comments written by author in the given forum\n","\n","for category in categories_selected:\n","    comment_count[category] = dict()\n","\n","    for comment in comments[category]:\n","        author = comment['author']\n","        try:\n","            comment_count[category][author] += 1\n","        except KeyError:\n","            comment_count[category][author] = 1"],"metadata":{"id":"eznaQaL2lJYG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["user_list = set()\n","\n","for category in categories_selected:\n","    for comment in comments[category]:\n","        user_list.add(comment['author'])\n","\n","user_list = list(user_list)"],"metadata":{"id":"wOQ6CSOwlLZx"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["first_post_time = dict()\n","# key: category -> user\n","# value: post time of the first comment by given user in the given category\n","#        It is an integer as returned by parse_tstring routine\n","\n","for category in categories_selected:\n","    first_post_time[category] = dict()\n","\n","    for comment in comments[category]: \n","        if comment['time'] == 'Not Available':\n","            continue\n","        author = comment['author']\n","        try:\n","            first_post_time[category][author] = min(first_post_time[category][author], parse_tstring(comment['time']))\n","        except KeyError:\n","            first_post_time[category][author] = parse_tstring(comment['time'])"],"metadata":{"id":"Y-OnH5zylNae"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_migrated_users(categories_1, categories_2, categories_1_origin=True, require_migration=True):\n","    \"\"\"\n","    Returns a list of usernames who migrated from categories_1 to categories_2\n","\n","    If categories_1_origin is True, we will consider all other major categories\n","    to compute post_time_2, so as to ensure that first post by the user is in \n","    categories_1\n","\n","    If require_migration is True, post_time_1 < post_time_2 condition is relaxed\n","    \"\"\"\n","\n","    resultant_list = list()\n","\n","    for user in user_list:\n","        post_time_1 = 20220101000000\n","        post_time_2 = 20220101000000\n","\n","        if not isinstance(categories_1, set):\n","            categories_1 = set(categories_1)\n","        if not isinstance(categories_2, set): \n","            categories_2 = set(categories_2)\n","        \n","        for category in categories_1:\n","            try:\n","                cur_post_time = first_post_time[category][user]\n","                post_time_1 = min(post_time_1, cur_post_time)\n","            except KeyError:\n","                pass\n","        \n","        for category in categories_2:\n","            try:\n","                cur_post_time = first_post_time[category][user]\n","                post_time_2 = min(post_time_2, cur_post_time) \n","            except KeyError:\n","                pass\n","\n","        if post_time_1 == 20220101000000 or post_time_2 == 20220101000000:\n","            continue\n","\n","        if categories_1_origin:\n","            for category in categories_selected:\n","                if not ((category in categories_1) or (category in categories_2)):\n","                    try:\n","                        cur_post_time = first_post_time[category][user]\n","                        post_time_2 = min(post_time_2, cur_post_time)\n","                    except KeyError:\n","                        pass\n","\n","        if post_time_1 < post_time_2 or not require_migration:\n","            resultant_list.append(user)\n","        \n","    return resultant_list"],"metadata":{"id":"u-J09ydqlQCp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def partition_migrated_users(migration_list, categories_1, categories_2):\n","    \"\"\"\n","    Partitions the users into 4 categories: \n","        AH-AH\n","        AH-NonAH\n","        NonAH-AH\n","        NonAH-NonAH\n","\n","    Users are classified as AH in a given category if they post at least one \n","    ad hominem comment in that category\n","    \n","    Note: migration_list should be obtained using get_migrated_users method\n","    \"\"\"\n","\n","    ah_ah_list = []\n","    ah_nonah_list = []\n","    nonah_ah_list = []\n","    nonah_nonah_list = []\n","\n","    for user in migration_list:\n","        max_score_1 = 0\n","        max_score_2 = 0\n","        for category in categories_1:\n","            max_score_1 = max(max_score_1, ah_score_max[category].get(user, 0))\n","        for category in categories_2:\n","            max_score_2 = max(max_score_2, ah_score_max[category].get(user, 0))\n","\n","        if max_score_1 > 0.5 and max_score_2 > 0.5:\n","            ah_ah_list.append(user)\n","\n","        elif max_score_1 > 0.5 and max_score_2 < 0.5:\n","            ah_nonah_list.append(user)\n","        \n","        elif max_score_1 < 0.5 and max_score_2 > 0.5:\n","            nonah_ah_list.append(user)\n","\n","        elif max_score_1 < 0.5 and max_score_2 < 0.5:\n","            nonah_nonah_list.append(user)\n","        \n","        else:\n","            print(user)\n","\n","    return ah_ah_list, ah_nonah_list, nonah_ah_list, nonah_nonah_list "],"metadata":{"id":"NX0bw1rhlTOc"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Attention Visualizer"],"metadata":{"id":"oNylBrQMl5lN"}},{"cell_type":"code","source":["class Visualizer:\n","    \"\"\"Wrapper for creating heatmaps for documents\"\"\"\n","    def __init__(self):\n","        self._header = r'''\\documentclass[10pt,a4paper]{article}\n","\\usepackage[left=1.00cm, right=1.00cm, top=1.00cm, bottom=2.00cm]{geometry}\n","\\usepackage{color}\n","\\usepackage{tcolorbox}\n","\\usepackage{CJK}\n","\\usepackage{adjustbox}\n","\\tcbset{width=0.9\\textwidth,boxrule=0pt,colback=red,arc=0pt,auto outer arc,left=0pt,right=0pt,boxsep=5pt}\n","\\begin{document}\n","\\begin{CJK*}{UTF8}{gbsn}''' + '\\n\\n'\n","\n","        self._footer = r'''\\end{CJK*}\n","\\end{document}'''\n","\n","    def visualize(self,\n","                  word_list,\n","                  attention_list,\n","                  label_list,\n","                  latex_file,\n","                  title,\n","                  batch_size=20,\n","                  color='blue'):\n","        \"\"\"Routine to generate attention heatmaps for given texts\n","        ---------------------------------------------------------\n","        Input:\n","        :param word_list: list of texts (each text is a list of words)\n","        :param attention_list: scores for each word, dimension same as word_list\n","        :param label_list: label for each text\n","        :param latex_file: name of the latex file\n","        :param title: title of latex file\n","        :param batch_size: Number of comments in each batch\n","        :param color: color used for visualization, can be 'blue', 'red', 'green', etc.\n","        \"\"\"\n","        word_list_processed = []\n","        for x in word_list:\n","            word_list_processed.append(self._clean_word(x))\n","\n","        with open(latex_file, 'w', encoding='utf-8') as f:\n","            f.write(self._header)\n","            f.write('\\\\section{%s}\\n\\n' % title)\n","\n","            n_examples = len(word_list)\n","            n_batches = n_examples // batch_size\n","\n","            for i in range(n_batches):\n","                batch_word_list = word_list_processed[i * batch_size: (i + 1) * batch_size]\n","                batch_attention_list = attention_list[i * batch_size: (i + 1) * batch_size]\n","                batch_label_list = label_list[i * batch_size: (i + 1) * batch_size]\n","                f.write('\\\\subsection{Batch %d}\\n\\n' % (i + 1))\n","                for j in range(batch_size):\n","                    f.write('\\\\subsubsection{Comment %d - %s}\\n\\n' % (j + 1, batch_label_list[j]))\n","                    sentence = batch_word_list[j]\n","                    score = batch_attention_list[j]\n","                    assert len(sentence) == len(score)\n","                    f.write('\\\\noindent')\n","                    for k in range(len(sentence)):\n","                        f.write('\\\\colorbox{%s!%s}{' % (color, score[k]) + '\\\\strut ' + sentence[k] + '} ')\n","                    f.write('\\n\\n')\n","\n","            f.write(self._footer)\n","\n","    @staticmethod\n","    def _clean_word(word_list):\n","        new_word_list = []\n","        for word in word_list:\n","            for latex_sensitive in [\"\\\\\", \"%\", \"&\", \"^\", \"#\", \"_\", \"{\", \"}\"]:\n","                if latex_sensitive in word:\n","                    word = word.replace(latex_sensitive, '\\\\' + latex_sensitive)\n","            new_word_list.append(word)\n","        return new_word_list\n","\n","viz = Visualizer()"],"metadata":{"id":"h7Q-Ork1lWb1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["model_version = '/content/gdrive/MyDrive/DL/cnerg-bert-adhominem'\n","do_lower_case = True\n","model = BertModel.from_pretrained(model_version, output_attentions=True)\n","tokenizer = BertTokenizer.from_pretrained(model_version, do_lower_case=do_lower_case)"],"metadata":{"id":"JVaWY5JdmElT"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["INTENSITY = 70\n","\n","def attention_scores(text, layers=None, heads=None):\n","    sentence_a = text\n","    inputs = tokenizer.encode_plus(sentence_a, None, return_tensors='pt', add_special_tokens=True)\n","    input_ids = inputs['input_ids']\n","    attention = model(input_ids)[-1]\n","    input_id_list = input_ids[0].tolist() # Batch index 0\n","    tokens = tokenizer.convert_ids_to_tokens(input_id_list) \n","    sz = len(tokens)\n","    matrix = [0 for j in range(sz)]\n","    if layers is None:\n","        layers = [x for x in range(12)]\n","    if heads is None:\n","        heads = [x for x in range(12)]\n","    for layer in layers:\n","        for head in heads:\n","            for j in range(sz):\n","                matrix[j] += attention[layer][0, head, 0, j].item()\n","    for j in range(sz):\n","        matrix[j] = (matrix[j]) / (len(layers) * len(heads))\n","    return (tokens, matrix)"],"metadata":{"id":"2QnnqlXDmISK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def clean_array(w, a):\n","    W = []\n","    A = []\n","    for i in range(len(w)):\n","        if (w[i].startswith('##')):\n","            W[len(W) - 1] += w[i][2:]\n","            A[len(A) - 1] = (A[len(A) - 1] + a[i]) / 2\n","        else:\n","            W.append(w[i])\n","            A.append(a[i])\n","    return clean_apos(W, A)\n","\n","def clean_apos(w, a):\n","    W = []\n","    A = []\n","    ctr = 0\n","    while ctr != len(w):\n","        if w[ctr] == '\\'':\n","            W[-1] += w[ctr] + w[ctr + 1]\n","            A[-1] = min(INTENSITY, A[-1] + a[ctr] + a[ctr + 1])\n","            ctr += 2\n","        else:\n","            W.append(w[ctr])\n","            A.append(a[ctr])\n","            ctr += 1\n","    return W, A"],"metadata":{"id":"khwmcTpNmU26"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def top_three_tokens(text):\n","    words, attentions = attention_scores(text)\n","    words = words[1:-1] # Remove start and end tags\n","    attentions = attentions[1:-1]\n","    assert len(words) == len(attentions)\n","    words, attentions = clean_array(words, attentions)\n","    assert len(words) == len(attentions)\n","    top_tokens = list()\n","    for i in range(len(words)):\n","        top_tokens.append((attentions[i], i))\n","    top_tokens = sorted(top_tokens, reverse=True)\n","    ind = [0]\n","    cur = 1\n","    while len(ind) < 3:\n","        take = True\n","        for ids in ind:\n","            take = take and abs(top_tokens[ids][1] - top_tokens[cur][1]) > 2\n","        if take:\n","            ind.append(cur)\n","        cur += 1\n","    xx = []\n","    for x in ind:\n","        xx.append(top_tokens[x][1])\n","    scores = [0 for i in range(len(words))]\n","    for w in xx:\n","        lst = [w - 1, w, w + 1]\n","        for j in lst:\n","            if j >= 0 and j < len(words):\n","                scores[j] = INTENSITY\n","    return words, scores"],"metadata":{"id":"RaeH-5AOmY7m"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def create_latex_file(texts):\n","    words_list = list()\n","    scores_list = list()\n","\n","    for comment in texts:\n","        try:\n","            words, scores = top_three_tokens(comment)\n","        except:\n","            continue\n","        words_list.append(words)\n","        scores_list.append(scores)\n","    \n","    label = ' '\n","    labels_list = [label for _ in range(len(words_list))]\n","    \n","    viz.visualize(words_list, scores_list, labels_list,\n","                  latex_file='sample.tex',\n","                  title=f'Top three triggers',\n","                  batch_size=len(words_list),\n","                  color='cyan')"],"metadata":{"id":"wF3FVTi5vTTx"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Slur Words"],"metadata":{"id":"swzI9KGUm3_l"}},{"cell_type":"code","source":["SLUR_WORDS = {\n","  \"jews\": [\n","    \"jews\",\n","    \"oven dodger\",\n","    \"nazi\",\n","    \"dirty jew\",\n","    \"holocaust\",\n","    \"kikesucker\",\n","    \"hook nose\",\n","    \"kike\"\n","  ],\n","  \"homosexual\": [\n","    \"faggots usually\",\n","    \"fucking queer\",\n","    \"the biggest faggot\",\n","    \"dyke\",\n","    \"you fucking faggot\",\n","    \"hate faggots\",\n","    \"queer\",\n","    \"homosexual\",\n","    \"the faggots\",\n","    \"faggot\",\n","    \"faggots usually have\",\n","    \"gay\",\n","    \"faggots\",\n","    \"dykey\",\n","    \"ugly dyke\",\n","    \"faggots like you\",\n","    \"you a fag\",\n","    \"lesbian\",\n","    \"homo\",\n","    \"is a faggot\",\n","    \"like a faggot\",\n","    \"dykes\",\n","    \"faggots like\",\n","    \"faggot if you ever\"\n","  ],\n","  \"women\": [\n","    \"ugly dyke\",\n","    \"woman terrorist\",\n","    \"nigress\",\n","    \"bitch\",\n","    \"slut\",\n","    \"women\",\n","    \"sheeboon\",\n","    \"negress\",\n","    \"mud shark\",\n","    \"women threat\",\n","    \"you a lame bitch\",\n","    \"your a cunt\",\n","    \"white bitch\",\n","    \"niggeress\",\n","    \"hoe\",\n","    \"dykes\",\n","    \"niggress\",\n","    \"sheboon\",\n","    \"feminazi\"\n","  ],\n","  \"blacks\": [\n","    \"pavement ape\",\n","    \"the niggers\",\n","    \"negress\",\n","    \"porch monkey\",\n","    \"that nigger\",\n","    \"this nigger\",\n","    \"sheboon\",\n","    \"all niggers\",\n","    \"eurafrica\",\n","    \"shut up nigger\",\n","    \"picaninny\",\n","    \"african attack\",\n","    \"spearchucker\",\n","    \"how many niggers\",\n","    \"nigger\",\n","    \"africa\",\n","    \"niggers are in my\",\n","    \"dindu nuffin\",\n","    \"stupid nigger\",\n","    \"moolie\",\n","    \"niggers\",\n","    \"bluegum\",\n","    \"nigger ass\",\n","    \"you niggers\",\n","    \"fucking nigger\",\n","    \"nigger music\",\n","    \"niggress\",\n","    \"you a nigger\",\n","    \"many niggers are\",\n","    \"nigress\",\n","    \"blacks\",\n","    \"teenaper\",\n","    \"sheeboon\",\n","    \"dumb nigger\",\n","    \"niggeress\",\n","    \"pickaninny\",\n","    \"nigga\"\n","  ],\n","  \"muslim\": [\n","    \"muslim immigrant\",\n","    \"islam\",\n","    \"mudslime\",\n","    \"mooslem\",\n","    \"muslim refugee\",\n","    \"musslime\",\n","    \"shitlam\",\n","    \"muslim invasion\",\n","    \"moslime\",\n","    \"mooslamic\",\n","    \"muzzie\",\n","    \"allah akbar\",\n","    \"mooslime\",\n","    \"musloid\",\n","    \"mudslimes\",\n","    \"muslim\",\n","    \"muslimes\",\n","    \"moslum\",\n","    \"mussie\",\n","    \"muzrat\",\n","    \"muslim countries\",\n","    \"muzzy\",\n","    \"moslim\",\n","    \"jihadi\",\n","    \"muslim country\",\n","    \"moslem\",\n","    \"muzzrat\",\n","    \"mooslim\"\n","  ],\n","  \"arabs\": [\n","    \"towel head\",\n","    \"goatfucker\",\n","    \"arabs\",\n","    \"goathumper\",\n","    \"raghead\",\n","    \"rag head\",\n","    \"goathumping\",\n","    \"towelhead\",\n","    \"camel jockey\",\n","    \"sandnigger\",\n","    \"camel fucker\",\n","    \"sand nigger\"\n","  ],\n","  \"generic\": [\n","    \"to rape\",\n","    \"raped and\",\n","    \"shithole country\",\n","    \"get raped\",\n","    \"raped\",\n","    \"is a fucking\",\n","    \"shit skin\",\n","    \"raped by\",\n","    \"hate you\",\n","    \"fake empowerment\",\n","    \"abusive women\",\n","    \"fuck you too\",\n","    \"violence\",\n","    \"wit a lame nigga\",\n","    \"they all look\",\n","    \"alllivesmatter\",\n","    \"shithole countries\",\n","    \"fucking hate\",\n","    \"trailer trash\",\n","    \"kill all\",\n","    \"terrorist threat\",\n","    \"harassment\",\n","    \"kill yourself\",\n","    \"shitskin\",\n","    \"okay to be white\",\n","    \"fucking hate you\"\n","  ],\n","  \"white\": [\n","    \"full of white\",\n","    \"white trash\",\n","    \"white devil\",\n","    \"white\",\n","    \"are all white\",\n","    \"white boy\",\n","    \"white ass\",\n","    \"white bitch\",\n","    \"hillbilly\",\n","    \"whigger\",\n","    \"white christian\",\n","    \"white person\",\n","    \"all white\",\n","    \"white nigger\",\n","    \"redneck\",\n","    \"white honky\",\n","    \"wigger\",\n","    \"them white\"\n","  ],\n","  \"economy\": [\n","    \"ghetto\"\n","  ],\n","  \"immigrant\": [\n","    \"illegal immigrants\",\n","    \"immigrant not welcome\",\n","    \"immigrant terror\",\n","    \"mexcrement\",\n","    \"go back to where you come from\",\n","    \"muslim refugee\",\n","    \"illegal aliens\",\n","    \"refugee\",\n","    \"protect from immigrants\",\n","    \"negro\",\n","    \"refugees\",\n","    \"immigrant\",\n","    \"refugee invasion\",\n","    \"go back to where they come from\",\n","    \"refugees impact\",\n","    \"bring ebola\",\n","    \"immigrants\",\n","    \"illegal alien\",\n","    \"immigrant invasion\",\n","    \"bring disease\"\n","  ],\n","  \"mental\": [\n","    \"retard\",\n","    \"mongoloid\",\n","    \"retarded\"\n","  ],\n","  \"asians\": [\n","    \"asians\",\n","    \"ching chong\",\n","    \"chinaman\"\n","  ]\n","}"],"metadata":{"id":"OPMwDVlcmhgC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def create_corpus_from_comments(comment_list):\n","    corpus = list()\n","    for comment in comment_list:\n","        corpus.append(' '.join(clean_text(comment['body'])))\n","    return ' '.join(corpus)"],"metadata":{"id":"lQpRS8FVqAKN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_slur_prob(corpus):\n","    \"\"\"\"corpus must be generated using create_corpus_from_comments function\"\"\"\n","    token_count = len(corpus.split())\n","    slur_prob   = dict()\n","\n","    for slur_group in SLUR_WORDS.keys(): \n","        slur_prob[slur_group] = 0  \n","        \n","        for slur_words in SLUR_WORDS[slur_group]: \n","            slur_prob[slur_group] += corpus.count(slur_words) * len(slur_words.split())\n","        \n","        slur_prob[slur_group] /= token_count\n","    \n","    return slur_prob"],"metadata":{"id":"NIojuSpJrD8f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def compare_slur_probs(slur_prob_1, slur_prob_2, label1, label2):\n","    \"\"\"Comparing two slur probabilities by plotting them\n","    \"\"\"\n","    x = list(SLUR_WORDS.keys())\n","    y1 = [] \n","    y2 = [] \n","\n","    for slur_group in x:\n","        y1.append(slur_prob_1[slur_group])\n","        y2.append(slur_prob_2[slur_group])\n","\n","    ticks = np.arange(len(x))\n","    width = 0.30\n","\n","    fig, ax = plt.subplots()\n","    subplot1 = ax.bar(ticks - width / 2, y1, width, label=label1, tick_label=x)\n","    subplot2 = ax.bar(ticks + width / 2, y2, width, label=label2, tick_label=x)\n","\n","    ax.set_ylabel('Probability')\n","    ax.set_title(f'Probability of being a slur word')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=45, ha='right')\n","    ax.legend()\n","    plt.show()"],"metadata":{"id":"6pf8vQRCsQiN"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Analysis"],"metadata":{"id":"C2Incf-BnvtM"}},{"cell_type":"code","source":["categories_1 = ['religion']\n","categories_2 = ['politics2']"],"metadata":{"id":"CTwGbCXgm20M"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["migration_list = get_migrated_users(categories_1, categories_2, categories_1_origin=True, require_migration=True)"],"metadata":{"id":"kdMsJLx6n4uv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(migration_list)"],"metadata":{"id":"Y_W_eKTdn7QW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["AA, AN, NA, NN = partition_migrated_users(migration_list, categories_1, categories_2)"],"metadata":{"id":"MSOZQ9p-n9Ja"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["AAset = set(AA)\n","ANset = set(AN)\n","NAset = set(NA) \n","NNset = set(NN)"],"metadata":{"id":"OybHBgsPoBCe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def partition_comments(user_subset): # user_subset can be AAset, ANset, ..., \n","    categories_1_comments = list()\n","    categories_2_comments = list()\n","\n","    for cat in categories_1:\n","        for comment in comments[cat]:\n","            if comment['author'] in user_subset: \n","                categories_1_comments.append(deepcopy(comment))\n","    \n","    for cat in categories_2:\n","        for comment in comments[cat]:\n","            if comment['author'] in user_subset: \n","                categories_2_comments.append(deepcopy(comment))\n","\n","    return categories_1_comments, categories_2_comments"],"metadata":{"id":"aKDxxOBOoHHt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def get_random_comments(comment_list, n=100):\n","    n = min(n, len(comment_list)) \n","    samples = random.sample(comment_list, n) \n","    texts = list()\n","    for comment in samples:\n","        texts.append(' '.join(clean_text(comment['body']))) \n","    return texts"],"metadata":{"id":"a6iIx0xTuIyB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["cat1c, cat2c = partition_comments(NNset)"],"metadata":{"id":"y0BmybTMpXkp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(f'{len(cat1c)}, {len(cat2c)}')"],"metadata":{"id":"aRJdd62WpcTc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["corpus_1 = create_corpus_from_comments(cat1c)\n","corpus_2 = create_corpus_from_comments(cat2c)"],"metadata":{"id":"6OPO3toJpk6f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["slur_prob_1 = get_slur_prob(corpus_1) \n","slur_prob_2 = get_slur_prob(corpus_2)"],"metadata":{"id":"uaGdl3mxqdzL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["compare_slur_probs(slur_prob_1, slur_prob_2, str(categories_1), str(categories_2))"],"metadata":{"id":"Zv3YOoA9tJUY"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["random_comments_1 = get_random_comments(cat1c)\n","random_comments_2 = get_random_comments(cat2c)"],"metadata":{"id":"oHmU5hOAtjBI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["create_latex_file(random_comments_1)"],"metadata":{"id":"onhmUt3JwEvV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["create_latex_file(random_comments_2)"],"metadata":{"id":"2X2HFEQOyslp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"7Vkk6oTBzneP"},"execution_count":null,"outputs":[]}]}