47-Slur-Words-in-CreateDebate.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyOUqtKrM8YpoJcKmss4KIrT"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["# Setup environment"],"metadata":{"id":"gY5PrH-Rpb4k"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"-j-hgd6KoG2e"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["!pip install scikit-bio"],"metadata":{"id":"lV15KkdJlVxC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Clone `CreateDebateScraper` library from github\n","!git clone https://github.com/utkarsh512/CreateDebateScraper.git\n","%cd CreateDebateScraper/src/nested/"],"metadata":{"id":"Eg2z7O3DoWds"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from   copy                     import deepcopy\n","from   itertools                import accumulate\n","import json\n","from   matplotlib               import pyplot as plt\n","import networkx as nx\n","import nltk\n","import numpy as np\n","import pandas as pd\n","import pickle\n","import re\n","from   scipy                    import stats\n","import textwrap\n","from   thread                   import Comment, Thread\n","from   tqdm                     import tqdm\n","nltk.download('punkt') # For tokenizers\n","import matplotlib\n","from   nltk.tokenize            import TweetTokenizer\n","from   pprint                   import pprint\n","import skbio\n","matplotlib.rcParams.update({'font.size': 18})\n","matplotlib.rcParams[\"figure.figsize\"] = (12, 5)"],"metadata":{"id":"qHsV_pvjoloS"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Helper functions"],"metadata":{"id":"8R8pw2wspp-M"}},{"cell_type":"code","source":["tknz = TweetTokenizer()\n","\n","def clean_text(text):\n","    \"\"\"\n","    Preprocessing text\n","    \"\"\"\n","    text = text.lower()\n","    text = re.sub(r\"http\\S+\", \"\", text)\n","    text = re.sub(r\"www\\S+\", \"\", text)\n","    text = re.sub(\"-\", \" \", text)\n","    text = re.sub(\"\\s+\", \" \", text)\n","    text = re.sub(\"\\u2018\", \"X\", text) \n","    text = re.sub(\"\\u2019\", \"X\", text) \n","    text = re.sub(\"\\'\", \"X\", text) \n","    wordTokens_ = tknz.tokenize(text)\n","    wordTokens = list()\n","    for x in wordTokens_:\n","        x = ''.join([v for v in x if v.isalnum() or v == ' '])\n","        if len(x) > 0 and x != 'X':\n","            x = x.replace('X', '\\'')\n","            wordTokens.append(x)\n","    return wordTokens"],"metadata":{"id":"N8eB1QUdpQZO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["SLUR_WORDS = {\n","  \"jews\": [\n","    \"jews\",\n","    \"oven dodger\",\n","    \"nazi\",\n","    \"dirty jew\",\n","    \"holocaust\",\n","    \"kikesucker\",\n","    \"hook nose\",\n","    \"kike\"\n","  ],\n","  \"homosexual\": [\n","    \"faggots usually\",\n","    \"fucking queer\",\n","    \"the biggest faggot\",\n","    \"dyke\",\n","    \"you fucking faggot\",\n","    \"hate faggots\",\n","    \"queer\",\n","    \"homosexual\",\n","    \"the faggots\",\n","    \"faggot\",\n","    \"faggots usually have\",\n","    \"gay\",\n","    \"faggots\",\n","    \"dykey\",\n","    \"ugly dyke\",\n","    \"faggots like you\",\n","    \"you a fag\",\n","    \"lesbian\",\n","    \"homo\",\n","    \"is a faggot\",\n","    \"like a faggot\",\n","    \"dykes\",\n","    \"faggots like\",\n","    \"faggot if you ever\"\n","  ],\n","  \"women\": [\n","    \"ugly dyke\",\n","    \"woman terrorist\",\n","    \"nigress\",\n","    \"bitch\",\n","    \"slut\",\n","    \"women\",\n","    \"sheeboon\",\n","    \"negress\",\n","    \"mud shark\",\n","    \"women threat\",\n","    \"you a lame bitch\",\n","    \"your a cunt\",\n","    \"white bitch\",\n","    \"niggeress\",\n","    \"hoe\",\n","    \"dykes\",\n","    \"niggress\",\n","    \"sheboon\",\n","    \"feminazi\"\n","  ],\n","  \"blacks\": [\n","    \"pavement ape\",\n","    \"the niggers\",\n","    \"negress\",\n","    \"porch monkey\",\n","    \"that nigger\",\n","    \"this nigger\",\n","    \"sheboon\",\n","    \"all niggers\",\n","    \"eurafrica\",\n","    \"shut up nigger\",\n","    \"picaninny\",\n","    \"african attack\",\n","    \"spearchucker\",\n","    \"how many niggers\",\n","    \"nigger\",\n","    \"africa\",\n","    \"niggers are in my\",\n","    \"dindu nuffin\",\n","    \"stupid nigger\",\n","    \"moolie\",\n","    \"niggers\",\n","    \"bluegum\",\n","    \"nigger ass\",\n","    \"you niggers\",\n","    \"fucking nigger\",\n","    \"nigger music\",\n","    \"niggress\",\n","    \"you a nigger\",\n","    \"many niggers are\",\n","    \"nigress\",\n","    \"blacks\",\n","    \"teenaper\",\n","    \"sheeboon\",\n","    \"dumb nigger\",\n","    \"niggeress\",\n","    \"pickaninny\",\n","    \"nigga\"\n","  ],\n","  \"muslim\": [\n","    \"muslim immigrant\",\n","    \"islam\",\n","    \"mudslime\",\n","    \"mooslem\",\n","    \"muslim refugee\",\n","    \"musslime\",\n","    \"shitlam\",\n","    \"muslim invasion\",\n","    \"moslime\",\n","    \"mooslamic\",\n","    \"muzzie\",\n","    \"allah akbar\",\n","    \"mooslime\",\n","    \"musloid\",\n","    \"mudslimes\",\n","    \"muslim\",\n","    \"muslimes\",\n","    \"moslum\",\n","    \"mussie\",\n","    \"muzrat\",\n","    \"muslim countries\",\n","    \"muzzy\",\n","    \"moslim\",\n","    \"jihadi\",\n","    \"muslim country\",\n","    \"moslem\",\n","    \"muzzrat\",\n","    \"mooslim\"\n","  ],\n","  \"arabs\": [\n","    \"towel head\",\n","    \"goatfucker\",\n","    \"arabs\",\n","    \"goathumper\",\n","    \"raghead\",\n","    \"rag head\",\n","    \"goathumping\",\n","    \"towelhead\",\n","    \"camel jockey\",\n","    \"sandnigger\",\n","    \"camel fucker\",\n","    \"sand nigger\"\n","  ],\n","  \"generic\": [\n","    \"to rape\",\n","    \"raped and\",\n","    \"shithole country\",\n","    \"get raped\",\n","    \"raped\",\n","    \"is a fucking\",\n","    \"shit skin\",\n","    \"raped by\",\n","    \"hate you\",\n","    \"fake empowerment\",\n","    \"abusive women\",\n","    \"fuck you too\",\n","    \"violence\",\n","    \"wit a lame nigga\",\n","    \"they all look\",\n","    \"alllivesmatter\",\n","    \"shithole countries\",\n","    \"fucking hate\",\n","    \"trailer trash\",\n","    \"kill all\",\n","    \"terrorist threat\",\n","    \"harassment\",\n","    \"kill yourself\",\n","    \"shitskin\",\n","    \"okay to be white\",\n","    \"fucking hate you\"\n","  ],\n","  \"white\": [\n","    \"full of white\",\n","    \"white trash\",\n","    \"white devil\",\n","    \"white\",\n","    \"are all white\",\n","    \"white boy\",\n","    \"white ass\",\n","    \"white bitch\",\n","    \"hillbilly\",\n","    \"whigger\",\n","    \"white christian\",\n","    \"white person\",\n","    \"all white\",\n","    \"white nigger\",\n","    \"redneck\",\n","    \"white honky\",\n","    \"wigger\",\n","    \"them white\"\n","  ],\n","  \"economy\": [\n","    \"ghetto\"\n","  ],\n","  \"immigrant\": [\n","    \"illegal immigrants\",\n","    \"immigrant not welcome\",\n","    \"immigrant terror\",\n","    \"mexcrement\",\n","    \"go back to where you come from\",\n","    \"muslim refugee\",\n","    \"illegal aliens\",\n","    \"refugee\",\n","    \"protect from immigrants\",\n","    \"negro\",\n","    \"refugees\",\n","    \"immigrant\",\n","    \"refugee invasion\",\n","    \"go back to where they come from\",\n","    \"refugees impact\",\n","    \"bring ebola\",\n","    \"immigrants\",\n","    \"illegal alien\",\n","    \"immigrant invasion\",\n","    \"bring disease\"\n","  ],\n","  \"mental\": [\n","    \"retard\",\n","    \"mongoloid\",\n","    \"retarded\"\n","  ],\n","  \"asians\": [\n","    \"asians\",\n","    \"ching chong\",\n","    \"chinaman\"\n","  ]\n","}"],"metadata":{"id":"2r5wSgQxqNC6"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Loading CreateDebate dataset"],"metadata":{"id":"oo8Lh02lqc_a"}},{"cell_type":"code","source":["comments = dict()\n","\n","# Topical forums on CreateDebate. We have scraped comments for all of the\n","# following forurm.\n","categories = ['business', 'comedy', 'entertainment', 'health', 'law', 'nsfw',\n","              'politics2', 'religion', 'science', 'shopping', 'sports',\n","              'technology', 'travel', 'world']\n","\n","# However, we will be analyzing comments from selected forum only!\n","# These forum have at least 10k comments each.\n","categories_selected = ['politics2', 'religion', 'world', \n","                       'science', 'law', 'technology']\n","\n","for x in categories_selected:\n","    comments[x] = list()"],"metadata":{"id":"Kt0p5a9WqUtK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading comments from select forums\n","\n","for cat in tqdm(categories_selected):\n","    fp = open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/threads.log', 'rb')\n","\n","    # Get all the `Thread` objects pickled while scraping.\n","    threads = list()\n","    try:\n","        while True:\n","            e = pickle.load(fp)\n","            threads.append(e)\n","    except EOFError:\n","        fp.close()\n","\n","    # While classifying CreateDebate comments, we used comments as per author mode.\n","    # Hence, using the same mode to attach classification score with the comments.\n","    # \n","    # score < 0.5 -> ad hominem comment\n","    #       > 0.5 -> non ad hominem comment\n","    authors = dict()\n","    for thread in threads:\n","        for k, v in thread.comments.items():\n","            try:\n","                authors[v.author].append(v)\n","            except:\n","                authors[v.author] = list()\n","                authors[v.author].append(v)\n","\n","    ctr = 0\n","    # Load the classification score of the comments.\n","    with open('/content/gdrive/MyDrive/DL/CreateDebate/' + cat + '/comments_with_score.log', 'rb') as fp:\n","        cws = pickle.load(fp)\n","    # Attach classification score with the comments.\n","    for author in authors.keys():\n","        for i in range(len(authors[author])):\n","            comment = authors[author][i]\n","            foo = deepcopy(comment.__dict__)\n","            foo['tag'] = cat\n","            foo['score'] = cws[ctr][0]\n","            foo['validation'] = cws[ctr][1][0]\n","            comments[cat].append(foo)\n","            ctr += 1"],"metadata":{"id":"WJMTOtYqsv7V"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Analyzing dataset for slur words across ad hominem score"],"metadata":{"id":"siDsVN1LJkEJ"}},{"cell_type":"code","source":["ah_score = dict()\n","slur_count = dict()\n","slur_count_v2 = dict()\n","\n","# addressing order: \n","#   ah_score: \n","#       key: category -> author\n","#       value: list of score of each comment written\n","#\n","#   slur_count:\n","#       key: category -> slur_group -> author \n","#       value: list of slur count of each comment written\n","#   \n","#   slur_count_v2:\n","#       key: category -> slur_group_1 -> slur_group_2 -> author\n","#       value: list of slur count of each comment written"],"metadata":{"id":"fhVr_wNoujF6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# # Computation is already cached.\n","\n","# for cat in categories_selected:\n","#     ah_score[cat] = dict()\n","#     slur_count[cat] = dict()\n","#     slur_count_v2[cat] = dict()\n","\n","#     for slur_group in SLUR_WORDS.keys():\n","#         slur_count[cat][slur_group] = dict()\n","#         slur_count_v2[cat][slur_group] = dict()\n","#         for slur_group_ in SLUR_WORDS.keys():\n","#             slur_count_v2[cat][slur_group][slur_group_] = dict()\n","\n","#     for comment in tqdm(comments[cat]): \n","#         author = comment['author']\n","\n","#         if author not in ah_score[cat]:\n","#             ah_score[cat][author] = list()\n","#             for slur_group in SLUR_WORDS.keys():\n","#                 slur_count[cat][slur_group][author] = list()\n","#                 for slur_group_ in SLUR_WORDS.keys():\n","#                     slur_count_v2[cat][slur_group][slur_group_][author] = list()\n","\n","#         ah_score[cat][author].append(1 - comment['score'])\n","#         cleaned_comment_text = clean_text(comment['body'])\n","\n","#         for slur_group, slur_words in SLUR_WORDS.items():\n","#             current_count = sum([cleaned_comment_text.count(slur_word) for slur_word in slur_words])\n","#             slur_count[cat][slur_group][author].append(current_count)\n","\n","#         for slur_group_1 in SLUR_WORDS.keys():\n","#             for slur_group_2 in SLUR_WORDS.keys():\n","#                 if slur_group_1 == slur_group_2:\n","#                     continue\n","#                 current_count = sum(\n","#                     [\n","#                         cleaned_comment_text.count(slur_word_1 + ' ' + slur_word_2)\n","#                         for slur_word_1 in SLUR_WORDS[slur_group_1]\n","#                         for slur_word_2 in SLUR_WORDS[slur_group_2]\n","#                     ]\n","#                 )\n","#                 slur_count_v2[cat][slur_group_1][slur_group_2][author].append(current_count)           "],"metadata":{"id":"Dfem4yAU0BkQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# # Caching computation\n","\n","# with open('/content/gdrive/MyDrive/Temp/47-ah-score.pkl', 'wb') as fp:\n","#     pickle.dump(ah_score, fp)\n","\n","# with open('/content/gdrive/MyDrive/Temp/47-slur-count.pkl', 'wb') as fp:\n","#     pickle.dump(slur_count, fp)\n","\n","# with open('/content/gdrive/MyDrive/Temp/47-slur-count-v2.pkl', 'wb') as fp:\n","#     pickle.dump(slur_count_v2, fp)"],"metadata":{"id":"JvLNMYrW8caK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading computation from cache\n","\n","with open('/content/gdrive/MyDrive/Temp/47-ah-score.pkl', 'rb') as fp:\n","    ah_score = pickle.load(fp)\n","\n","with open('/content/gdrive/MyDrive/Temp/47-slur-count.pkl', 'rb') as fp:\n","    slur_count = pickle.load(fp)\n","\n","with open('/content/gdrive/MyDrive/Temp/47-slur-count-v2.pkl', 'rb') as fp:\n","    slur_count_v2 = pickle.load(fp)"],"metadata":{"id":"5hFA9DCc_Cpg"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Testing where v2 are computed correctly!\n","\n","pair_count = 0\n","\n","for category in categories_selected:\n","    for slur_group_1 in SLUR_WORDS.keys():\n","        for slur_group_2 in SLUR_WORDS.keys():\n","            for lst in slur_count_v2[category][slur_group_1][slur_group_2].values():\n","                pair_count += sum(lst)\n","\n","print(pair_count)"],"metadata":{"id":"Sau7uA3RN6QK"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Median ah score per category per author\n","#   key: category -> author\n","#   value: median ah score\n","ah_score_median = dict()\n","\n","for category, author_data in ah_score.items():\n","    ah_score_median[category] = dict()\n","    for author, ah_scores in author_data.items():\n","        ah_score_median[category][author] = np.median(ah_scores)"],"metadata":{"id":"-_2wtIqK3HQj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["comment_count = dict()\n","# key: category -> author\n","# value: number of comments written by author in the given forum\n","\n","for category in categories_selected:\n","    comment_count[category] = dict()\n","\n","    for comment in comments[category]:\n","        author = comment['author']\n","        try:\n","            comment_count[category][author] += 1\n","        except KeyError:\n","            comment_count[category][author] = 1"],"metadata":{"id":"iF9z2yTGvpsV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["IGNORE_POLITICS = True\n","IGNORE_RELIGION = True\n","\n","EXTREME_AH_UPPER = 1\n","EXTREME_AH_LOWER = 0.95\n","\n","MODERATE_AH_UPPER = 0.8\n","MODERATE_AH_LOWER = 0.7\n","\n","LOW_AH_UPPER = 0.6\n","LOW_AH_LOWER = 0.5\n","\n","extreme_ah_group = dict()\n","moderate_ah_group = dict()\n","low_ah_group = dict()\n","# key: category -> slur_group\n","# value: list of slur_count\n","\n","extreme_ah_group_v2 = dict()\n","moderate_ah_group_v2 = dict()\n","low_ah_group_v2 = dict()\n","# key: category -> slur_group_1 -> slur_group_2\n","# value: list of slur_count"],"metadata":{"id":"j2Rr6Rxb9fD_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    extreme_ah_group[category] = dict()\n","    moderate_ah_group[category] = dict()\n","    low_ah_group[category] = dict()\n","\n","    extreme_ah_group_v2[category] = dict()\n","    moderate_ah_group_v2[category] = dict()\n","    low_ah_group_v2[category] = dict()\n","\n","    for slur_group in SLUR_WORDS.keys():\n","        extreme_ah_group[category][slur_group] = list()\n","        moderate_ah_group[category][slur_group] = list()\n","        low_ah_group[category][slur_group] = list()\n","\n","        extreme_ah_group_v2[category][slur_group] = dict()\n","        moderate_ah_group_v2[category][slur_group] = dict()\n","        low_ah_group_v2[category][slur_group] = dict()\n","\n","        for slur_group_ in SLUR_WORDS.keys():\n","            extreme_ah_group_v2[category][slur_group][slur_group_] = list()\n","            moderate_ah_group_v2[category][slur_group][slur_group_] = list()\n","            low_ah_group_v2[category][slur_group][slur_group_] = list()\n","\n","    for author, median_ah_score in ah_score_median[category].items():\n","        n_politics_cnt = 0\n","        n_religion_cnt = 0\n","        try:\n","            n_politics_cnt = comment_count['politics2'][author]\n","        except KeyError:\n","            pass\n","        try:\n","            n_religion_cnt = comment_count['religion'][author]\n","        except KeyError:\n","            pass\n","\n","        if IGNORE_POLITICS and n_politics_cnt > 0:\n","            continue\n","\n","        if IGNORE_RELIGION and n_religion_cnt > 0:\n","            continue\n","\n","        if LOW_AH_LOWER <= median_ah_score and median_ah_score <= LOW_AH_UPPER:\n","            for slur_group in SLUR_WORDS.keys():\n","                low_ah_group[category][slur_group].append(np.sum(slur_count[category][slur_group][author]) / len(ah_score[category][author]))\n","                for slur_group_ in SLUR_WORDS.keys():\n","                    if slur_group == slur_group_:\n","                        continue\n","                    low_ah_group_v2[category][slur_group][slur_group_].append(np.sum(slur_count_v2[category][slur_group][slur_group_][author]) / len(ah_score[category][author]))\n","\n","        elif MODERATE_AH_LOWER <= median_ah_score and median_ah_score <= MODERATE_AH_UPPER:\n","            for slur_group in SLUR_WORDS.keys():\n","                moderate_ah_group[category][slur_group].append(np.sum(slur_count[category][slur_group][author]) / len(ah_score[category][author]))\n","                for slur_group_ in SLUR_WORDS.keys():\n","                    if slur_group == slur_group_:\n","                        continue\n","                    moderate_ah_group_v2[category][slur_group][slur_group_].append(np.sum(slur_count_v2[category][slur_group][slur_group_][author]) / len(ah_score[category][author]))\n","                \n","        elif EXTREME_AH_LOWER <= median_ah_score and median_ah_score <= EXTREME_AH_UPPER:\n","            for slur_group in SLUR_WORDS.keys():\n","                extreme_ah_group[category][slur_group].append(np.sum(slur_count[category][slur_group][author]) / len(ah_score[category][author]))\n","                for slur_group_ in SLUR_WORDS.keys():\n","                    if slur_group == slur_group_:\n","                        continue\n","                    extreme_ah_group_v2[category][slur_group][slur_group_].append(np.sum(slur_count_v2[category][slur_group][slur_group_][author]) / len(ah_score[category][author]))\n","        "],"metadata":{"id":"_0OQRI6E91v4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    for slur_group_1 in SLUR_WORDS.keys():\n","        for slur_group_2 in SLUR_WORDS.keys():\n","            avg_extreme = sum(extreme_ah_group_v2[category][slur_group_1][slur_group_2])\n","            avg_moderate = sum(moderate_ah_group_v2[category][slur_group_1][slur_group_2])\n","            avg_low = sum(low_ah_group_v2[category][slur_group_1][slur_group_2])\n","            print(f'{category:>20} - {slur_group_1:>15} - {slur_group_2:>15} - {[avg_extreme, avg_moderate, avg_low]}')"],"metadata":{"id":"2bRLFrspfs_h"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# How many users considered for each case\n","\n","report = ' ' * 20 + ', '\n","for category in categories_selected:\n","    report += f'{category:>20}, '\n","for slur_group in SLUR_WORDS.keys():\n","    report += '\\n' + f'{slur_group:20}, '\n","    for category in categories_selected:\n","        s = f'{len(extreme_ah_group[category][slur_group])} / {len(moderate_ah_group[category][slur_group])} / {len(low_ah_group[category][slur_group])}'\n","        report += f'{s:>20}, '\n","print(report)"],"metadata":{"id":"7YiNTvPLfv-O"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_category(category, saveas='sample.eps', func=np.average):\n","    x = list(SLUR_WORDS.keys())\n","    y1 = []\n","    y2 = []\n","    y3 = []\n","\n","    for slur_group in x:\n","        y1.append(func(extreme_ah_group[category][slur_group]))\n","        y2.append(func(moderate_ah_group[category][slur_group]))\n","        y3.append(func(low_ah_group[category][slur_group]))\n","\n","    ticks = np.arange(len(x))\n","    width = 0.20\n","\n","    fig, ax = plt.subplots()\n","    fig.set_size_inches(15, 12)\n","    subplot1 = ax.bar(ticks - width, y1, width, label='Extreme', tick_label=x)\n","    subplot2 = ax.bar(ticks, y2, width, label='Moderate', tick_label=x)\n","    subplot3 = ax.bar(ticks + width, y3, width, label='Low', tick_label=x)\n","    ax.set_ylabel('Slur count')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=30, ha='right')\n","    ax.legend()\n","    plt.savefig(saveas)\n","\n","def plot_slurs(slur_group, saveas='sample.eps', func=np.average):\n","    x = categories_selected\n","    y1 = []\n","    y2 = []\n","    y3 = []\n","\n","    for category in x:\n","        y1.append(func(extreme_ah_group[category][slur_group]))\n","        y2.append(func(moderate_ah_group[category][slur_group]))\n","        y3.append(func(low_ah_group[category][slur_group]))\n","\n","    ticks = np.arange(len(x))\n","    width = 0.20\n","\n","    fig, ax = plt.subplots()\n","    subplot1 = ax.bar(ticks - width, y1, width, label='Extreme', tick_label=x)\n","    subplot2 = ax.bar(ticks, y2, width, label='Moderate', tick_label=x)\n","    subplot3 = ax.bar(ticks + width, y3, width, label='Low', tick_label=x)\n","    ax.set_ylabel('Slur count')\n","    ax.set_title(f'Slur count distribution for \\'{slur_group}\\' slurs')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=45, ha='right')\n","    ax.legend()\n","    plt.show()\n","    plt.savefig(saveas, format='eps')\n","\n","def plot_slurs_v2(slur_group_1, slur_group_2, func=np.average):\n","    x = categories_selected\n","    y1 = []\n","    y2 = []\n","    y3 = []\n","\n","    for category in x:\n","        y1.append(func(extreme_ah_group_v2[category][slur_group_1][slur_group_2]))\n","        y2.append(func(moderate_ah_group_v2[category][slur_group_1][slur_group_2]))\n","        y3.append(func(low_ah_group_v2[category][slur_group_1][slur_group_2]))\n","\n","    ticks = np.arange(len(x))\n","    width = 0.20\n","\n","    fig, ax = plt.subplots()\n","    subplot1 = ax.bar(ticks - width, y1, width, label='Extreme', tick_label=x)\n","    subplot2 = ax.bar(ticks, y2, width, label='Moderate', tick_label=x)\n","    subplot3 = ax.bar(ticks + width, y3, width, label='Low', tick_label=x)\n","    ax.set_ylabel('Slur count')\n","    ax.set_title(f'Slur count distribution for \\'{slur_group}\\' slurs')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=45, ha='right')\n","    ax.legend()\n","    plt.show()"],"metadata":{"id":"JI6KBLdR_H2L"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('politics2', '/content/gdrive/MyDrive/Temp/MTPImages/politics_slur_count_2.pdf')"],"metadata":{"id":"RUFrlhPiJ5u_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('religion', '/content/gdrive/MyDrive/Temp/MTPImages/religion_slur_count_2.pdf')"],"metadata":{"id":"FX9gssCSaCaI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('world', '/content/gdrive/MyDrive/Temp/MTPImages/world_slur_count_2.pdf')"],"metadata":{"id":"gxfGA6Vzje73"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('science', '/content/gdrive/MyDrive/Temp/MTPImages/science_slur_count_2.pdf')"],"metadata":{"id":"X50Dci1AjkzN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('law', '/content/gdrive/MyDrive/Temp/MTPImages/law_slur_count_2.pdf')"],"metadata":{"id":"ruOaAfVwjrU9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('technology', '/content/gdrive/MyDrive/Temp/MTPImages/technology_slur_count_2.pdf')"],"metadata":{"id":"GIV2eW7kjzIQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('jews')"],"metadata":{"id":"UCGQXQqrkAMh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('homosexual')"],"metadata":{"id":"uUVMedJTkJIQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('women')"],"metadata":{"id":"sRMPo2rqkPUV"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('blacks')"],"metadata":{"id":"TblGpl09kSdj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('muslim')"],"metadata":{"id":"dP0nYUynkWNI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('arabs')"],"metadata":{"id":"vsU8F2YhkaZ9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('generic')"],"metadata":{"id":"CTsyIw8-kfrd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('white')"],"metadata":{"id":"zhxhtFf5kjvN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('economy')"],"metadata":{"id":"PypYa__SkpS4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('immigrant')"],"metadata":{"id":"qDP0wZR4ktAE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('mental')"],"metadata":{"id":"o_uq-iUQkwaL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_slurs('asians')"],"metadata":{"id":"5W8BiJHlk0_d"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Analyzing dataset for slur words across diversity"],"metadata":{"id":"tFdMHCf1GYcP"}},{"cell_type":"code","source":["user_list = set()\n","\n","for category in categories_selected:\n","    for comment in comments[category]:\n","        user_list.add(comment['author'])\n","\n","user_list = list(user_list)"],"metadata":{"id":"xDYDZfJNJ4f_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(user_list)"],"metadata":{"id":"k_-s6lA-gCV_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["ah_activity_matrix = [[0 for j in range(4)] for i in range(len(user_list))]"],"metadata":{"id":"VGrZSKaPiDIA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for i, user in enumerate(user_list):\n","    for j, category in enumerate(categories_selected):\n","        try:\n","            current_score = ah_score_median[category][user]\n","            if EXTREME_AH_LOWER <= current_score and current_score <= EXTREME_AH_UPPER:\n","                ah_activity_matrix[i][1] += 1\n","            elif MODERATE_AH_LOWER <= current_score and current_score <= MODERATE_AH_UPPER:\n","                ah_activity_matrix[i][2] += 1\n","            elif LOW_AH_LOWER <= current_score and current_score <= LOW_AH_UPPER:\n","                ah_activity_matrix[i][3] += 1\n","            else:\n","                ah_activity_matrix[i][0] += 1\n","        except KeyError:\n","            ah_activity_matrix[i][0] += 1\n","            pass"],"metadata":{"id":"WMDVVkb2iE8B"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["diversity_map = dict()\n","# key: author\n","# value: diversity of the user\n","\n","# Note that we include only those users who are present in the following categories for at least one topical forum:\n","# * EXTREME_AH\n","# * MODERATE_AH\n","# * LOW_AH\n","\n","for i, user in enumerate(user_list):\n","\n","    # Check whether to include this user in the study or not\n","    flag = ah_activity_matrix[i][1] or ah_activity_matrix[i][2] or ah_activity_matrix[i][3]\n","    if not flag:\n","        continue\n","    \n","    # It's okay to include this user in the study\n","    div = skbio.diversity.alpha.shannon(ah_activity_matrix[i]) / 2 # divided by 2 to normalize `div` in [0, 1] range\n","    diversity_map[user] = div"],"metadata":{"id":"ZiCo0ijpdbe8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["len(diversity_map)"],"metadata":{"id":"C4qcA_q9fC6O"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plt.hist(diversity_map.values(), bins=[0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1])\n","plt.xlabel('Normalized Shannon diversity index')\n","plt.ylabel('#Users (in log-scale)')\n","plt.yscale('log')\n","plt.title('Normalized Shannon diversity index for CreateDebate users')"],"metadata":{"id":"r8F2sIB9P1s2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Thresholds to partition these users as \n","# - highly diverse\n","# - moderately diverse\n","\n","HIGH_DIVERSITY_LOWER = 0.6\n","\n","MODERATE_DIVERSITY_LOWER = 0.3\n","MODERATE_DIVERSITY_UPPER = 0.4\n","\n","high_diversity_users = set()\n","moderate_diversity_users = set()"],"metadata":{"id":"6AEgj4D2vCup"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user, d in diversity_map.items():\n","    if d >= HIGH_DIVERSITY_LOWER:\n","        high_diversity_users.add(user)\n","    elif d >= MODERATE_DIVERSITY_LOWER and d <= MODERATE_DIVERSITY_UPPER:\n","        moderate_diversity_users.add(user)"],"metadata":{"id":"wh1y3K9Lkwxi"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(\"Number of highly diverse users: \", len(high_diversity_users))\n","print(\"Number of moderately diverse users: \", len(moderate_diversity_users))"],"metadata":{"id":"1xH5ZAAflg3K"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["high_diversity_slur_count = dict()\n","moderate_diversity_slur_count = dict() \n","# key: category -> slur_group\n","# value: list of slur count for every comment written by users belonging to that group"],"metadata":{"id":"zdNJYbtBlxmS"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    high_diversity_slur_count[category] = dict()\n","    moderate_diversity_slur_count[category] = dict()\n","\n","    for slur_group in SLUR_WORDS.keys():\n","        high_diversity_slur_count[category][slur_group] = list()\n","        moderate_diversity_slur_count[category][slur_group] = list()\n","\n","        for user in high_diversity_users:\n","            try:\n","                high_diversity_slur_count[category][slur_group].extend(slur_count[category][slur_group][user])\n","            except KeyError:\n","                pass\n","        \n","        for user in moderate_diversity_users:\n","            try:\n","                moderate_diversity_slur_count[category][slur_group].extend(slur_count[category][slur_group][user])\n","            except KeyError:\n","                pass"],"metadata":{"id":"IWwLO2WBmrK8"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_category(category, func=np.average):\n","    x = list(SLUR_WORDS.keys())\n","    y1 = []\n","    y2 = []\n","\n","    for slur_group in x:\n","        y1.append(func(high_diversity_slur_count[category][slur_group]))\n","        y2.append(func(moderate_diversity_slur_count[category][slur_group]))\n","\n","    ticks = np.arange(len(x))\n","    width = 0.35\n","\n","    fig, ax = plt.subplots()\n","    subplot1 = ax.bar(ticks - width / 2, y1, width, label='High', tick_label=x)\n","    subplot2 = ax.bar(ticks + width / 2, y2, width, label='Moderate', tick_label=x)\n","    ax.set_ylabel('Slur count')\n","    ax.set_title(f'Slur count distribution for \\'{category}\\' forum')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=45, ha='right')\n","    ax.legend()\n","    plt.show()\n","\n","def plot_slurs(slur_group, func=np.average):\n","    x = categories_selected\n","    y1 = []\n","    y2 = []\n","\n","    for category in x:\n","        y1.append(func(high_diversity_slur_count[category][slur_group]))\n","        y2.append(func(moderate_diversity_slur_count[category][slur_group]))\n","\n","    ticks = np.arange(len(x))\n","    width = 0.35\n","\n","    fig, ax = plt.subplots()\n","    subplot1 = ax.bar(ticks - width / 2, y1, width, label='High', tick_label=x)\n","    subplot2 = ax.bar(ticks + width / 2, y2, width, label='Moderate', tick_label=x)\n","    ax.set_ylabel('Slur count')\n","    ax.set_title(f'Slur count distribution for \\'{slur_group}\\' slurs')\n","    ax.set_xticks(ticks)\n","    ax.set_xticklabels(x, rotation=45, ha='right')\n","    ax.legend()\n","    plt.show()"],"metadata":{"id":"sHG_BDP7no_R"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('politics2')"],"metadata":{"id":"EFM-zFqbo-LU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('religion')"],"metadata":{"id":"JGX2-CIQpjoh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('world')"],"metadata":{"id":"WFY5zr4yptgG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('science')"],"metadata":{"id":"qF2BK1sfqCFo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('law')"],"metadata":{"id":"4EVRUIr_sgAn"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_category('technology')"],"metadata":{"id":"bnxKDYVzs_fs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["categories_selected"],"metadata":{"id":"x-NnLB6upCp-"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["# Temporal variation in diversity of users"],"metadata":{"id":"yGTYluLIf6hb"}},{"cell_type":"code","source":["comments['politics2'][0]"],"metadata":{"id":"z8yWNB2SpjGp"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["diversity = dict()\n","# key: author -> year\n","# value: diversity value\n","\n","ah_score_list = dict()\n","# key: author -> year -> category\n","# value: ah score for comments"],"metadata":{"id":"fC8mDAlfgCmX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user in user_list:\n","    ah_score_list[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        ah_score_list[user][syear] = dict()\n","        for category in categories_selected:\n","            ah_score_list[user][syear][category] = list()"],"metadata":{"id":"sxOzpzPfgp4i"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for category in categories_selected:\n","    for comment in tqdm(comments[category]):\n","        year = comment['time'][:4]\n","        try:\n","            int(year)\n","        except:\n","            # Time is not available for given comment\n","            continue\n","        author = comment['author']\n","        score = 1 - comment['score']\n","        ah_score_list[author][year][category].append(score)"],"metadata":{"id":"6EamsPHEhTWG"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["for user in tqdm(user_list):\n","    if not (user in high_diversity_users or user in moderate_diversity_users):\n","        continue\n","    diversity[user] = dict()\n","    for year in range(2008, 2022):\n","        syear = str(year)\n","        freq = [0, 0, 0, 0]\n","        for category in categories_selected:\n","            try:\n","                median_score = np.median(ah_score_list[user][syear][category])\n","                if median_score >= EXTREME_AH_LOWER and median_score <= EXTREME_AH_UPPER:\n","                    freq[1] += 1\n","                elif median_score >= MODERATE_AH_LOWER and median_score <= MODERATE_AH_UPPER:\n","                    freq[2] += 1\n","                elif median_score >= LOW_AH_LOWER and median_score <= LOW_AH_UPPER:\n","                    freq[3] += 1\n","                else:\n","                    freq[0] += 1\n","            except KeyError:\n","                freq[0] += 1\n","        diversity[user][syear] = skbio.diversity.alpha.shannon(freq) / 2"],"metadata":{"id":"Fj0d2JQ1iHKk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_diversity_user(user):\n","    x = [str(year) for year in range(2008, 2022)]\n","    y = [diversity[user][year] for year in x]\n","    plt.plot(x, y)\n","    plt.xlabel('Year')\n","    plt.ylabel('Diversity')\n","    plt.title(f'Shannon diveristy index for user:{user}')"],"metadata":{"id":"dTOAvhkOlhAW"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["list(moderate_diversity_users)[5:10]"],"metadata":{"id":"JdDowEI-n9ji"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_diversity_user('EmoKillMeNow')"],"metadata":{"id":"ExnUtbl-mheo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_diversity_users():\n","    x = [str(year) for year in range(2008, 2022)]\n","    y1 = [np.average([diversity[user][year] for user in high_diversity_users]) for year in x]\n","    y2 = [np.average([diversity[user][year] for user in moderate_diversity_users]) for year in x]\n","    plt.plot(x, y1, label='High')\n","    plt.plot(x, y2, label='Moderate')\n","    plt.xlabel('Year')\n","    plt.ylabel('Diversity')\n","    plt.legend()\n","    plt.title(f'Avg. Shannon diveristy index for highly and moderately diverse group')"],"metadata":{"id":"GKMGnuu-mkdL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["plot_diversity_users()"],"metadata":{"id":"62Kyaz7RqSD2"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def plot_moderate_diversity_users():\n","    x = [str(year) for year in range(2008, 2022)]\n","    y = [np.average([diversity[user][year] for user in high_diversity_users]) for year in x]\n","    plt.plot(x, y)\n","    plt.xlabel('Year')\n","    plt.ylabel('Diversity')\n","    plt.title(f'Avg. Shannon diveristy index for overall high diversity users')"],"metadata":{"id":"9icd7ze4qU2I"},"execution_count":null,"outputs":[]}]}