40-Analyzing-Million-Tweet-Replies-NYTimes.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"40-Analyzing-Million-Tweet-Replies-NYTimes.ipynb","private_outputs":true,"provenance":[],"authorship_tag":"ABX9TyPevvWJudmSQMqPSTjLOoYa"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["__Objective__: To perform data analysis on Million+ tweet replies of NYTimes\n","\n","__Runtime__: CPU"],"metadata":{"id":"nQ-b5axrdGec"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"DlMA1HEvc6_6"},"outputs":[],"source":["from google.colab import drive\n","drive.mount('/content/gdrive', force_remount=True)"]},{"cell_type":"code","source":["from tqdm import tqdm\n","import pickle\n","import pandas as pd \n","import numpy as np "],"metadata":{"id":"x37tNxHMdTbl"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# loading tweets\n","\n","before_df = pd.read_json('/content/gdrive/MyDrive/DL/Twitter/million/nytimes/classified/nytimes_before.json', lines=True)\n","after_df = pd.read_json('/content/gdrive/MyDrive/DL/Twitter/million/nytimes/classified/nytimes_after.json', lines=True)"],"metadata":{"id":"K7Rv2Hc1dhJB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df = pd.concat([before_df, after_df])"],"metadata":{"id":"9W3nU4i4d3ZA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df"],"metadata":{"id":"2ap2TC9ZeLI7"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df.columns"],"metadata":{"id":"3TQLhD3xeiOa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['score_'] = merged_df.apply(lambda x: x['score'] if x['label'] == 'NONE' else 1 - x['score'], axis=1)"],"metadata":{"id":"e87QLwDRXXrh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['score_augmented_'] = merged_df.apply(lambda x: x['score_augmented'] if x['label_augmented'] == 'NONE' else 1 - x['score_augmented'], axis=1)"],"metadata":{"id":"VsD3FYuqXxo0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df"],"metadata":{"id":"B6V7YhVtYAGC"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['score_'].hist(bins=[x/100 for x in range(101)])"],"metadata":{"id":"3usZzlxZj1ze"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['score_augmented_'].hist(bins=[x/100 for x in range(101)])"],"metadata":{"id":"9RhFJGF2k7wo"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['label'].describe()"],"metadata":{"id":"F8j-bPbWn1AI"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["(993754 - 762781) / 993754 * 100"],"metadata":{"id":"0KVXujUWaxzt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["merged_df['label_augmented'].describe()"],"metadata":{"id":"KitK1J0Ha8hq"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["(993754 - 754533) / 993754 * 100"],"metadata":{"id":"c9SWWnLrbAL0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[""],"metadata":{"id":"RHKXA3srbHPI"},"execution_count":null,"outputs":[]}]}