Data_Exploration.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"collapsed_sections":["vE1rON7FugVv"],"mount_file_id":"1UyrslC0nBgAuxxgJUQVRl3jiHzO1HMrW","authorship_tag":"ABX9TyO9ZBKjokq/y3GJn5WHAKZW"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["### CMU Summaries"],"metadata":{"id":"vE1rON7FugVv"}},{"cell_type":"code","execution_count":null,"metadata":{"id":"c60skQNyQ_GX"},"outputs":[],"source":["# Extracting all the files\n","import tarfile\n","my_tar = tarfile.open('/content/drive/MyDrive/Visual Story Telling/Dataset - Story Generation/MovieSummaries.tar.gz')\n","my_tar.extractall('./CMU_Summaries')\n","my_tar.close()"]},{"cell_type":"code","source":["# Readme file for explanation of the dataset\n","with open('/content/CMU_Summaries/MovieSummaries/README.txt','r') as f:\n","  readme_data = f.read()\n","# print(readme_data)"],"metadata":{"id":"TcymvbAwv0t5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading the movie metadata\n","import pandas as pd\n","df_movie_meta = pd.read_table('/content/CMU_Summaries/MovieSummaries/movie.metadata.tsv', header=None, names = ['Wikipedia movie ID','Freebase movie ID','Movie name','Movie release date','Movie box office revenue','Movie runtime','Movie language','Movie countries','Movie genres'])"],"metadata":{"id":"oJFvapk3u5LE"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_movie_meta"],"metadata":{"id":"YBGxLdagu7jU"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading the character metadata\n","df_charac_meta = pd.read_table('/content/CMU_Summaries/MovieSummaries/character.metadata.tsv',header=None, names = ['Wikipedia movie ID', 'Freebase movie ID','Movie release date','Character name','Actor date of birth','Actor gender','Actor height','Actor ethnicity','Actor name','Actor age at movie release','Freebase character/actor map ID','Freebase character ID','Freebase actor ID'])"],"metadata":{"id":"OgaS110TvegQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_charac_meta"],"metadata":{"id":"RRhZQs7lxyPQ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Reading the summaries from a text file\n","# Can directly load the summaries into a dataframe - done in CMU Books dataset\n","with open('/content/CMU_Summaries/MovieSummaries/plot_summaries.txt','r') as f:\n","  summaries = f.read()"],"metadata":{"id":"fgzRFS8MuZql"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Splitting different summaries\n","summaries = summaries.split('\\n')"],"metadata":{"id":"D7--yfOoxzjc"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["summaries"],"metadata":{"id":"sx4Lc9IZykd0"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading summaries in a dataframe\n","df_summaries = pd.DataFrame(columns=['Wikipedia movie ID', 'Summary'])"],"metadata":{"id":"-rPYJVQbylTa"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries['Summary'] = summaries"],"metadata":{"id":"XZYsKkrBy445"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Functions to separate movie ID from the summary\n","def sep_id(x):\n","  return x.split('\\t')[0]\n","def sep_summary(x):\n","  try:\n","    return x.split('\\t')[1]\n","  except:\n","    return None"],"metadata":{"id":"3wWvW9YXzSpk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries['Wikipedia movie ID']  = df_summaries['Summary'].apply(sep_id)\n","df_summaries['Summary'] = df_summaries['Summary'].apply(sep_summary)"],"metadata":{"id":"Cs0rf0F7zUPj"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries.dropna(inplace=True)"],"metadata":{"id":"reGBOUa23IKz"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries.reset_index(inplace=True, drop=True)"],"metadata":{"id":"e-K2xAOQ4Fq3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Adding additional column to store different characters of the story\n","df_summaries = df_summaries.assign(Genre=\"\", Movie_Name=\"\",Characters=\"\")"],"metadata":{"id":"HpAVXlpHS3-P"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries"],"metadata":{"id":"uOlBuB0iUJpt"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Extarcting the movie genre and its title \n","labels = []\n","for idx, row in df_summaries.iterrows():\n","  # for idx2, row2 in df_movie_meta.iterrows():\n","    #Get index and then assign values, avoid the for loop and if condition (do the same for characters)\n","    # if row2['Wikipedia movie ID'] == row['Wikipedia movie ID']:\n","      # df_summaries['Movie_Name'].iloc[idx] = row2['Movie name']\n","      # df_summaries['Genre'].iloc[idx] = row2['Movie genres']\n","      # break\n","  try: \n","    index = df_movie_meta['Wikipedia movie ID'].values.tolist().index(int(row['Wikipedia movie ID']))\n","    df_summaries['Movie_Name'].iloc[idx] = df_movie_meta['Movie name'].iloc[index]\n","    df_summaries['Genre'].iloc[idx] = df_movie_meta['Movie genres'].iloc[index]\n","  except:\n","    labels.append(idx)\n","df_summaries.drop(index=labels, axis=0, inplace=True)\n","df_summaries.reset_index(drop=True, inplace=True)"],"metadata":{"id":"rg1tYif5SG5C"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Extract Genre \n","import json\n","def get_genre(x):\n","  return ', '.join(json.loads(x).values())\n","df_summaries['Genre'] = df_summaries['Genre'].apply(get_genre)"],"metadata":{"id":"ZgjNyeWQrfDk"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Extraction of different characters of a movie from the metadata\n","# Using BookNLP for this task, as current metadata skips a lot of characters from different stories\n","labels=[]\n","for idx, row in df_summaries.iterrows():\n","  index=-1\n","  check = True\n","  # for idx2, row2 in df_charac_meta.iterrows():\n","  #   if row2['Wikipedia movie ID'] == row['Wikipedia movie ID']:\n","  #     df_summaries['Characters'].iloc[idx] = df_summaries['Characters'].iloc[idx]+row2['Character name']\n","  print()\n","  print(row['Movie_Name'], end=' ')\n","  try:  \n","    while True:\n","      index = df_charac_meta['Wikipedia movie ID'].values.tolist()[index+1:].index(int(row['Wikipedia movie ID']))+index+1\n","      df_summaries['Characters'].iloc[idx] = df_summaries['Characters'].iloc[idx] + \", \" + df_charac_meta['Character name'].iloc[index]\n","      print(df_charac_meta['Character name'].iloc[index], end=' ')\n","      check = False\n","  except:\n","    if check:\n","      labels.append(idx)\n","df_summaries.drop(['Characters'], axis=1, inplace=True)\n","df_summaries = df_summaries.assign(Characters=\"\")"],"metadata":{"id":"dc7AJyU5URJu"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Save dataframe\n","df_summaries.to_csv('/content/drive/MyDrive/Visual Story Telling/Dataset - Story Generation/CMU_Movie_Dataset', index=False)"],"metadata":{"id":"WWVbO2N0t3vf"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":["### CMU Book Summaries"],"metadata":{"id":"6gADXA0gv3wY"}},{"cell_type":"code","source":["# Unzipping files from the folder\n","import tarfile\n","my_tar = tarfile.open('/content/drive/MyDrive/Visual Story Telling/Dataset - Story Generation/booksummaries.tar.gz')\n","my_tar.extractall('./CMU_Book_Summaries')\n","my_tar.close()"],"metadata":{"id":"YbC5fcY1v5yy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Readme file for the file description and content\n","with open('/content/CMU_Book_Summaries/booksummaries/README','r') as f:\n","  readme_data = f.read()\n","print(readme_data)"],"metadata":{"id":"IG_wm9PrwGKD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Loading the summaries\n","columns = ['Wikipedia article ID','Freebase ID','Book title','Author','Publication date','Book genres','Plot summary']\n","df_summaries = pd.read_table('/content/CMU_Book_Summaries/booksummaries/booksummaries.txt', header=None, names=columns)"],"metadata":{"id":"dlvSR944wTrJ"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Retaining only the useful columns and dropping rest of them\n","df_summaries.drop(['Freebase ID','Author','Publication date'], axis=1, inplace=True)"],"metadata":{"id":"0j7GyUaI0fAR"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries.dropna(axis=0, inplace=True)"],"metadata":{"id":"uVZGM6E-12Ss"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Genre extarction\n","import json\n","def extract_genres(x):\n","  return \", \".join(json.loads(x).values())"],"metadata":{"id":"X5Ny3EUS1ZhX"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["df_summaries['Book genres'] = df_summaries['Book genres'].apply(extract_genres)"],"metadata":{"id":"6kCj5UnK1ihw"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Saving the extracted dataset\n","df_summaries.to_csv('/content/drive/MyDrive/Visual Story Telling/Dataset - Story Generation/CMU_Book_Dataset', index=False)"],"metadata":{"id":"XtWZShLv1xe5"},"execution_count":null,"outputs":[]}]}