preprocessData.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1097db88-affa-416b-8934-6f2d9b9efe6b",
   "metadata": {},
   "source": [
    "### Importing the necessary libraries, modules and files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "75d50f8f-eb6e-4137-8088-a7c3878b384c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import datetime\n",
    "import random\n",
    "random.seed(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c7af0eb8-6911-4943-9da1-1a64634b92a7",
   "metadata": {},
   "source": [
    "##### Handling the case of Database as subjectDomain for query4 by adding database in its fieldOfStudy when the paper title contains the word database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "69a514b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(file = 'data/dataset.json', encoding = 'utf-8') as inputfile:\n",
    "    df = json.load(inputfile)\n",
    "with open(file = 'data/dataset2.json', mode = 'w+', encoding = 'utf-8') as outputfile:\n",
    "    df2 = df\n",
    "    for i in range(len(df2)):\n",
    "        details = df2[i]['_data']  \n",
    "        paperTitle = details['title']\n",
    "        if 'database' in paperTitle.lower():\n",
    "            if details['fieldsOfStudy']:\n",
    "                details['fieldsOfStudy'].append('Database')\n",
    "            else:\n",
    "                details['fieldsOfStudy'] = ['Database']\n",
    "    json.dump(df2,outputfile)\n",
    "inputfile.close()\n",
    "outputfile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "38c06104-b018-4c2b-b58c-5762c750248d",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('data/dataset2.json', encoding='utf-8') as inputfile:\n",
    "    df = json.load(inputfile)\n",
    "inputfile.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad263006-b970-4fea-9d81-6456fc385e7c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Collecting details about journals, conferences, authors, fieldsOfStudy"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f950897-bd57-4d77-831c-3cbc755becee",
   "metadata": {},
   "source": [
    "Extracting details from the dataset2.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9e967c89-efae-4d1b-abb4-f39d8fd13360",
   "metadata": {},
   "outputs": [],
   "source": [
    "conferences = []\n",
    "conferenceUrls = []\n",
    "\n",
    "journals = []\n",
    "journalUrls = []\n",
    "\n",
    "authors = []\n",
    "authorUrls = []\n",
    "\n",
    "subjectDomains = []\n",
    "\n",
    "for i in range(len(df)):\n",
    "    details = df[i]['_data']  \n",
    "    if details['publicationVenue']:\n",
    "        if 'type' in details['publicationVenue']:\n",
    "            if details['publicationVenue']['type'] == 'journal':\n",
    "                jName = details['publicationVenue']['name']\n",
    "                myId = details['publicationVenue']['id']\n",
    "                if jName not in journals:\n",
    "                    journals.append(jName)\n",
    "                    journalUrls.append(myId)\n",
    "            elif details['publicationVenue']['type'] == 'conference':\n",
    "                cName = details['publicationVenue']['name']\n",
    "                myId = details['publicationVenue']['id']\n",
    "                if cName not in conferences:\n",
    "                    conferences.append(cName)\n",
    "                    conferenceUrls.append(myId)\n",
    "    if details['authors']:\n",
    "        for i in range(len(details['authors'])):\n",
    "            aName = details['authors'][i]['name']\n",
    "            myId = details['authors'][i]['authorId']\n",
    "            if aName not in authors:\n",
    "                authors.append(aName)\n",
    "                authorUrls.append(str(myId))\n",
    "    if details['fieldsOfStudy']:\n",
    "        for i in details['fieldsOfStudy']:\n",
    "            if i not in subjectDomains:\n",
    "                subjectDomains.append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "86751277-d94e-4928-acbe-3d7b2a4df0cc",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Proceedings"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b98a093-3c36-4d59-8d45-46028e264786",
   "metadata": {},
   "source": [
    "It is assumed that each proceeding will be unique, i.e. proceedings will be the same amount as conferences."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "81fce26f-52b3-491f-9158-6bdd12a0b410",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning proceedings ids\n",
    "proceedingChoices = []\n",
    "proceedingYears = []\n",
    "ids = []\n",
    "for i in range(len(conferences)):\n",
    "    ids.append('cp'+str(i)) # stands for conferenceProceedings\n",
    "    \n",
    "    # Creating proceedings name as a random integer between (1, 30)\n",
    "    proceedingChoices.append('proceeding' + str(i))\n",
    "    \n",
    "    # Creating proceedings year as a random year between (2001, 2022)\n",
    "    proceedingYears.append(random.randint(2001,2022))\n",
    "        \n",
    "proceedingsDF = pd.DataFrame(ids, columns = ['proceedingId'])\n",
    "proceedingsDF['proceedingName'] = proceedingChoices\n",
    "proceedingsDF['proceedingYear'] = proceedingYears\n",
    "proceedingsDF.to_csv('data/proceedings.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8caaa3bd-54f1-4f2b-b9cd-7df086d1f343",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>proceedingId</th>\n",
       "      <th>proceedingName</th>\n",
       "      <th>proceedingYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cp0</td>\n",
       "      <td>proceeding0</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cp1</td>\n",
       "      <td>proceeding1</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  proceedingId proceedingName  proceedingYear\n",
       "0          cp0    proceeding0            2019\n",
       "1          cp1    proceeding1            2002"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proceedingsDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6aee8257-9054-4719-8cc9-8709bc9bb604",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Conferences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "36758d36-e114-48d7-8c93-e2320fb6fcd0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning conference ids which can be added to the uri's in the graph\n",
    "\n",
    "ids = []\n",
    "proceedings = []\n",
    "for i in range(len(conferences)):\n",
    "    ids.append('c'+str(i))\n",
    "\n",
    "conferencesDF = pd.DataFrame(ids, columns = ['conferenceId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "eeeda63e-bc57-48f9-8bc4-68a03dd2fc35",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning conference subclasses\n",
    "# randomly choose from cTypes\n",
    "cTypes = ['workshop','expertGroup','symposium','regularConference']\n",
    "\n",
    "conferenceTypes = []\n",
    "conferenceTypes = random.choices(cTypes, weights=[0.25,0.25,0.25,0.25], k=len(conferencesDF))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8701dd2d-196e-45b7-bbda-bca837c19096",
   "metadata": {},
   "outputs": [],
   "source": [
    "conferencesDF['conferenceUrl'] = conferenceUrls\n",
    "conferencesDF['conferenceTitle'] = conferences\n",
    "conferencesDF['conferenceType'] = conferenceTypes\n",
    "\n",
    "# Creating proceedings data for conferences\n",
    "conferencesDF['conferenceProceedingIds'] = proceedingsDF['proceedingId']\n",
    "conferencesDF.to_csv('data/conferences.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "106bb40e-f207-457b-b2f5-e35b8709796b",
   "metadata": {},
   "source": [
    "#### ConferenceProceedings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "ee711c86-e0bb-43b5-be3a-8ea5d36c771a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conferenceId</th>\n",
       "      <th>conferenceUrl</th>\n",
       "      <th>conferenceTitle</th>\n",
       "      <th>conferenceType</th>\n",
       "      <th>conferenceProceedingIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c0</td>\n",
       "      <td>b55b50b1-aae7-47a7-b042-8aecc930073d</td>\n",
       "      <td>International Conference on Human Factors in C...</td>\n",
       "      <td>regularConference</td>\n",
       "      <td>cp0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c1</td>\n",
       "      <td>2c9ecac6-f875-4a9b-acc2-10bd9f6782df</td>\n",
       "      <td>Technical Symposium on Computer Science Education</td>\n",
       "      <td>workshop</td>\n",
       "      <td>cp1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  conferenceId                         conferenceUrl  \\\n",
       "0           c0  b55b50b1-aae7-47a7-b042-8aecc930073d   \n",
       "1           c1  2c9ecac6-f875-4a9b-acc2-10bd9f6782df   \n",
       "\n",
       "                                     conferenceTitle     conferenceType  \\\n",
       "0  International Conference on Human Factors in C...  regularConference   \n",
       "1  Technical Symposium on Computer Science Education           workshop   \n",
       "\n",
       "  conferenceProceedingIds  \n",
       "0                     cp0  \n",
       "1                     cp1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conferencesDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0197e45e-4999-4fa6-ad57-1222b499ec48",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>proceedingId</th>\n",
       "      <th>proceedingName</th>\n",
       "      <th>proceedingYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>cp0</td>\n",
       "      <td>proceeding0</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>cp1</td>\n",
       "      <td>proceeding1</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  proceedingId proceedingName  proceedingYear\n",
       "0          cp0    proceeding0            2019\n",
       "1          cp1    proceeding1            2002"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proceedingsDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5118cd68-2503-4a00-bc0f-3b45e4ee1bfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "conferenceProceedingsDF = conferencesDF.merge(\n",
    "    proceedingsDF,\n",
    "    how = 'inner',\n",
    "    left_on = ['conferenceProceedingIds'],\n",
    "    right_on = ['proceedingId']\n",
    "    ).drop(columns = ['proceedingId'], axis = 1)\n",
    "conferenceProceedingsDF.to_csv('data/conferenceProceedings.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5ddce648-30b1-451f-b0f2-30474bc9774a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conferenceId</th>\n",
       "      <th>conferenceUrl</th>\n",
       "      <th>conferenceTitle</th>\n",
       "      <th>conferenceType</th>\n",
       "      <th>conferenceProceedingIds</th>\n",
       "      <th>proceedingName</th>\n",
       "      <th>proceedingYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c0</td>\n",
       "      <td>b55b50b1-aae7-47a7-b042-8aecc930073d</td>\n",
       "      <td>International Conference on Human Factors in C...</td>\n",
       "      <td>regularConference</td>\n",
       "      <td>cp0</td>\n",
       "      <td>proceeding0</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c1</td>\n",
       "      <td>2c9ecac6-f875-4a9b-acc2-10bd9f6782df</td>\n",
       "      <td>Technical Symposium on Computer Science Education</td>\n",
       "      <td>workshop</td>\n",
       "      <td>cp1</td>\n",
       "      <td>proceeding1</td>\n",
       "      <td>2002</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  conferenceId                         conferenceUrl  \\\n",
       "0           c0  b55b50b1-aae7-47a7-b042-8aecc930073d   \n",
       "1           c1  2c9ecac6-f875-4a9b-acc2-10bd9f6782df   \n",
       "\n",
       "                                     conferenceTitle     conferenceType  \\\n",
       "0  International Conference on Human Factors in C...  regularConference   \n",
       "1  Technical Symposium on Computer Science Education           workshop   \n",
       "\n",
       "  conferenceProceedingIds proceedingName  proceedingYear  \n",
       "0                     cp0    proceeding0            2019  \n",
       "1                     cp1    proceeding1            2002  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conferenceProceedingsDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "290facbe-dae5-405d-adbd-9845086482bc",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Volumes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5769a62-a116-4158-881a-88f73adab13d",
   "metadata": {},
   "source": [
    "It is assumed that each volume will be unique, i.e. volumes will be the same amount as journals."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d729766a-dca9-4c0f-8f3b-2cb275e8a7d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning proceedings ids\n",
    "volumeChoices = []\n",
    "volumeYears = []\n",
    "ids = []\n",
    "for i in range(len(journals)):\n",
    "    ids.append('jv'+str(i)) # stands for journalVolume\n",
    "    \n",
    "    # Creating proceedings value as a random integer between (1,30)\n",
    "    volumeChoices.append('journal' + str(i))\n",
    "    volumeYears.append(random.randint(2001,2022))\n",
    "        \n",
    "volumesDF = pd.DataFrame(ids, columns = ['volumeId'])\n",
    "volumesDF['volumeName'] = volumeChoices\n",
    "volumesDF['volumeYear'] = volumeYears\n",
    "volumesDF.to_csv('data/volumes.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "df99a43b-5676-498f-8db7-2a6bae2a8887",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volumeId</th>\n",
       "      <th>volumeName</th>\n",
       "      <th>volumeYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jv0</td>\n",
       "      <td>journal0</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jv1</td>\n",
       "      <td>journal1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  volumeId volumeName  volumeYear\n",
       "0      jv0   journal0        2006\n",
       "1      jv1   journal1        2020"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volumesDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3367214f-a482-4d4d-981e-f9a4476fa1e9",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### Journals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ececeec9-ce8c-4098-bb80-5bdd7f0006b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning journals ids which can be added to the uri's in the graph\n",
    "\n",
    "ids = []\n",
    "volumes = []\n",
    "for i in range(len(journals)):\n",
    "    ids.append('j'+str(i))\n",
    "\n",
    "journalsDF = pd.DataFrame(ids, columns = ['journalId'])\n",
    "journalsDF['journalUrl'] = journalUrls\n",
    "journalsDF['journalTitle'] = journals\n",
    "\n",
    "# Creating volumes data for journals\n",
    "journalsDF['journalVolumeIds'] = volumesDF['volumeId']\n",
    "\n",
    "journalsDF.to_csv('data/journals.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ba4cf652-c9ed-4401-93ba-dcd305f9db17",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>journalId</th>\n",
       "      <th>journalUrl</th>\n",
       "      <th>journalTitle</th>\n",
       "      <th>journalVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>j0</td>\n",
       "      <td>458166b3-de17-4bf3-bbbb-e53782de2f0f</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>j1</td>\n",
       "      <td>c6840156-ee10-4d78-8832-7f8909811576</td>\n",
       "      <td>IEEE Transactions on Knowledge and Data Engine...</td>\n",
       "      <td>jv1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  journalId                            journalUrl  \\\n",
       "0        j0  458166b3-de17-4bf3-bbbb-e53782de2f0f   \n",
       "1        j1  c6840156-ee10-4d78-8832-7f8909811576   \n",
       "\n",
       "                                        journalTitle journalVolumeIds  \n",
       "0                               Nature Biotechnology              jv0  \n",
       "1  IEEE Transactions on Knowledge and Data Engine...              jv1  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "journalsDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5409aa16-1d73-4d21-a9c6-93cb37e64043",
   "metadata": {},
   "source": [
    "#### JournalVolumes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "b12591ce-677b-4053-9c44-8a5e7178adef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>journalId</th>\n",
       "      <th>journalUrl</th>\n",
       "      <th>journalTitle</th>\n",
       "      <th>journalVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>j0</td>\n",
       "      <td>458166b3-de17-4bf3-bbbb-e53782de2f0f</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>j1</td>\n",
       "      <td>c6840156-ee10-4d78-8832-7f8909811576</td>\n",
       "      <td>IEEE Transactions on Knowledge and Data Engine...</td>\n",
       "      <td>jv1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  journalId                            journalUrl  \\\n",
       "0        j0  458166b3-de17-4bf3-bbbb-e53782de2f0f   \n",
       "1        j1  c6840156-ee10-4d78-8832-7f8909811576   \n",
       "\n",
       "                                        journalTitle journalVolumeIds  \n",
       "0                               Nature Biotechnology              jv0  \n",
       "1  IEEE Transactions on Knowledge and Data Engine...              jv1  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "journalsDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "386d4481-58c5-433a-afde-c3e0d9a8a5c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>volumeId</th>\n",
       "      <th>volumeName</th>\n",
       "      <th>volumeYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>jv0</td>\n",
       "      <td>journal0</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>jv1</td>\n",
       "      <td>journal1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  volumeId volumeName  volumeYear\n",
       "0      jv0   journal0        2006\n",
       "1      jv1   journal1        2020"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "volumesDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "b8140561-5bf9-435e-b062-a3a247acbccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "journalVolumesDF = journalsDF.merge(\n",
    "    volumesDF,\n",
    "    how = 'inner',\n",
    "    left_on = ['journalVolumeIds'],\n",
    "    right_on = ['volumeId']\n",
    "    ).drop(columns = ['volumeId'], axis = 1)\n",
    "journalVolumesDF.to_csv('data/journalVolumes.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "b19bb6ec-8645-4086-aa7c-697b70c2c76a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>journalId</th>\n",
       "      <th>journalUrl</th>\n",
       "      <th>journalTitle</th>\n",
       "      <th>journalVolumeIds</th>\n",
       "      <th>volumeName</th>\n",
       "      <th>volumeYear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>j0</td>\n",
       "      <td>458166b3-de17-4bf3-bbbb-e53782de2f0f</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>journal0</td>\n",
       "      <td>2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>j1</td>\n",
       "      <td>c6840156-ee10-4d78-8832-7f8909811576</td>\n",
       "      <td>IEEE Transactions on Knowledge and Data Engine...</td>\n",
       "      <td>jv1</td>\n",
       "      <td>journal1</td>\n",
       "      <td>2020</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  journalId                            journalUrl  \\\n",
       "0        j0  458166b3-de17-4bf3-bbbb-e53782de2f0f   \n",
       "1        j1  c6840156-ee10-4d78-8832-7f8909811576   \n",
       "\n",
       "                                        journalTitle journalVolumeIds  \\\n",
       "0                               Nature Biotechnology              jv0   \n",
       "1  IEEE Transactions on Knowledge and Data Engine...              jv1   \n",
       "\n",
       "  volumeName  volumeYear  \n",
       "0   journal0        2006  \n",
       "1   journal1        2020  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "journalVolumesDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50af79f4-5538-4485-812e-e66e41faa132",
   "metadata": {},
   "source": [
    "#### Authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f90d9d95-7df9-4792-a5bf-c85bb6c71250",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning author ids which can be added to the uri's in the graph\n",
    "\n",
    "ids = []\n",
    "for i in range(len(authors)):\n",
    "    ids.append('a'+str(i))\n",
    "\n",
    "authorsDF = pd.DataFrame(ids, columns = ['authorId'])\n",
    "authorsDF['authorUrl'] = authorUrls\n",
    "authorsDF['authorName'] = authors\n",
    "authorsDF.to_csv('data/authors.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "50490190-59ff-43f3-a5a1-62366835f9b6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>authorName</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a0</td>\n",
       "      <td>6391996</td>\n",
       "      <td>E. Bolyen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a1</td>\n",
       "      <td>6307534</td>\n",
       "      <td>J. Rideout</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>a2</td>\n",
       "      <td>40950692</td>\n",
       "      <td>Matthew R. Dillon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>a3</td>\n",
       "      <td>4230536</td>\n",
       "      <td>N. Bokulich</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a4</td>\n",
       "      <td>5538503</td>\n",
       "      <td>C. Abnet</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorId authorUrl         authorName\n",
       "0       a0   6391996          E. Bolyen\n",
       "1       a1   6307534         J. Rideout\n",
       "2       a2  40950692  Matthew R. Dillon\n",
       "3       a3   4230536        N. Bokulich\n",
       "4       a4   5538503           C. Abnet"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authorsDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71f566c3-8379-45dd-b267-dd8caa2e6016",
   "metadata": {},
   "source": [
    "#### SubjectDomain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "3c119ba8-9ee8-4281-bd2e-15d3fa684e28",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assigning subjectdomain ids which can be added to the uri's in the graph\n",
    "\n",
    "ids = []\n",
    "for i in range(len(subjectDomains)):\n",
    "    ids.append('d'+str(i))\n",
    "\n",
    "subjectDomainDF = pd.DataFrame(ids, columns = ['subjectDomainId'])\n",
    "subjectDomainDF['subjectDomainName'] = subjectDomains\n",
    "subjectDomainDF.to_csv('data/subjectDomain.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8ed274c-5f47-4bc1-9b04-919d3b59c8f6",
   "metadata": {},
   "source": [
    "### Collecting information about papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2ed77e71-eed7-4b3d-9613-e5c95e869bae",
   "metadata": {},
   "outputs": [],
   "source": [
    "papers = []\n",
    "\n",
    "paperUrlNum = 0\n",
    "\n",
    "for i in range(len(df)):\n",
    "    paperCurr = [] \n",
    "    details = df[i]['_data']\n",
    "    \n",
    "    currPaperUrlNum = 'p'+str(paperUrlNum)\n",
    "    paperCurr.append(currPaperUrlNum)\n",
    "    paperCurr.append(details['paperId'])\n",
    "    \n",
    "    if details['publicationVenue']:\n",
    "        if 'type' in details['publicationVenue']:\n",
    "            if details['publicationVenue']['type'] == 'journal':\n",
    "                jName = details['publicationVenue']['name']\n",
    "                paperCurr.append(journalsDF[journalsDF['journalTitle'] == jName]['journalId'].values[0])\n",
    "                paperCurr.append(jName)\n",
    "            elif details['publicationVenue']['type'] == 'conference':\n",
    "                cName = details['publicationVenue']['name']\n",
    "                paperCurr.append(conferencesDF[conferencesDF['conferenceTitle'] == cName]['conferenceId'].values[0])\n",
    "                paperCurr.append(cName)\n",
    "\n",
    "            else:\n",
    "                paperCurr.append('None')\n",
    "                paperCurr.append('None')\n",
    "        else:\n",
    "            paperCurr.append('None')\n",
    "            paperCurr.append('None')\n",
    "    else:\n",
    "        paperCurr.append('None') # conferenceJournalUrl\n",
    "        paperCurr.append('None') # conferenceJournal\n",
    "    \n",
    "    paperCurr.append(details['title'])\n",
    "    paperCurr.append(details['abstract'])\n",
    "    paperUrlNum += 1\n",
    "    papers.append(paperCurr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b27679a4-3ebf-4128-bdb9-e33892c01de7",
   "metadata": {},
   "outputs": [],
   "source": [
    "paperColumns = ['paperId','paperUrl','conferenceJournalId','conferenceJournalTitle','paperTitle','paperAbstract']\n",
    "\n",
    "papersDF = pd.DataFrame(papers, columns = paperColumns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a6b592c8-75a5-4445-9891-bf2644ebeadf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assigning paper subclasses\n",
    "# Assign type poster if not a conference/journal\n",
    "# Else, randomly choose from pTypes\n",
    "\n",
    "paperTypes = []\n",
    "pTypes = ['shortPaper','demoPaper','fullPaper']\n",
    "for index, row in papersDF.iterrows():\n",
    "    if row['conferenceJournalId'] == 'None':\n",
    "        paperTypes.append('poster')\n",
    "    else:\n",
    "        paperTypes.append(random.choice(pTypes))\n",
    "papersDF['paperType'] = paperTypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "55db92d9-9983-4ef4-94d1-e0360171a3c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p0</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>None</td>\n",
       "      <td>shortPaper</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  paperId                                  paperUrl conferenceJournalId  \\\n",
       "0      p0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "\n",
       "  conferenceJournalTitle                                         paperTitle  \\\n",
       "0   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "  paperAbstract   paperType  \n",
       "0          None  shortPaper  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papersDF.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "50ebd2df-bd60-4d2c-a4d4-844ff1172845",
   "metadata": {},
   "outputs": [],
   "source": [
    "papersDF['conferenceJournalId'] = papersDF.apply(lambda x: random.choice(conferencesDF.conferenceId) if x['paperType'] == 'poster' else x['conferenceJournalId'], axis = 1)\n",
    "papersDF['conferenceJournalTitle'] = papersDF.apply(lambda x: conferencesDF[conferencesDF['conferenceId'] == x['conferenceJournalId']]['conferenceTitle'].item() if x['paperType'] == 'poster' else x['conferenceJournalTitle'], axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "b356f5a7-13a4-41bc-9e2c-9fa4f723fab2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Getting proceedings and volumes for conferences/journals for each paper\n",
    "procVols = []\n",
    "for k in range(len(papersDF['paperId'])):\n",
    "    currConfJourId = papersDF['conferenceJournalId'][k]\n",
    "    if currConfJourId[0] == 'c':\n",
    "        getProcVol = conferencesDF[conferencesDF['conferenceId'] == currConfJourId]['conferenceProceedingIds'].item()\n",
    "    else:\n",
    "        getProcVol = journalsDF[journalsDF['journalId'] == currConfJourId]['journalVolumeIds'].item()\n",
    "    procVols.append(getProcVol)\n",
    "papersDF['proceedingsVolumeIds'] = procVols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "865109d0-c010-467d-816e-a2df5426e1dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p0</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>None</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>p1</td>\n",
       "      <td>fa5853fdef7d2f6bb68203d187ddacbbddc63a8b</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>poster</td>\n",
       "      <td>cp14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  paperId                                  paperUrl conferenceJournalId  \\\n",
       "0      p0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "1      p1  fa5853fdef7d2f6bb68203d187ddacbbddc63a8b                 c14   \n",
       "\n",
       "                              conferenceJournalTitle  \\\n",
       "0                               Nature Biotechnology   \n",
       "1  International Conference on Exploring Services...   \n",
       "\n",
       "                                          paperTitle  \\\n",
       "0  Reproducible, interactive, scalable and extens...   \n",
       "1  High-Dimensional Probability: An Introduction ...   \n",
       "\n",
       "                                       paperAbstract   paperType  \\\n",
       "0                                               None  shortPaper   \n",
       "1  © 2018, Cambridge University Press Let us summ...      poster   \n",
       "\n",
       "  proceedingsVolumeIds  \n",
       "0                  jv0  \n",
       "1                 cp14  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "papersDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5726ac3-810f-43b2-a7f6-1a75870e5268",
   "metadata": {},
   "source": [
    "Since, some of the paper's abstract is null, we are imputing with a default value of \"Abstract content goes here ...\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "c5c2921b-a773-41f7-aa98-08c48ad31a66",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Imputing None values for abstract\n",
    "papersDF['paperAbstract'] = papersDF.apply(lambda x: 'Abstract content goes here ...' if x['paperAbstract'] == None else x['paperAbstract'], axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5052b386-62b2-4407-8311-ee22bebe129f",
   "metadata": {},
   "source": [
    "Imputing conferenceJournalId as randomly chosen conference from the list of conferences and corresponding conferenceJournalTitle for the paperType as poster."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "94564970-900d-426e-a9f1-3d76ac8ddafa",
   "metadata": {},
   "outputs": [],
   "source": [
    "papersDF.to_csv('data/papers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40310f57-9e7c-4d58-8621-199f1e1f0c33",
   "metadata": {},
   "source": [
    "### Creating authors subjectDomain relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "87de7d68-97aa-4367-bb0e-fb16df5be2a6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subjectDomainId</th>\n",
       "      <th>subjectDomainName</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d0</td>\n",
       "      <td>Engineering</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>d1</td>\n",
       "      <td>Medicine</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  subjectDomainId subjectDomainName\n",
       "0              d0       Engineering\n",
       "1              d1          Medicine"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subjectDomainDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "52419b04-74ae-4c0c-ae9d-6c1e8849f9a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "subject_domain_ids = []\n",
    "pIds = []\n",
    "\n",
    "paperUrlNum = 0\n",
    "\n",
    "for i in range(len(df)):\n",
    "    paperCurr = [] \n",
    "    details = df[i]['_data']\n",
    "    \n",
    "    currPaperUrlNum = 'p'+str(paperUrlNum)\n",
    "    \n",
    "    # Adding pId to subjectDomainId\n",
    "    if details['fieldsOfStudy']:\n",
    "        for i in range(len(details['fieldsOfStudy'])):\n",
    "            subjectDomainId = details['fieldsOfStudy'][i]\n",
    "            subject_domain_ids.append(subjectDomainId)\n",
    "            pIds.append(currPaperUrlNum)\n",
    "    paperUrlNum += 1\n",
    "\n",
    "subjectsPapersDF = pd.DataFrame(subject_domain_ids, columns = ['subjectDomainName'])\n",
    "subjectsPapersDF['paperId'] = pIds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "6c3424ed-d970-47db-a73b-2fc0e4ac18cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "subjectsPapersDF = subjectsPapersDF.merge(\n",
    "                    subjectDomainDF,\n",
    "                    how = 'inner',\n",
    "                    left_on = ['subjectDomainName'],\n",
    "                    right_on = ['subjectDomainName']\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "92d49805-398b-411a-b45f-71a98bd367f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "subjectsPapersDF = subjectsPapersDF.merge(\n",
    "                    papersDF,\n",
    "                    how = 'inner',\n",
    "                    left_on = ['paperId'],\n",
    "                    right_on = ['paperId']\n",
    "                    )\n",
    "subjectsPapersDF.to_csv('data/subjectsPapers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "fa99f7c7-90ca-49dd-ba98-915d2b6cf6c3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subjectDomainName</th>\n",
       "      <th>paperId</th>\n",
       "      <th>subjectDomainId</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Engineering</td>\n",
       "      <td>p0</td>\n",
       "      <td>d0</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Medicine</td>\n",
       "      <td>p0</td>\n",
       "      <td>d1</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Engineering</td>\n",
       "      <td>p19</td>\n",
       "      <td>d0</td>\n",
       "      <td>2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034</td>\n",
       "      <td>c70</td>\n",
       "      <td>International Conference on Intelligent Roboti...</td>\n",
       "      <td>Spectral Methods for Data Science: A Statistic...</td>\n",
       "      <td>Spectral methods have emerged as a simple yet ...</td>\n",
       "      <td>poster</td>\n",
       "      <td>cp70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Computer Science</td>\n",
       "      <td>p19</td>\n",
       "      <td>d3</td>\n",
       "      <td>2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034</td>\n",
       "      <td>c70</td>\n",
       "      <td>International Conference on Intelligent Roboti...</td>\n",
       "      <td>Spectral Methods for Data Science: A Statistic...</td>\n",
       "      <td>Spectral methods have emerged as a simple yet ...</td>\n",
       "      <td>poster</td>\n",
       "      <td>cp70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Mathematics</td>\n",
       "      <td>p19</td>\n",
       "      <td>d5</td>\n",
       "      <td>2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034</td>\n",
       "      <td>c70</td>\n",
       "      <td>International Conference on Intelligent Roboti...</td>\n",
       "      <td>Spectral Methods for Data Science: A Statistic...</td>\n",
       "      <td>Spectral methods have emerged as a simple yet ...</td>\n",
       "      <td>poster</td>\n",
       "      <td>cp70</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  subjectDomainName paperId subjectDomainId  \\\n",
       "0       Engineering      p0              d0   \n",
       "1          Medicine      p0              d1   \n",
       "2       Engineering     p19              d0   \n",
       "3  Computer Science     p19              d3   \n",
       "4       Mathematics     p19              d5   \n",
       "\n",
       "                                   paperUrl conferenceJournalId  \\\n",
       "0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "1  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "2  2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034                 c70   \n",
       "3  2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034                 c70   \n",
       "4  2d6adb9636df5a8a5dbcbfaecd0c4d34d7c85034                 c70   \n",
       "\n",
       "                              conferenceJournalTitle  \\\n",
       "0                               Nature Biotechnology   \n",
       "1                               Nature Biotechnology   \n",
       "2  International Conference on Intelligent Roboti...   \n",
       "3  International Conference on Intelligent Roboti...   \n",
       "4  International Conference on Intelligent Roboti...   \n",
       "\n",
       "                                          paperTitle  \\\n",
       "0  Reproducible, interactive, scalable and extens...   \n",
       "1  Reproducible, interactive, scalable and extens...   \n",
       "2  Spectral Methods for Data Science: A Statistic...   \n",
       "3  Spectral Methods for Data Science: A Statistic...   \n",
       "4  Spectral Methods for Data Science: A Statistic...   \n",
       "\n",
       "                                       paperAbstract   paperType  \\\n",
       "0                     Abstract content goes here ...  shortPaper   \n",
       "1                     Abstract content goes here ...  shortPaper   \n",
       "2  Spectral methods have emerged as a simple yet ...      poster   \n",
       "3  Spectral methods have emerged as a simple yet ...      poster   \n",
       "4  Spectral methods have emerged as a simple yet ...      poster   \n",
       "\n",
       "  proceedingsVolumeIds  \n",
       "0                  jv0  \n",
       "1                  jv0  \n",
       "2                 cp70  \n",
       "3                 cp70  \n",
       "4                 cp70  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subjectsPapersDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "144629f4-5f1a-49f4-a973-fa26f4074258",
   "metadata": {},
   "source": [
    "### Creating proceedings - subjectDomain and volumes - subjectDomain relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "11a869c7-18d8-4306-8a19-ed5f4751223e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>subjectDomainName</th>\n",
       "      <th>paperId</th>\n",
       "      <th>subjectDomainId</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Engineering</td>\n",
       "      <td>p0</td>\n",
       "      <td>d0</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  subjectDomainName paperId subjectDomainId  \\\n",
       "0       Engineering      p0              d0   \n",
       "\n",
       "                                   paperUrl conferenceJournalId  \\\n",
       "0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "\n",
       "  conferenceJournalTitle                                         paperTitle  \\\n",
       "0   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "                    paperAbstract   paperType proceedingsVolumeIds  \n",
       "0  Abstract content goes here ...  shortPaper                  jv0  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subjectsPapersDF.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "292f7bc1-9b15-4bcc-a916-e87444c52b9b",
   "metadata": {},
   "outputs": [],
   "source": [
    "subjectsConferencesJournalsDF = subjectsPapersDF[['conferenceJournalId','proceedingsVolumeIds','subjectDomainId']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "71f0d9b1-c0d3-4041-ab8c-6106f17bce54",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "      <th>subjectDomainId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>j0</td>\n",
       "      <td>jv0</td>\n",
       "      <td>d0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>j0</td>\n",
       "      <td>jv0</td>\n",
       "      <td>d1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c70</td>\n",
       "      <td>cp70</td>\n",
       "      <td>d0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c70</td>\n",
       "      <td>cp70</td>\n",
       "      <td>d3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c70</td>\n",
       "      <td>cp70</td>\n",
       "      <td>d5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  conferenceJournalId proceedingsVolumeIds subjectDomainId\n",
       "0                  j0                  jv0              d0\n",
       "1                  j0                  jv0              d1\n",
       "2                 c70                 cp70              d0\n",
       "3                 c70                 cp70              d3\n",
       "4                 c70                 cp70              d5"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subjectsConferencesJournalsDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c4d35156-2d0a-4fd4-9c0e-354b90cb2ff0",
   "metadata": {},
   "outputs": [],
   "source": [
    "subjectsConferencesJournalsDF.to_csv('data/subjectsProceedingsVolumes.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67116c09-b6f7-4684-945e-d49323dd76e6",
   "metadata": {},
   "source": [
    "### Creating authors papers relation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "d33ce0dd-bdcf-4a8a-9f0c-b938fcc11830",
   "metadata": {},
   "outputs": [],
   "source": [
    "aUrlIds = []\n",
    "pIds = []\n",
    "\n",
    "paperUrlNum = 0\n",
    "\n",
    "for i in range(len(df)):\n",
    "    paperCurr = [] \n",
    "    details = df[i]['_data']\n",
    "    currPaperUrlNum = 'p'+str(paperUrlNum)\n",
    "    \n",
    "    # Appending the pId to authorsDF\n",
    "    if details['authors']:\n",
    "        for i in range(len(details['authors'])):\n",
    "            aUrlId = details['authors'][i]['authorId']\n",
    "            aUrlIds.append(str(aUrlId))\n",
    "            pIds.append(currPaperUrlNum)\n",
    "    paperUrlNum += 1\n",
    "\n",
    "authorsPapersDF = pd.DataFrame(aUrlIds, columns = ['authorUrl'])\n",
    "authorsPapersDF['paperId'] = pIds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "1bd240ee-2cb1-4ace-a102-aa200b2ccc84",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>authorName</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a0</td>\n",
       "      <td>6391996</td>\n",
       "      <td>E. Bolyen</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a1</td>\n",
       "      <td>6307534</td>\n",
       "      <td>J. Rideout</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorId authorUrl  authorName\n",
       "0       a0   6391996   E. Bolyen\n",
       "1       a1   6307534  J. Rideout"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authorsDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "112ca99e-dacf-443c-b004-79689fef8f5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "authorsPapersDF = authorsPapersDF.merge(\n",
    "                    authorsDF,\n",
    "                    how = 'inner',\n",
    "                    left_on = ['authorUrl'],\n",
    "                    right_on = ['authorUrl']\n",
    "                    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "59e7e92d-1079-4b07-bccb-196182a71f97",
   "metadata": {},
   "outputs": [],
   "source": [
    "authorsPapersDF = authorsPapersDF.merge(\n",
    "                    papersDF,\n",
    "                    how = 'inner',\n",
    "                    left_on = ['paperId'],\n",
    "                    right_on = ['paperId']\n",
    "                    )\n",
    "authorsPapersDF.head()\n",
    "authorsPapersDF.to_csv('data/authorsPapers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "f481013a-2c35-431a-af29-6080cadf3dbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>paperId</th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorName</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6391996</td>\n",
       "      <td>p0</td>\n",
       "      <td>a0</td>\n",
       "      <td>E. Bolyen</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6307534</td>\n",
       "      <td>p0</td>\n",
       "      <td>a1</td>\n",
       "      <td>J. Rideout</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorUrl paperId authorId  authorName  \\\n",
       "0   6391996      p0       a0   E. Bolyen   \n",
       "1   6307534      p0       a1  J. Rideout   \n",
       "\n",
       "                                   paperUrl conferenceJournalId  \\\n",
       "0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "1  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "\n",
       "  conferenceJournalTitle                                         paperTitle  \\\n",
       "0   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "1   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "                    paperAbstract   paperType proceedingsVolumeIds  \n",
       "0  Abstract content goes here ...  shortPaper                  jv0  \n",
       "1  Abstract content goes here ...  shortPaper                  jv0  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authorsPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61dbb708-37ba-4cc1-b988-983ba010cb7f",
   "metadata": {},
   "source": [
    "### Creating Submitted Paper Related Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "0599e328-e6d7-41dd-9800-2ba0e9409746",
   "metadata": {},
   "outputs": [],
   "source": [
    "decisionChoices = ['accepted','rejected']\n",
    "\n",
    "ids = []\n",
    "for i in range(len(papersDF)):\n",
    "    ids.append('s'+str(i))\n",
    "\n",
    "submittedPapersDF = pd.DataFrame(ids, columns = ['submissionId'])\n",
    "submittedPapersDF['paperId'] = papersDF['paperId']\n",
    "submittedPapersDF['paperTitle'] = papersDF['paperTitle']\n",
    "submittedPapersDF['paperAbstract'] = papersDF['paperAbstract']\n",
    "submittedPapersDF['conferenceJournalId'] = papersDF['conferenceJournalId']\n",
    "submittedPapersDF['conferenceJournalTitle'] = papersDF['conferenceJournalTitle']\n",
    "submittedPapersDF['proceedingsVolume'] = papersDF['proceedingsVolumeIds']\n",
    "submittedPapersDF['decision'] = random.choices(decisionChoices, weights = [0.8,0.2], k = len(submittedPapersDF))\n",
    "submittedPapersDF.to_csv('data/submittedPapers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "790b6a75-b5b0-438e-8951-ecc2daf5c573",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>submissionId</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>s0</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>s1</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>s2</td>\n",
       "      <td>p2</td>\n",
       "      <td>CRISP-DM Twenty Years Later: From Data Mining ...</td>\n",
       "      <td>CRISP-DM(CRoss-Industry Standard Process for D...</td>\n",
       "      <td>j1</td>\n",
       "      <td>IEEE Transactions on Knowledge and Data Engine...</td>\n",
       "      <td>jv1</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>s3</td>\n",
       "      <td>p3</td>\n",
       "      <td>AutoDS: Towards Human-Centered Automation of D...</td>\n",
       "      <td>Data science (DS) projects often follow a life...</td>\n",
       "      <td>c0</td>\n",
       "      <td>International Conference on Human Factors in C...</td>\n",
       "      <td>cp0</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>s4</td>\n",
       "      <td>p4</td>\n",
       "      <td>Data Science and Analytics: An Overview from D...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j2</td>\n",
       "      <td>SN Computer Science</td>\n",
       "      <td>jv2</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  submissionId paperId                                         paperTitle  \\\n",
       "0           s0      p0  Reproducible, interactive, scalable and extens...   \n",
       "1           s1      p1  High-Dimensional Probability: An Introduction ...   \n",
       "2           s2      p2  CRISP-DM Twenty Years Later: From Data Mining ...   \n",
       "3           s3      p3  AutoDS: Towards Human-Centered Automation of D...   \n",
       "4           s4      p4  Data Science and Analytics: An Overview from D...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0                     Abstract content goes here ...                  j0   \n",
       "1  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "2  CRISP-DM(CRoss-Industry Standard Process for D...                  j1   \n",
       "3  Data science (DS) projects often follow a life...                  c0   \n",
       "4                     Abstract content goes here ...                  j2   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0                               Nature Biotechnology               jv0   \n",
       "1  International Conference on Exploring Services...              cp14   \n",
       "2  IEEE Transactions on Knowledge and Data Engine...               jv1   \n",
       "3  International Conference on Human Factors in C...               cp0   \n",
       "4                                SN Computer Science               jv2   \n",
       "\n",
       "   decision  \n",
       "0  rejected  \n",
       "1  accepted  \n",
       "2  rejected  \n",
       "3  accepted  \n",
       "4  accepted  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submittedPapersDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "24922f2a-7248-4ee8-9032-bf72c7664bed",
   "metadata": {},
   "source": [
    "### Creating Reviews Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "5884f7d3-bb0c-4184-819d-ca774dede7ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "# assiging ids to reviews for adding to uri's of the graph\n",
    "\n",
    "ids = []\n",
    "for i in range(len(submittedPapersDF)):\n",
    "    ids.append('r'+str(i))\n",
    "\n",
    "reviewsDF = pd.DataFrame(ids, columns = ['reviewId'])\n",
    "reviewsDF['submittedPaperId'] = submittedPapersDF['submissionId']\n",
    "reviewsDF['reviewText'] = ['content of the reviewed text goes here....'] * len(reviewsDF)\n",
    "reviewsDF['reviewDecision'] = submittedPapersDF['decision']\n",
    "reviewsDF['reviewDecisionBoolean'] = reviewsDF.apply(lambda x: 1 if x['reviewDecision'] == 'accepted' else 0, axis = 1)\n",
    "reviewsDF['submittedPaperTitle'] = submittedPapersDF['paperTitle']\n",
    "reviewsDF.to_csv('data/reviews.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "450241d5-a819-4592-8947-5343385ff820",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>submissionId</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>s0</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>s1</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  submissionId paperId                                         paperTitle  \\\n",
       "0           s0      p0  Reproducible, interactive, scalable and extens...   \n",
       "1           s1      p1  High-Dimensional Probability: An Introduction ...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0                     Abstract content goes here ...                  j0   \n",
       "1  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0                               Nature Biotechnology               jv0   \n",
       "1  International Conference on Exploring Services...              cp14   \n",
       "\n",
       "   decision  \n",
       "0  rejected  \n",
       "1  accepted  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submittedPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cecf58f2-3945-43ef-a59c-0cad17646e8d",
   "metadata": {},
   "source": [
    "### Creating Final Paper Related Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "225a75b6-6056-4261-80eb-b7df5571c4fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalPapersDF = submittedPapersDF[(submittedPapersDF['decision'] == 'accepted')].reset_index().drop(['index'],axis = 1)\n",
    "\n",
    "ids = []\n",
    "for i in range(len(finalPapersDF)):\n",
    "    ids.append('f'+str(i))\n",
    "    \n",
    "finalPapersDF['finalPaperId'] = ids\n",
    "finalPapersDF['proceedingVolumeYear'] = finalPapersDF.apply(lambda x: proceedingsDF[proceedingsDF['proceedingId'] == x['proceedingsVolume']]['proceedingYear'].item() if x['proceedingsVolume'][0] == 'c' else volumesDF[volumesDF['volumeId'] == x['proceedingsVolume']]['volumeYear'].item(),axis = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a763443-8145-4dc2-ab50-3c6607ebc716",
   "metadata": {},
   "source": [
    "It is assumed that all the proceedings/volumes are between 2001 and 2022 and all the publications are accepted on a random date in their publicationYear."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "486f08ed-d0e6-4cd2-afee-15739e8bcf00",
   "metadata": {},
   "outputs": [],
   "source": [
    "acceptanceDates = []\n",
    "for i in range(len(finalPapersDF['paperId'])):  \n",
    "    year = finalPapersDF['proceedingVolumeYear'][i]\n",
    "    \n",
    "    # Generate a random acceptance date in the year of publication\n",
    "    start_date = datetime.date(year, 1, 1)\n",
    "    end_date = datetime.date(year, 12, 31)\n",
    "    \n",
    "    num_days = (end_date - start_date).days\n",
    "    rand_days = random.randint(1, num_days)\n",
    "    random_date = start_date + datetime.timedelta(days=rand_days)\n",
    "    acceptanceDates.append(random_date)\n",
    "\n",
    "finalPapersDF['acceptanceDate'] = acceptanceDates "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c01a292f-cb1c-4b26-a281-8a65a49c4ef9",
   "metadata": {},
   "outputs": [],
   "source": [
    "finalPapersDF.to_csv('data/finalPapers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f4e9fa6f-d175-4900-929c-e6315c7647b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>submissionId</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "      <th>finalPaperId</th>\n",
       "      <th>proceedingVolumeYear</th>\n",
       "      <th>acceptanceDate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>s1</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "      <td>f0</td>\n",
       "      <td>2016</td>\n",
       "      <td>2016-05-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>s3</td>\n",
       "      <td>p3</td>\n",
       "      <td>AutoDS: Towards Human-Centered Automation of D...</td>\n",
       "      <td>Data science (DS) projects often follow a life...</td>\n",
       "      <td>c0</td>\n",
       "      <td>International Conference on Human Factors in C...</td>\n",
       "      <td>cp0</td>\n",
       "      <td>accepted</td>\n",
       "      <td>f1</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019-10-29</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  submissionId paperId                                         paperTitle  \\\n",
       "0           s1      p1  High-Dimensional Probability: An Introduction ...   \n",
       "1           s3      p3  AutoDS: Towards Human-Centered Automation of D...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "1  Data science (DS) projects often follow a life...                  c0   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0  International Conference on Exploring Services...              cp14   \n",
       "1  International Conference on Human Factors in C...               cp0   \n",
       "\n",
       "   decision finalPaperId  proceedingVolumeYear acceptanceDate  \n",
       "0  accepted           f0                  2016     2016-05-08  \n",
       "1  accepted           f1                  2019     2019-10-29  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finalPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ddfddf0-1fd0-4dd0-b526-7e74448a8ab7",
   "metadata": {},
   "source": [
    "### Creating Reviewer Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "cbf4f2d9-6e79-48f8-84f3-fa2260cc4c89",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_authors = list(authorsPapersDF['authorId'].unique())\n",
    "paperAuthors = authorsPapersDF.groupby('paperId', as_index = False, sort = False)['authorId'].agg(lambda x: [l for l in x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "26ac598b-7a9e-4a8f-b94f-caa1ee0c1264",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>paperId</th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorName</th>\n",
       "      <th>paperUrl</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>paperType</th>\n",
       "      <th>proceedingsVolumeIds</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6391996</td>\n",
       "      <td>p0</td>\n",
       "      <td>a0</td>\n",
       "      <td>E. Bolyen</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6307534</td>\n",
       "      <td>p0</td>\n",
       "      <td>a1</td>\n",
       "      <td>J. Rideout</td>\n",
       "      <td>0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>shortPaper</td>\n",
       "      <td>jv0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  authorUrl paperId authorId  authorName  \\\n",
       "0   6391996      p0       a0   E. Bolyen   \n",
       "1   6307534      p0       a1  J. Rideout   \n",
       "\n",
       "                                   paperUrl conferenceJournalId  \\\n",
       "0  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "1  0c2d3b28d48426b8b72f7214a7708ba8b4efa9d6                  j0   \n",
       "\n",
       "  conferenceJournalTitle                                         paperTitle  \\\n",
       "0   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "1   Nature Biotechnology  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "                    paperAbstract   paperType proceedingsVolumeIds  \n",
       "0  Abstract content goes here ...  shortPaper                  jv0  \n",
       "1  Abstract content goes here ...  shortPaper                  jv0  "
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "authorsPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "1260b3d5-c79a-49fe-b920-0b904718fdcd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paperId</th>\n",
       "      <th>authorId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>p0</td>\n",
       "      <td>[a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>p13</td>\n",
       "      <td>[a0, a1, a2, a3, a4, a5, a6, a6, a7, a8, a9, a...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>p2039</td>\n",
       "      <td>[a15, a8363, a8365, a8366, a8369, a8372, a8402...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>p1191</td>\n",
       "      <td>[a39, a5600, a5601, a5602]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>p2009</td>\n",
       "      <td>[a76, a8168, a8169, a8170, a8171, a8172, a8173...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2479</th>\n",
       "      <td>p2490</td>\n",
       "      <td>[a10642, a10643]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2480</th>\n",
       "      <td>p2491</td>\n",
       "      <td>[a10644, a10645, a10646, a10647]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2481</th>\n",
       "      <td>p2493</td>\n",
       "      <td>[a10652, a10653, a10654, a10655, a10656, a1065...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2482</th>\n",
       "      <td>p2494</td>\n",
       "      <td>[a10670, a10671, a10672, a10673, a10674]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2483</th>\n",
       "      <td>p2499</td>\n",
       "      <td>[a10683]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2484 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     paperId                                           authorId\n",
       "0         p0  [a0, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, ...\n",
       "1        p13  [a0, a1, a2, a3, a4, a5, a6, a6, a7, a8, a9, a...\n",
       "2      p2039  [a15, a8363, a8365, a8366, a8369, a8372, a8402...\n",
       "3      p1191                         [a39, a5600, a5601, a5602]\n",
       "4      p2009  [a76, a8168, a8169, a8170, a8171, a8172, a8173...\n",
       "...      ...                                                ...\n",
       "2479   p2490                                   [a10642, a10643]\n",
       "2480   p2491                   [a10644, a10645, a10646, a10647]\n",
       "2481   p2493  [a10652, a10653, a10654, a10655, a10656, a1065...\n",
       "2482   p2494           [a10670, a10671, a10672, a10673, a10674]\n",
       "2483   p2499                                           [a10683]\n",
       "\n",
       "[2484 rows x 2 columns]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "paperAuthors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "fececacc-ce2f-4804-88f4-641879025863",
   "metadata": {},
   "outputs": [],
   "source": [
    "totalPapers = len(submittedPapersDF)\n",
    "set_authors = set(all_authors)\n",
    "\n",
    "ids = []\n",
    "submissions = []\n",
    "reviewers = []\n",
    "\n",
    "for i in range(totalPapers * 2):\n",
    "    ids.append('r'+str(i))\n",
    "    submissions.append(submittedPapersDF['submissionId'][i//2])\n",
    "    \n",
    "    curr_pId = submittedPapersDF['paperId'][i//2]\n",
    "    curr_authors = paperAuthors[paperAuthors['paperId'] == curr_pId]['authorId'].tolist()\n",
    "    \n",
    "    availableReviewers = [x for x in set_authors if not x in curr_authors]\n",
    "    reviewers.append(random.choice(availableReviewers))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "f79540b8-4cc8-456d-8365-44a67b35d28c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reviewer (author id: aId) reviews the submission (with id: sId)\n",
    "reviewersDF = pd.DataFrame(ids, columns = ['rId'])\n",
    "reviewersDF['submissionId'] = submissions\n",
    "reviewersDF['authorId'] = reviewers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "10215116-fc1f-4627-a0b1-4f1555324197",
   "metadata": {},
   "outputs": [],
   "source": [
    "reviewersDF = reviewersDF.merge(\n",
    "            authorsDF,\n",
    "            how = 'inner',\n",
    "            left_on = ['authorId'],\n",
    "            right_on = ['authorId']\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "fd50e8f7-7e2f-4e74-8211-a5a57c8f3b6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "reviewersDF = reviewersDF.merge(\n",
    "            submittedPapersDF,\n",
    "            how = 'inner',\n",
    "            left_on = ['submissionId'],\n",
    "            right_on = ['submissionId']\n",
    "            )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "de66b74c-4151-4a6d-90e1-e390ac125d0d",
   "metadata": {},
   "outputs": [],
   "source": [
    "reviewersDF.to_csv('data/reviewers.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "e6a50a0e-c0fd-47e6-8112-200e0a56095c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rId</th>\n",
       "      <th>submissionId</th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>authorName</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>r0</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1576</td>\n",
       "      <td>49051967</td>\n",
       "      <td>Gabriella M. Harari</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>r1</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1989</td>\n",
       "      <td>74025114</td>\n",
       "      <td>S. Managau</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>r2</td>\n",
       "      <td>s1</td>\n",
       "      <td>a3224</td>\n",
       "      <td>12184914</td>\n",
       "      <td>C. Ragin</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  rId submissionId authorId authorUrl           authorName paperId  \\\n",
       "0  r0           s0    a1576  49051967  Gabriella M. Harari      p0   \n",
       "1  r1           s0    a1989  74025114           S. Managau      p0   \n",
       "2  r2           s1    a3224  12184914             C. Ragin      p1   \n",
       "\n",
       "                                          paperTitle  \\\n",
       "0  Reproducible, interactive, scalable and extens...   \n",
       "1  Reproducible, interactive, scalable and extens...   \n",
       "2  High-Dimensional Probability: An Introduction ...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0                     Abstract content goes here ...                  j0   \n",
       "1                     Abstract content goes here ...                  j0   \n",
       "2  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0                               Nature Biotechnology               jv0   \n",
       "1                               Nature Biotechnology               jv0   \n",
       "2  International Conference on Exploring Services...              cp14   \n",
       "\n",
       "   decision  \n",
       "0  rejected  \n",
       "1  rejected  \n",
       "2  accepted  "
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviewersDF.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e2190c5-8469-4076-98fc-2b5a3f143fe2",
   "metadata": {},
   "source": [
    "### Creating chairs data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "270a6861-1335-477c-aeb8-64c0042db479",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>submissionId</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>s0</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>s1</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  submissionId paperId                                         paperTitle  \\\n",
       "0           s0      p0  Reproducible, interactive, scalable and extens...   \n",
       "1           s1      p1  High-Dimensional Probability: An Introduction ...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0                     Abstract content goes here ...                  j0   \n",
       "1  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0                               Nature Biotechnology               jv0   \n",
       "1  International Conference on Exploring Services...              cp14   \n",
       "\n",
       "   decision  \n",
       "0  rejected  \n",
       "1  accepted  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submittedPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "4e228bf4-e03f-406e-94d8-e1c18caddb4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "confPapers = submittedPapersDF[submittedPapersDF['conferenceJournalId'].str[:1] == 'c']['conferenceJournalId'].tolist()\n",
    "\n",
    "confPapersDF = pd.DataFrame(confPapers, columns = ['conferenceId'])\n",
    "\n",
    "ids = []\n",
    "for i in range(len(confPapersDF)):\n",
    "    ids.append('chair'+str(i))\n",
    "confPapersDF['chairId'] = ids\n",
    "\n",
    "confPapersDF['authorId'] = random.choices(authorsDF['authorId'], k = len(confPapersDF))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "298753ff-a935-45ae-8e42-a7c269ee550d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>conferenceId</th>\n",
       "      <th>chairId</th>\n",
       "      <th>authorId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>c14</td>\n",
       "      <td>chair0</td>\n",
       "      <td>a8258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c0</td>\n",
       "      <td>chair1</td>\n",
       "      <td>a9757</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  conferenceId chairId authorId\n",
       "0          c14  chair0    a8258\n",
       "1           c0  chair1    a9757"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aeac4843-9957-4068-b3a2-71610d5e20fd",
   "metadata": {},
   "source": [
    "It is assumed that any author from our database can chair any conference, irrespective of the number of papers written, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "d4420b72-37f7-49f3-a61c-5f1405a9ad9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "confPapersDF.to_csv('data/chairs.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "08fb001d-587c-4635-8d99-be7f97ee94f0",
   "metadata": {},
   "source": [
    "This implies that for author with x id chairs the conference y with c chairId or author x handles the conference y."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bc60d492-8dbf-45a5-8dd1-e0c27cb3f710",
   "metadata": {},
   "source": [
    "### Creating editors data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "6f525ccd-fb02-42cf-b24d-ee0a98fbddc1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>submissionId</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>s0</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>s1</td>\n",
       "      <td>p1</td>\n",
       "      <td>High-Dimensional Probability: An Introduction ...</td>\n",
       "      <td>© 2018, Cambridge University Press Let us summ...</td>\n",
       "      <td>c14</td>\n",
       "      <td>International Conference on Exploring Services...</td>\n",
       "      <td>cp14</td>\n",
       "      <td>accepted</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  submissionId paperId                                         paperTitle  \\\n",
       "0           s0      p0  Reproducible, interactive, scalable and extens...   \n",
       "1           s1      p1  High-Dimensional Probability: An Introduction ...   \n",
       "\n",
       "                                       paperAbstract conferenceJournalId  \\\n",
       "0                     Abstract content goes here ...                  j0   \n",
       "1  © 2018, Cambridge University Press Let us summ...                 c14   \n",
       "\n",
       "                              conferenceJournalTitle proceedingsVolume  \\\n",
       "0                               Nature Biotechnology               jv0   \n",
       "1  International Conference on Exploring Services...              cp14   \n",
       "\n",
       "   decision  \n",
       "0  rejected  \n",
       "1  accepted  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submittedPapersDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "id": "54f64a16-43f8-4f48-9097-e3712f092cc2",
   "metadata": {},
   "outputs": [],
   "source": [
    "jourPapers = submittedPapersDF[submittedPapersDF['conferenceJournalId'].str[:1] == 'j']['conferenceJournalId'].tolist()\n",
    "\n",
    "jourPapersDF = pd.DataFrame(jourPapers, columns = ['journalId'])\n",
    "\n",
    "ids = []\n",
    "for i in range(len(jourPapersDF)):\n",
    "    ids.append('editor'+str(i))\n",
    "jourPapersDF['editorId'] = ids\n",
    "\n",
    "jourPapersDF['authorId'] = random.choices(authorsDF['authorId'], k = len(jourPapersDF))\n",
    "jourPapersDF.to_csv('data/editors.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95e158da-8a70-430a-99db-2d5c07ac0318",
   "metadata": {},
   "source": [
    "It is assumed that any author from our database can chair any conference, irrespective of the number of papers written, etc."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "701a5b1b-65a3-4ded-8adb-dbf7c5a54ae4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>journalId</th>\n",
       "      <th>editorId</th>\n",
       "      <th>authorId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>j0</td>\n",
       "      <td>editor0</td>\n",
       "      <td>a6023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>j1</td>\n",
       "      <td>editor1</td>\n",
       "      <td>a6532</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>j2</td>\n",
       "      <td>editor2</td>\n",
       "      <td>a6892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>j3</td>\n",
       "      <td>editor3</td>\n",
       "      <td>a2485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>j4</td>\n",
       "      <td>editor4</td>\n",
       "      <td>a10028</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  journalId editorId authorId\n",
       "0        j0  editor0    a6023\n",
       "1        j1  editor1    a6532\n",
       "2        j2  editor2    a6892\n",
       "3        j3  editor3    a2485\n",
       "4        j4  editor4   a10028"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jourPapersDF.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5d08ae2-146f-4530-822d-7953e51e8b3d",
   "metadata": {},
   "source": [
    "This implies that for author with x id is the editor of the journal y with e editorId or author x handles the journal y."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "af715e07-e4bb-429f-9022-8e20696cdfad",
   "metadata": {},
   "outputs": [],
   "source": [
    "jourPapersDF.to_csv('data/editors.csv',index = False, header = True, mode = 'w')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b77de68-5d70-452d-a5f3-79e2c7ad8344",
   "metadata": {},
   "source": [
    "### HandlerReviewer Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "556fdc99-e838-47df-b255-ac6847bfd67a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>journalId</th>\n",
       "      <th>editorId</th>\n",
       "      <th>authorId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>j383</td>\n",
       "      <td>editor821</td>\n",
       "      <td>a6247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>822</th>\n",
       "      <td>j175</td>\n",
       "      <td>editor822</td>\n",
       "      <td>a5962</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    journalId   editorId authorId\n",
       "821      j383  editor821    a6247\n",
       "822      j175  editor822    a5962"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "jourPapersDF.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "2ec1b2ac-3205-4c29-b773-331a0248fc79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rId</th>\n",
       "      <th>submissionId</th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>authorName</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>r0</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1576</td>\n",
       "      <td>49051967</td>\n",
       "      <td>Gabriella M. Harari</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>r1</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1989</td>\n",
       "      <td>74025114</td>\n",
       "      <td>S. Managau</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  rId submissionId authorId authorUrl           authorName paperId  \\\n",
       "0  r0           s0    a1576  49051967  Gabriella M. Harari      p0   \n",
       "1  r1           s0    a1989  74025114           S. Managau      p0   \n",
       "\n",
       "                                          paperTitle  \\\n",
       "0  Reproducible, interactive, scalable and extens...   \n",
       "1  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "                    paperAbstract conferenceJournalId conferenceJournalTitle  \\\n",
       "0  Abstract content goes here ...                  j0   Nature Biotechnology   \n",
       "1  Abstract content goes here ...                  j0   Nature Biotechnology   \n",
       "\n",
       "  proceedingsVolume  decision  \n",
       "0               jv0  rejected  \n",
       "1               jv0  rejected  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviewersDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "4efc509e-fdf5-4300-a60b-58e92e608213",
   "metadata": {},
   "outputs": [],
   "source": [
    "handlerReviewersDF1 = pd.DataFrame(confPapersDF[['chairId']].values, columns = ['temp'])\n",
    "handlerReviewersDF1['confJour'] = confPapersDF['conferenceId']\n",
    "\n",
    "handlerReviewersDF2 = pd.DataFrame(jourPapersDF[['editorId']].values, columns = ['temp'])\n",
    "handlerReviewersDF2['confJour'] = jourPapersDF['journalId']\n",
    "\n",
    "handlerReviewersDF = pd.concat([handlerReviewersDF1,handlerReviewersDF2]).rename({'temp':'handlerId'}, axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "fc8a88b5-c2be-4221-b1e2-4563f17e94d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handlerId</th>\n",
       "      <th>confJour</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>821</th>\n",
       "      <td>editor821</td>\n",
       "      <td>j383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>822</th>\n",
       "      <td>editor822</td>\n",
       "      <td>j175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     handlerId confJour\n",
       "821  editor821     j383\n",
       "822  editor822     j175"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handlerReviewersDF.tail(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "d49e18f6-4f0b-4329-a5f2-2005712017e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rId</th>\n",
       "      <th>submissionId</th>\n",
       "      <th>authorId</th>\n",
       "      <th>authorUrl</th>\n",
       "      <th>authorName</th>\n",
       "      <th>paperId</th>\n",
       "      <th>paperTitle</th>\n",
       "      <th>paperAbstract</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>conferenceJournalTitle</th>\n",
       "      <th>proceedingsVolume</th>\n",
       "      <th>decision</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>r0</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1576</td>\n",
       "      <td>49051967</td>\n",
       "      <td>Gabriella M. Harari</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>r1</td>\n",
       "      <td>s0</td>\n",
       "      <td>a1989</td>\n",
       "      <td>74025114</td>\n",
       "      <td>S. Managau</td>\n",
       "      <td>p0</td>\n",
       "      <td>Reproducible, interactive, scalable and extens...</td>\n",
       "      <td>Abstract content goes here ...</td>\n",
       "      <td>j0</td>\n",
       "      <td>Nature Biotechnology</td>\n",
       "      <td>jv0</td>\n",
       "      <td>rejected</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  rId submissionId authorId authorUrl           authorName paperId  \\\n",
       "0  r0           s0    a1576  49051967  Gabriella M. Harari      p0   \n",
       "1  r1           s0    a1989  74025114           S. Managau      p0   \n",
       "\n",
       "                                          paperTitle  \\\n",
       "0  Reproducible, interactive, scalable and extens...   \n",
       "1  Reproducible, interactive, scalable and extens...   \n",
       "\n",
       "                    paperAbstract conferenceJournalId conferenceJournalTitle  \\\n",
       "0  Abstract content goes here ...                  j0   Nature Biotechnology   \n",
       "1  Abstract content goes here ...                  j0   Nature Biotechnology   \n",
       "\n",
       "  proceedingsVolume  decision  \n",
       "0               jv0  rejected  \n",
       "1               jv0  rejected  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviewersDF.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "1fcea1d6-b661-4c9e-ba58-f3f092dd38b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "handlerReviewersDF = handlerReviewersDF.merge(\n",
    "    right = reviewersDF,\n",
    "    how = 'inner',\n",
    "    left_on = 'confJour',\n",
    "    right_on = 'conferenceJournalId'\n",
    ")[['handlerId','conferenceJournalId','authorId']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "6f048967-7c41-4b2e-a0a1-46c66b3e015a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handlerId</th>\n",
       "      <th>conferenceJournalId</th>\n",
       "      <th>authorId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>chair0</td>\n",
       "      <td>c14</td>\n",
       "      <td>a3224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>chair0</td>\n",
       "      <td>c14</td>\n",
       "      <td>a7848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>chair0</td>\n",
       "      <td>c14</td>\n",
       "      <td>a9181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>chair0</td>\n",
       "      <td>c14</td>\n",
       "      <td>a8402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>chair0</td>\n",
       "      <td>c14</td>\n",
       "      <td>a2277</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  handlerId conferenceJournalId authorId\n",
       "0    chair0                 c14    a3224\n",
       "1    chair0                 c14    a7848\n",
       "2    chair0                 c14    a9181\n",
       "3    chair0                 c14    a8402\n",
       "4    chair0                 c14    a2277"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handlerReviewersDF.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a46e44dd-1552-4176-836a-de9efdea946e",
   "metadata": {},
   "outputs": [],
   "source": [
    "handlerReviewersDF.to_csv('data/handlerReviewers.csv',index = False, header = True, mode = 'w')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}