Corona.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Figure size 1600x600 with 0 Axes>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "import pandas as pd\n",
    "from datetime import datetime\n",
    "import re\n",
    "\n",
    "pd.set_option(\"display.max_rows\", 1000)\n",
    "\n",
    "\n",
    "sns.set(style=\"ticks\", color_codes=True)\n",
    "sns.set(rc={'figure.figsize':(30,15)})\n",
    "plt.figure(figsize=(16, 6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pathBase = \"./data/CORD-19-research-challenge\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Metadata-File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['.DS_Store', 'custom_license', 'metadata.readme', 'json_schema.txt', 'noncomm_use_subset', 'metadata.csv', 'biorxiv_medrxiv', 'COVID.DATA.LIC.AGMT.pdf', 'comm_use_subset']\n"
     ]
    }
   ],
   "source": [
    "dirs = os.listdir(pathBase)\n",
    "\n",
    "print(dirs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "rawMeta = pd.read_csv(pathBase + '/' + dirs[5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DateTime-Conversion Done!\n"
     ]
    }
   ],
   "source": [
    "# Parse Published Columns to dates\n",
    "\n",
    "dates = []\n",
    "\n",
    "dashNot = re.compile(\"^(19|20)\\d\\d[- /.](0[1-9]|1[012])[- /.](0[1-9]|[12][0-9]|3[01])$\")\n",
    "nameNot = re.compile(\"^(\\d\\d\\d\\d)\\s(\\w\\w\\w)\\s(\\d*)$\")\n",
    "yearmoNot = re.compile(\"^(\\d\\d\\d\\d)\\s(\\w\\w\\w)$\")\n",
    "\n",
    "for date in rawMeta[\"publish_time\"].values:\n",
    "    date = str(date) # Convert datee to string to avoid exceptions \n",
    "    tmp = None\n",
    "    \n",
    "    if(date == \"nan\"):\n",
    "        tmp = \"Unknown\"\n",
    "\n",
    "    # Match cases with dash notation e.g.: 1996-03-27\n",
    "    elif(dashNot.match(date)):\n",
    "        tmp = datetime.strptime(date, \"%Y-%m-%d\")\n",
    "\n",
    "    # Match cases with name notation e.g.: 2018 Jun 31\n",
    "    elif(nameNot.match(date)):\n",
    "        if(\"-\" in date):\n",
    "            date = date[:11].rstrip()\n",
    "            \n",
    "        # Try/Except is being used for invalid dates such as Feb 31\n",
    "        try:\n",
    "            tmp = datetime.strptime(date, \"%Y %b %d\")\n",
    "        except ValueError:\n",
    "            # Removes the Day and simply adds 01 as the day\n",
    "            tmp = datetime.strptime((date[:8] + \" 01\"), \"%Y %b %d\")\n",
    "\n",
    "    # Match cases with Year + Monthshorthand e.g.: 2007 May\n",
    "    elif(yearmoNot.match(date)):\n",
    "\n",
    "        if(\"-\" in date):\n",
    "            # ===== IS THIS REASONABLE? =====\n",
    "            # Fix the error-dates such as \"2006 Jun-Dec\" => \"2006 Dec\"\n",
    "            frmt = date.split(\"-\")\n",
    "            date = frmt[0][:5] + frmt[1]\n",
    "        tmp = datetime.strptime(date, \"%Y %b\")\n",
    "\n",
    "    # Match Year only\n",
    "    elif(len(date) == 4):\n",
    "        tmp = datetime.strptime(date, \"%Y\")\n",
    "\n",
    "    # Some dates come in \"string-array\"-notation:\n",
    "    #['2020-02-05', '2020-02'], ['2020-02-04', '2020-02'], ['2020-02-04', '2020-02'], ['2019-09-11', '2020']\n",
    "    elif(\"[]\" in date):\n",
    "        date = date[2:12]\n",
    "        tmp = datetime.strptime(date, \"%Y-%m-%d\")\n",
    "\n",
    "    dates.append(tmp)\n",
    "\n",
    "    \n",
    "print(\"DateTime-Conversion Done!\")\n",
    "   \n",
    "extract = {\"Reference-ID\": rawMeta[\"doi\"].values,\n",
    "           \"Title\": rawMeta[\"title\"].values,\n",
    "           \"Authors\": rawMeta[\"authors\"].values,\n",
    "           \"Abstract\": rawMeta[\"abstract\"].values,\n",
    "           \"Published\": dates,\n",
    "           \"Has_Fulltext\": rawMeta[\"has_full_text\"].values,\n",
    "           \"Directory\": rawMeta[\"full_text_file\"].values\n",
    "          }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 44220 entries, 0 to 44219\n",
      "Data columns (total 7 columns):\n",
      "Reference-ID    40750 non-null object\n",
      "Title           43996 non-null object\n",
      "Authors         41074 non-null object\n",
      "Abstract        35806 non-null object\n",
      "Published       43981 non-null object\n",
      "Has_Fulltext    44220 non-null bool\n",
      "Directory       32829 non-null object\n",
      "dtypes: bool(1), object(6)\n",
      "memory usage: 2.1+ MB\n"
     ]
    }
   ],
   "source": [
    "meta = pd.DataFrame(extract)\n",
    "\n",
    "meta.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta.Directory.fillna(\"n/a\", inplace=True)\n",
    "meta.Published.fillna(pd.NaT, inplace=True)\n",
    "\n",
    "# Drop all entries without Refernce ID, Title and publishing date;\n",
    "# Since these are virtually worth nothing (All properties = NaN)\n",
    "drop = meta.loc[(meta[\"Reference-ID\"].isna()) & (meta[\"Title\"].isna()) & (meta.Published == \"Unknown\")].index\n",
    "meta.drop(drop, inplace=True)\n",
    "\n",
    "meta.loc[(meta.Published == \"Unknown\")] = pd.NaT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Reference-ID</th>\n",
       "      <th>Title</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Published</th>\n",
       "      <th>Has_Fulltext</th>\n",
       "      <th>Directory</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3123</th>\n",
       "      <td>10.1016/B978-0-12-814966-9.00003-2</td>\n",
       "      <td>Chapter 3 Infectious Bronchitis Virus in Poult...</td>\n",
       "      <td>Ennaji, Youssef; Khataby, Khadija; Ennaji, Mou...</td>\n",
       "      <td>Abstract Infectious bronchitis virus (IBV) is ...</td>\n",
       "      <td>2020-12-31 00:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>custom_license</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3162</th>\n",
       "      <td>10.1016/B978-0-12-816331-3.00006-4</td>\n",
       "      <td>Chapter 6 Virus population dynamics examined w...</td>\n",
       "      <td>Domingo, Esteban</td>\n",
       "      <td>Abstract Experimental evolution permits explor...</td>\n",
       "      <td>2020-12-31 00:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>custom_license</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3066</th>\n",
       "      <td>10.1016/B978-0-12-811924-2.00019-5</td>\n",
       "      <td>Chapter 19 Current and New Approaches for Muco...</td>\n",
       "      <td>Rhee, Joon Haeng</td>\n",
       "      <td>Abstract Mucosal surfaces are the interface be...</td>\n",
       "      <td>2020-12-31 00:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>custom_license</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17773</th>\n",
       "      <td>10.1016/bs.ircmb.2019.10.004</td>\n",
       "      <td>Chapter Three Type I interferons and endoplasm...</td>\n",
       "      <td>Sprooten, Jenny; Garg, Abhishek D.</td>\n",
       "      <td>Abstract Type I interferons (IFNs) comprise of...</td>\n",
       "      <td>2020-12-31 00:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>custom_license</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17592</th>\n",
       "      <td>10.1016/bs.pmbts.2020.01.001</td>\n",
       "      <td>Chapter Seven Computer simulations of protein–...</td>\n",
       "      <td>Loschwitz, Jennifer; Olubiyi, Olujide O.; Hub,...</td>\n",
       "      <td>Abstract The interactions between proteins and...</td>\n",
       "      <td>2020-12-31 00:00:00</td>\n",
       "      <td>True</td>\n",
       "      <td>custom_license</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43550</th>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43557</th>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "      <td>NaT</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43671</th>\n",
       "      <td>10.1111/jvim.15548</td>\n",
       "      <td>Plasma and tissue angiotensin‐converting enzym...</td>\n",
       "      <td>Larouche‐Lebel, Éva; Loughran, Kerry A.; Oyama...</td>\n",
       "      <td>BACKGROUND: Angiotensin‐converting enzyme 2 (A...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>True</td>\n",
       "      <td>noncomm_use_subset</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43950</th>\n",
       "      <td>10.1111/jvim.15481</td>\n",
       "      <td>Efficacy of an orally administered anti‐diarrh...</td>\n",
       "      <td>Nixon, Sophie L.; Rose, Lindsay; Muller, Annik...</td>\n",
       "      <td>BACKGROUND: Acute diarrhea is a common clinica...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>True</td>\n",
       "      <td>noncomm_use_subset</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44191</th>\n",
       "      <td>10.1111/jvim.15386</td>\n",
       "      <td>Disease features of equine coronavirus and ent...</td>\n",
       "      <td>Manship, Arlie J.; Blikslager, Anthony T.; Elf...</td>\n",
       "      <td>BACKGROUND: Equine coronavirus (ECoV) is an em...</td>\n",
       "      <td>NaT</td>\n",
       "      <td>True</td>\n",
       "      <td>noncomm_use_subset</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>44150 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                             Reference-ID  \\\n",
       "3123   10.1016/B978-0-12-814966-9.00003-2   \n",
       "3162   10.1016/B978-0-12-816331-3.00006-4   \n",
       "3066   10.1016/B978-0-12-811924-2.00019-5   \n",
       "17773        10.1016/bs.ircmb.2019.10.004   \n",
       "17592        10.1016/bs.pmbts.2020.01.001   \n",
       "...                                   ...   \n",
       "43550                                 NaT   \n",
       "43557                                 NaT   \n",
       "43671                  10.1111/jvim.15548   \n",
       "43950                  10.1111/jvim.15481   \n",
       "44191                  10.1111/jvim.15386   \n",
       "\n",
       "                                                   Title  \\\n",
       "3123   Chapter 3 Infectious Bronchitis Virus in Poult...   \n",
       "3162   Chapter 6 Virus population dynamics examined w...   \n",
       "3066   Chapter 19 Current and New Approaches for Muco...   \n",
       "17773  Chapter Three Type I interferons and endoplasm...   \n",
       "17592  Chapter Seven Computer simulations of protein–...   \n",
       "...                                                  ...   \n",
       "43550                                                NaT   \n",
       "43557                                                NaT   \n",
       "43671  Plasma and tissue angiotensin‐converting enzym...   \n",
       "43950  Efficacy of an orally administered anti‐diarrh...   \n",
       "44191  Disease features of equine coronavirus and ent...   \n",
       "\n",
       "                                                 Authors  \\\n",
       "3123   Ennaji, Youssef; Khataby, Khadija; Ennaji, Mou...   \n",
       "3162                                    Domingo, Esteban   \n",
       "3066                                    Rhee, Joon Haeng   \n",
       "17773                 Sprooten, Jenny; Garg, Abhishek D.   \n",
       "17592  Loschwitz, Jennifer; Olubiyi, Olujide O.; Hub,...   \n",
       "...                                                  ...   \n",
       "43550                                                NaT   \n",
       "43557                                                NaT   \n",
       "43671  Larouche‐Lebel, Éva; Loughran, Kerry A.; Oyama...   \n",
       "43950  Nixon, Sophie L.; Rose, Lindsay; Muller, Annik...   \n",
       "44191  Manship, Arlie J.; Blikslager, Anthony T.; Elf...   \n",
       "\n",
       "                                                Abstract            Published  \\\n",
       "3123   Abstract Infectious bronchitis virus (IBV) is ...  2020-12-31 00:00:00   \n",
       "3162   Abstract Experimental evolution permits explor...  2020-12-31 00:00:00   \n",
       "3066   Abstract Mucosal surfaces are the interface be...  2020-12-31 00:00:00   \n",
       "17773  Abstract Type I interferons (IFNs) comprise of...  2020-12-31 00:00:00   \n",
       "17592  Abstract The interactions between proteins and...  2020-12-31 00:00:00   \n",
       "...                                                  ...                  ...   \n",
       "43550                                                NaT                  NaT   \n",
       "43557                                                NaT                  NaT   \n",
       "43671  BACKGROUND: Angiotensin‐converting enzyme 2 (A...                  NaT   \n",
       "43950  BACKGROUND: Acute diarrhea is a common clinica...                  NaT   \n",
       "44191  BACKGROUND: Equine coronavirus (ECoV) is an em...                  NaT   \n",
       "\n",
       "      Has_Fulltext           Directory  \n",
       "3123          True      custom_license  \n",
       "3162          True      custom_license  \n",
       "3066          True      custom_license  \n",
       "17773         True      custom_license  \n",
       "17592         True      custom_license  \n",
       "...            ...                 ...  \n",
       "43550          NaT                 NaT  \n",
       "43557          NaT                 NaT  \n",
       "43671         True  noncomm_use_subset  \n",
       "43950         True  noncomm_use_subset  \n",
       "44191         True  noncomm_use_subset  \n",
       "\n",
       "[44150 rows x 7 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "meta.sort_values(by=\"Published\", ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title: \n",
      "\n",
      "Chapter 3 Infectious Bronchitis Virus in Poultry: Molecular Epidemiology and Factors Leading to the Emergence and Reemergence of Novel Strains of Infectious Bronchitis Virus\n",
      "\n",
      "\n",
      "Abstract: \n",
      "\n",
      "Abstract Infectious bronchitis virus (IBV) is a coronavirus that causes an acute and highly contagious disease in chickens. The virus can cause substantial economic losses throughout the poultry industry worldwide. It can affect the upper respiratory tract and the reproductive tract, and some strains can cause nephritis. The causative agent IBV is an RNA virus with great ability for mutation and recombination, thus capable of generating new virus strains that are difficult to control. There are many IBV strains found worldwide, including the Massachusetts, 4/91, D274, and QX-like strains that can be grouped under the classic or variant serotypes. In addition, new types of the virus continue to arise due to mutations and recombination events in the viral genome and even more factors, making this virus difficult to identify and extremely difficult to control. Surveillance and identification of IBV types are extremely important for control of the disease and the advancement of molecular methods has aided in this pursuit. Genetic typing of IBV, which involves reverse transcription polymerase chain reaction amplification and sequence analysis of the S1 glycoprotein gene, has revolutionized diagnosis and identification of this virus by making it possible to type and compare the relatedness of a large number of virus isolates in a short period of time. Several conventional and molecular diagnostic methods have been described for the diagnosis of IB in chickens. “All-in/all-out” operations of rearing along with good biosafety measures form the basis of prevention, whereas vaccination forms the backbone of IB control program. Both live and inactivated (oil emulsified) conventional vaccines are available. The new generation vaccines (recombinant and vector-based) developed against locally prevailing IBV strains may be more helpful and avoid the reversion of virulence in live vaccine viruses.\n",
      "\n",
      "\n",
      "286 Words in Abstract.\n"
     ]
    }
   ],
   "source": [
    "x = meta.iloc[3123]\n",
    "tit = x.Title\n",
    "abst = x.Abstract\n",
    "\n",
    "print(\"Title: \\n\\n\" + tit)\n",
    "print(\"\\n\")\n",
    "print(\"Abstract: \\n\\n\" + abst)\n",
    "print(\"\\n\")\n",
    "\n",
    "splt = str.split(abst, \" \")\n",
    "print(str(len(splt)) + \" Words in Abstract.\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Analyze Titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Intrauterine',\n",
       " 'virus',\n",
       " 'infections',\n",
       " 'and',\n",
       " 'congenital',\n",
       " 'heart',\n",
       " 'disease']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "titles = meta.Title.values\n",
    "\n",
    "words = titles[0].split(\" \")\n",
    "\n",
    "words\n",
    "#for t in titles:\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use BioBERT for Stuff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize Tensorflow and BERT-Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Currently running Tensorflow v2.1.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow_hub as hub\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Model\n",
    "import tensorflow.keras as keras\n",
    "#import bert\n",
    "\n",
    "print(\"Currently running Tensorflow v\" + tf.version.VERSION)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "bertPath = \"./biobert/model/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TF-BERT Model ready to use!\n",
      "Model: \"model_3\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_word_ids (InputLayer)     [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_mask (InputLayer)         [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "segment_ids (InputLayer)        [(None, 128)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "keras_layer_4 (KerasLayer)      [(None, 768), (None, 109482241   input_word_ids[0][0]             \n",
      "                                                                 input_mask[0][0]                 \n",
      "                                                                 segment_ids[0][0]                \n",
      "==================================================================================================\n",
      "Total params: 109,482,241\n",
      "Trainable params: 109,482,240\n",
      "Non-trainable params: 1\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "max_seq_length = 128  # Your choice here.\n",
    "input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                       name=\"input_word_ids\")\n",
    "input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                   name=\"input_mask\")\n",
    "segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,\n",
    "                                    name=\"segment_ids\")\n",
    "bert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n",
    "                            trainable=True)\n",
    "pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n",
    "\n",
    "# model is now a Keras BERT-Model\n",
    "model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=[pooled_output, sequence_output])\n",
    "\n",
    "print(\"TF-BERT Model ready to use!\")\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import BioBERT-Weights (TODO)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "* Informations about checkpoint files:\n",
    " * The .meta-File is a protocol buffer which saves the complete Tensorflow Graph(Graph = Model?) \n",
    "     * This contains all variables, operations, collections, etc.\n",
    " * The .index-File seems to be referencing a decision tree about the file (????)\n",
    "     * \"Kinda source\": https://www.tensorflow.org/guide/checkpoint#loading_mechanics\n",
    " \n",
    "\n",
    "* Tensorflow >v1 seems to be missing the \"import_meta_graph()\" function\n",
    "    * The desired code would look like this: \"saver = tf.train.import_graph_def((bertPath + bioBertMeta))\"\n",
    "    * https://www.tensorflow.org/versions/r1.15/api_docs/python/tf/train/import_meta_graph\n",
    "\n",
    "    \n",
    "* How to import meta-graphs in Tensorflow v2?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'tensorflow.python.keras.engine.training.Model'>\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "SavedModel file does not exist at: ./biobert/model/model.ckpt-1000000.index/{saved_model.pbtxt|saved_model.pb}",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-159-772286613516>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;31m#bert = tf.train.load_checkpoint((bertPath + bertCheckpoint))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mtest\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msaved_model\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbertPath\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mbertCheckpoint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py\u001b[0m in \u001b[0;36mload\u001b[0;34m(export_dir, tags)\u001b[0m\n\u001b[1;32m    526\u001b[0m     \u001b[0mValueError\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mIf\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mtags\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mdon\u001b[0m\u001b[0;31m'\u001b[0m\u001b[0mt\u001b[0m \u001b[0mmatch\u001b[0m \u001b[0ma\u001b[0m \u001b[0mMetaGraph\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mSavedModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    527\u001b[0m   \"\"\"\n\u001b[0;32m--> 528\u001b[0;31m   \u001b[0;32mreturn\u001b[0m \u001b[0mload_internal\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexport_dir\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    530\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/tensorflow_core/python/saved_model/load.py\u001b[0m in \u001b[0;36mload_internal\u001b[0;34m(export_dir, tags, loader_cls)\u001b[0m\n\u001b[1;32m    535\u001b[0m     \u001b[0;31m# sequences for nest.flatten, so we put those through as-is.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    536\u001b[0m     \u001b[0mtags\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnest\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mflatten\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtags\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 537\u001b[0;31m   \u001b[0msaved_model_proto\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mloader_impl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse_saved_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexport_dir\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    538\u001b[0m   if (len(saved_model_proto.meta_graphs) == 1\n\u001b[1;32m    539\u001b[0m       and saved_model_proto.meta_graphs[0].HasField(\"object_graph_def\")):\n",
      "\u001b[0;32m/usr/local/lib/python3.7/site-packages/tensorflow_core/python/saved_model/loader_impl.py\u001b[0m in \u001b[0;36mparse_saved_model\u001b[0;34m(export_dir)\u001b[0m\n\u001b[1;32m     81\u001b[0m                   (export_dir,\n\u001b[1;32m     82\u001b[0m                    \u001b[0mconstants\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSAVED_MODEL_FILENAME_PBTXT\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m                    constants.SAVED_MODEL_FILENAME_PB))\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mOSError\u001b[0m: SavedModel file does not exist at: ./biobert/model/model.ckpt-1000000.index/{saved_model.pbtxt|saved_model.pb}"
     ]
    }
   ],
   "source": [
    "print(type(model))\n",
    "\n",
    "# Can't load the weights because the BioBERT Files aren't weights but TF (?) Checkpoints;\n",
    "# model.load_weights(\"./biobert/model/model.ckpt\")\n",
    "\n",
    "bertCheckpoint = \"model.ckpt-1000000.index\"\n",
    "bioBertMeta = \"model.ckpt-1000000.meta\"\n",
    "\n",
    "\n",
    "# What does this even do?\n",
    "#bert = tf.train.load_checkpoint((bertPath + bertCheckpoint))\n",
    "\n",
    "test = tf.saved_model.load((bertPath + bertCheckpoint))\n",
    "\n",
    "\n",
    "\n",
    "#with tf.Session() as sess:\n",
    "#       saver = tf.train.import_graph_def((bertPath + bioBertMeta))\n",
    "\n",
    "\n",
    "#bert = tf.train.import_meta_graph((bertPath + \"model.ckpt\"))\n",
    "\n",
    "#cp_callback = keras.callback.ModelCheckpoint(filepath=(bertPath + \"model.ckpt\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}