ProfNER.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Troy&AbedInTheMorning Team Participation in SMM4H-Spanish\n",
    "\n",
    "\n",
    "In these notebook we describe our proposed system based on an encoder-decoder architecture with an attention mechanism powered by a combination of word embeddings that include pre-trained fine-tuned spanish BERT embeddings.\n",
    "Our system serves as participation for ProfNER-ST focuses on the recognition of professions and occupations from Twitter using Spanish data.\n",
    "\n",
    "<a href=\"https://temu.bsc.es/smm4h-spanish/\">SMM4H-Spanish ProfNER-ST</a>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Settings\n",
    "\n",
    "1. Setting the parent dirrectory\n",
    "2. Checking GPU is avaible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "sst_home = '/home/sergio/Escritorio/ProfNER'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 13498016396183427099\n",
      ", name: \"/device:XLA_CPU:0\"\n",
      "device_type: \"XLA_CPU\"\n",
      "memory_limit: 17179869184\n",
      "locality {\n",
      "}\n",
      "incarnation: 4058650275646460332\n",
      "physical_device_desc: \"device: XLA_CPU device\"\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 3492282368\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 14382101316600260220\n",
      "physical_device_desc: \"device: 0, name: GeForce GTX 1050 Ti, pci bus id: 0000:01:00.0, compute capability: 6.1\"\n",
      ", name: \"/device:XLA_GPU:0\"\n",
      "device_type: \"XLA_GPU\"\n",
      "memory_limit: 17179869184\n",
      "locality {\n",
      "}\n",
      "incarnation: 2639639361459916750\n",
      "physical_device_desc: \"device: XLA_GPU device\"\n",
      "]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Requirements\n",
    "\n",
    "    Tensorflow 1.14.0\n",
    "    Keras 2.2.4\n",
    "    Fasttext 0.2.0\n",
    "    MeaningCloud-python\n",
    "    Spacy\n",
    "    Sklearn_crfsuite\n",
    "    Keras-contrib\n",
    "    Pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Format\n",
    "\n",
    "The data of the corpus was obtained from a Twitter crawl that used keywords like “Covid-19”, “epidemia” (epidemic) or “confinamiento” (lockdown), as well as hashtags such as “#yomequedoencasa” (#istayathome), to retrieve relevant tweets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cerramos nuestra querida Radio 😢 Nuestros colaboradores y conductores ¡Se quedan en casa! Desde mañana todos los programas de Radio Hoy se harán vía remoto X Skype. Seguimos al aire con el compromiso de siempre, nosotros apoyamos el #QuedateEnCasa #covıd19 #Coronavirus #RadioHoy https://t.co/Q81BBYpbdM\n"
     ]
    }
   ],
   "source": [
    "file_text = sst_home + '/profner-data/subtask-2/brat/train/1242399976644325376.txt' \n",
    "\n",
    "with open(file_text, 'r') as file:\n",
    "  print(file.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Annotations Format\n",
    "\n",
    "The corpus was annotated by linguist experts in an iterative process that included the creation of annotation guidelines specifically for this task. We could find the data annotated by BRAT and BIO format."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T1\tPROFESION 42 55\tcolaboradores\n",
      "T2\tPROFESION 58 69\tconductores\n",
      "\n"
     ]
    }
   ],
   "source": [
    "file_ann = sst_home + '/profner-data/subtask-2/brat/train/1242399976644325376.ann' \n",
    "\n",
    "with open(file_ann) as file:\n",
    "  print(file.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Requirements\n",
    "\n",
    "\n",
    "    Tensorflow 1.14.0\n",
    "    Keras 2.2.4\n",
    "    Fasttext 0.2.0\n",
    "    MeaningCloud-python\n",
    "    Spacy\n",
    "    Sklearn_crfsuite\n",
    "    Keras-contrib\n",
    "    Pandas\n",
    "    Pickle5\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Syllabizer\n",
    "\n",
    "Using the rules of Spanish orthography, the syllabizer allows a word to be broken into its component syllables. The necessary code has been extracted from https://github.com/mabodo/sibilizador/blob/master/Silabizator.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class char():\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "class char_line():\n",
    "    def __init__(self, word):\n",
    "        self.word = word\n",
    "        self.char_line = [(char, self.char_type(char)) for char in word]\n",
    "        self.type_line = ''.join(chartype for char, chartype in self.char_line)\n",
    "        \n",
    "    def char_type(self, char):\n",
    "        if char in set(['a', 'á', 'e', 'é','o', 'ó', 'í', 'ú']):\n",
    "            return 'V' #strong vowel\n",
    "        if char in set(['i', 'u', 'ü']):\n",
    "            return 'v' #week vowel\n",
    "        if char=='x':\n",
    "            return 'x'\n",
    "        if char=='s':\n",
    "            return 's'\n",
    "        else:\n",
    "            return 'c'\n",
    "            \n",
    "    def find(self, finder):\n",
    "        return self.type_line.find(finder)\n",
    "        \n",
    "    def split(self, pos, where):\n",
    "        return char_line(self.word[0:pos+where]), char_line(self.word[pos+where:])\n",
    "    \n",
    "    def split_by(self, finder, where):\n",
    "        split_point = self.find(finder)\n",
    "        if split_point!=-1:\n",
    "            chl1, chl2 = self.split(split_point, where)\n",
    "            return chl1, chl2\n",
    "        return self, False\n",
    "     \n",
    "    def __str__(self):\n",
    "        return self.word\n",
    "    \n",
    "    def __repr__(self):\n",
    "        return repr(self.word)\n",
    "\n",
    "class silabizer():\n",
    "    def __init__(self):\n",
    "        self.grammar = []\n",
    "        \n",
    "    def split(self, chars):\n",
    "        rules  = [('VV',1), ('cccc',2), ('xcc',1), ('ccx',2), ('csc',2), ('xc',1), ('cc',1), ('vcc',2), ('Vcc',2), ('sc',1), ('cs',1),('Vc',1), ('vc',1), ('Vs',1), ('vs',1)]\n",
    "        for split_rule, where in rules:\n",
    "            first, second = chars.split_by(split_rule,where)\n",
    "            if second:\n",
    "                if first.type_line in set(['c','s','x','cs']) or second.type_line in set(['c','s','x','cs']):\n",
    "                    #print 'skip1', first.word, second.word, split_rule, chars.type_line\n",
    "                    continue\n",
    "                if first.type_line[-1]=='c' and second.word[0] in set(['l','r']):\n",
    "                    continue\n",
    "                if first.word[-1]=='l' and second.word[-1]=='l':\n",
    "                    continue\n",
    "                if first.word[-1]=='r' and second.word[-1]=='r':\n",
    "                    continue\n",
    "                if first.word[-1]=='c' and second.word[-1]=='h':\n",
    "                    continue\n",
    "                return self.split(first)+self.split(second)\n",
    "        return [chars]\n",
    "        \n",
    "    def __call__(self, word):\n",
    "        return self.split(char_line(word))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Spacy Model\n",
    "\n",
    "The Spacy model selected is 'es_core_news_md'. Cause this model doesn't tokenize hashtags, a new component is added at the end of the pipeline (HashtagMerger)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: es_core_news_md==2.3.1 from https://github.com/explosion/spacy-models/releases/download/es_core_news_md-2.3.1/es_core_news_md-2.3.1.tar.gz#egg=es_core_news_md==2.3.1 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (2.3.1)\n",
      "Requirement already satisfied: spacy<2.4.0,>=2.3.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from es_core_news_md==2.3.1) (2.3.2)\n",
      "Requirement already satisfied: setuptools in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (49.1.0.post20200710)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (3.0.2)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.0.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (2.0.3)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (4.47.0)\n",
      "Requirement already satisfied: srsly<1.1.0,>=1.0.2 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.0.5)\n",
      "Requirement already satisfied: catalogue<1.1.0,>=0.0.7 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.0.0)\n",
      "Requirement already satisfied: numpy>=1.15.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.18.5)\n",
      "Requirement already satisfied: thinc==7.4.1 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (7.4.1)\n",
      "Requirement already satisfied: plac<1.2.0,>=0.9.6 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (0.9.6)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (2.24.0)\n",
      "Requirement already satisfied: blis<0.5.0,>=0.4.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (0.4.1)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.4.0 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (0.8.2)\n",
      "Requirement already satisfied: importlib-metadata>=0.20; python_version < \"3.8\" in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.7.0)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (1.25.9)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (3.0.4)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from requests<3.0.0,>=2.13.0->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (2020.6.20)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages (from importlib-metadata>=0.20; python_version < \"3.8\"->catalogue<1.1.0,>=0.0.7->spacy<2.4.0,>=2.3.0->es_core_news_md==2.3.1) (3.1.0)\n",
      "\u001b[38;5;2m✔ Download and installation successful\u001b[0m\n",
      "You can now load the model via spacy.load('es_core_news_md')\n"
     ]
    }
   ],
   "source": [
    "!python -m spacy download es_core_news_md\n",
    "\n",
    "import spacy\n",
    "import re\n",
    "import es_core_news_md\n",
    "\n",
    "nlp = es_core_news_md.load()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "from spacy.matcher import Matcher\n",
    "from spacy.tokens import Token\n",
    "\n",
    "# We're using a class because the component needs to be initialised with\n",
    "# the shared vocab via the nlp object\n",
    "class HashtagMerger(object):\n",
    "    def __init__(self, nlp):\n",
    "        # Register a new token extension to flag bad HTML\n",
    "        Token.set_extension(\"is_hashtag\", default=False)\n",
    "        self.matcher = Matcher(nlp.vocab)\n",
    "        # Add pattern for valid hashtag, i.e. '#' plus any ASCII token\n",
    "        self.matcher.add(\"HASHTAG\", None, [{\"ORTH\": \"#\"}, {\"TEXT\": {\"REGEX\": \"[A-Za-záéíóúÁÉÍÓÚ]+\"}}])\n",
    "\n",
    "    def __call__(self, doc):\n",
    "        # This method is invoked when the component is called on a Doc\n",
    "        matches = self.matcher(doc)\n",
    "        spans = []  # Collect the matched spans here\n",
    "        for match_id, start, end in matches:\n",
    "            spans.append(doc[start:end])\n",
    "        with doc.retokenize() as retokenizer:\n",
    "            for span in spans:\n",
    "                retokenizer.merge(span)\n",
    "                for token in span:\n",
    "                    token._.is_hashtag = True  # Mark token as bad HTML\n",
    "        return doc\n",
    "    \n",
    "hashtag_merger = HashtagMerger(nlp)\n",
    "nlp.add_pipe(hashtag_merger, last = True)  # Add component to the pipeline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BRAT to BIOES annotation\n",
    "\n",
    "The next code transform the BRAT annotations into BIOES format. The code return a dict with the token offset and the BIOES schema. In this schema,tokens are annotated using the following tags:\n",
    "\n",
    "\n",
    "    B: represents a token that conform the begining of an entity.\n",
    "    I: indicate that the token belongs to an entity.\n",
    "    O: represents that the token does not belong to an entity.\n",
    "    E: marks a token as the end of a given entity.\n",
    "    S: indicates that an entity is comprised of a single token.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "############# BIOES NOTATION #####################\n",
    "BEGIN = 'B'\n",
    "INSIDE = 'I'\n",
    "OUTSIDE = 'O'\n",
    "END = 'E'\n",
    "SINGLE = 'S'\n",
    "\n",
    "def getDictEntities(file_ann, ent_classes = ['PROFESION', 'SITUACION_LABORAL']):\n",
    "  entities = {}\n",
    "  with open(file_ann) as anns:\n",
    "    for ann in anns:\n",
    "      if ann.split('\\t')[1].split(' ')[0] in ent_classes:\n",
    "          ent = ann[:-1].split('\\t')[2]\n",
    "          #print(ent)\n",
    "          #ent = [token for token in nlp(ent) if not token.is_stop]\n",
    "          ent = nlp(ent)\n",
    "          start = int(ann[:-1].split('\\t')[1].split(' ')[1])\n",
    "          end = int(ann[:-1].split('\\t')[1].split(' ')[2])\n",
    "          if (len(ent) == 1):\n",
    "            entities[(start, end)] = SINGLE + '_' + ann.split('\\t')[1].split(' ')[0]\n",
    "          else:\n",
    "            entities[(start, start + len(ent[0].text))] = BEGIN + '_' + ann.split('\\t')[1].split(' ')[0]\n",
    "            entities[(end - len(ent[-1].text)), end] = END + '_' + ann.split('\\t')[1].split(' ')[0]\n",
    "            for i in range(len(ent) - 2):\n",
    "              spaces = (ent[i + 1].idx) - (ent[i].idx + len(ent[i].text))\n",
    "              start = start + len(ent[i].text) + spaces\n",
    "              entities[(start, start + len(ent[i + 1].text))] = INSIDE + '_' + ann.split('\\t')[1].split(' ')[0]\n",
    "            \n",
    "  return entities\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "def getElements(sst_home, max_len_seq, getTags = True):\n",
    "  _words = dict()\n",
    "  _doc_tags = {}\n",
    "  _entities = {}\n",
    "  _docs = {}\n",
    "  _docs_offset = {}\n",
    "    \n",
    "  for file in [file[:-4] for file in os.listdir(sst_home) if file.endswith('.txt') and not ' ' in file]:\n",
    "    file_text = os.path.join(sst_home, file + '.txt')\n",
    "    if getTags:\n",
    "      file_ann = os.path.join(sst_home, file + '.ann')\n",
    "      _entities = getDictEntities(file_ann)\n",
    "    with open(file_text) as f:\n",
    "      text = f.read()\n",
    "      spacy_text = nlp(text)\n",
    "    \n",
    "      _tweet = []\n",
    "      _tweet_tags = []\n",
    "      _tweet_pos = []\n",
    "      _tweet_gtags = []\n",
    "      _tweet_ent = []\n",
    "      _tweet_offset = []\n",
    "    \n",
    "      for token in spacy_text[0:max_len_seq]:\n",
    "          if not token.like_url:\n",
    "              _tweet.append(token.text)\n",
    "              _entity = _entities.get((token.idx, token.idx + len(token.text)), 'O')\n",
    "              _tweet_tags.append(_entity)\n",
    "              _words[token.text] = _words.get(token.text, 0) + 1\n",
    "              _tweet_pos.append(token.pos_)\n",
    "              _tweet_gtags.append(token.tag_)\n",
    "              _tweet_offset.append((token.text, token.idx, token.idx + len(token.text)))\n",
    "\n",
    "    _docs[file] = (_tweet, _tweet_pos, _tweet_gtags, _tweet_ent)\n",
    "    _doc_tags[file] = _tweet_tags\n",
    "    _docs_offset[file] = _tweet_offset\n",
    "\n",
    "  return _words, _docs, _doc_tags, _docs_offset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the 2idx\n",
    "\n",
    "Our model is based in a Bag of Words/Charachters/Syllables/PoS-Tags, so we need to create a dictionary to assign each of one see it during training to a cardinal id."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get2idx(words, chars, pos, gramm_tags, sylls):\n",
    "    \n",
    "  ET = ['B_SITUACION_LABORAL', 'B_PROFESION', \n",
    "        'I_SITUACION_LABORAL', 'I_PROFESION', \n",
    "        'E_SITUACION_LABORAL', 'E_PROFESION', \n",
    "        'S_SITUACION_LABORAL', 'S_PROFESION',\n",
    "        'O']\n",
    "\n",
    "  word2idx = {w: i + 2 for i, w in enumerate(words)}\n",
    "  word2idx[\"#ENDPAD\"] = 0\n",
    "  word2idx[\"UNK\"] = 1\n",
    "\n",
    "  char2idx = {char:i + 2 for i,char in enumerate(chars)}\n",
    "  char2idx[\"#ENDPAD\"] = 0  # to ignore this by mask_zero = True\n",
    "  char2idx[\"UNK\"] = 1\n",
    "\n",
    "  pos2idx = {pos:i + 2 for i,pos in enumerate(pos)}\n",
    "  pos2idx[\"#ENDPAD\"] = 0  # to ignore this by mask_zero = True\n",
    "  pos2idx[\"UNK\"] = 1\n",
    "\n",
    "  grammtags2idx = {tag:i + 2 for i,tag in enumerate(gramm_tags)}\n",
    "  grammtags2idx[\"#ENDPAD\"] = 0  # to ignore this by mask_zero = True\n",
    "  grammtags2idx[\"UNK\"] = 1\n",
    "\n",
    "  sylls2idx = {syll:i + 2 for i,syll in enumerate(sylls)}\n",
    "  sylls2idx[\"#ENDPAD\"] = 0  # to ignore this by mask_zero = True\n",
    "  sylls2idx[\"UNK\"] = 1\n",
    "\n",
    "  tag2idx = {tag:i + 1 for i,tag in enumerate(ET)}\n",
    "  tag2idx[\"#ENDPAD\"] = 0 \n",
    "\n",
    "  return word2idx, char2idx, tag2idx, pos2idx, grammtags2idx, sylls2idx"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the input Features\n",
    "\n",
    "The proposed system are feed with the next features:\n",
    "\n",
    "        Words: Two different 300 dimensional representations based on pre-trained word embeddings has been used with FastText. Both have been selected for their contribution of domain-specific knowledge since the former have been generated from Spanish medical corpora and the latter have been trained with Spanish Twitter data related to COVID-19. Contextual embeddings generated with a fine-tuned BETO model are also included, as these word representations are dynamically informed by the surrounding words improving performance.\n",
    "        \n",
    "        Part-of-speech: This feature has been considered due to the significant amount of information it offers about the word and its neighbors. It can also help in word sense disambiguation. The PoS-Tagging model used was the one provided by the Spacy. An embedding representaton of this feature is learned during training, resulting in a 40-dimensional vector.\n",
    "        \n",
    "        Characters: We also add character-level embeddings of the words, learned during training and resulting in a 30-dimensional vector. These have proven to be useful for specific-domain tasks and morphologically-rich languages.\n",
    "    \n",
    "       Syllables: Syllable-level embeddings of the words, learned during training and resulting in a 75-dimensional vector is also added. Like character-level embeddings, they help to deal with words outside the vocabulary and contribute to capturing common prefixes and suffixes in the domain and correctly classifying words.\n",
    "    \n",
    "       Cosine Similarity: The BETO embeddings of the entities found in the training and validation set are used to calculate the cosine similarity between the BETO representation of the word to be analyzed, since previous work has shown that could help to improve the results on data extracted from Twitter. This information is encoded as a 3717-dimensional vector. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "### CHARACTERS ###\n",
    "\n",
    "def getCharacterInput(sentences, max_seq_len, max_chars_len, char2idx):\n",
    "  X_char = []\n",
    "  for sentence in sentences:\n",
    "    sent_seq = []\n",
    "    for i in range(max_seq_len):\n",
    "      word_seq = []\n",
    "      # char sequence for words\n",
    "      for j in range(max_chars_len):\n",
    "        try:\n",
    "          # chars of specific sentence of i\n",
    "          word_seq.append(char2idx.get(sentence[i][j], 1)) \n",
    "        except:  # if char-sequence is out of range , pad it with \"PAD\" tag\n",
    "          word_seq.append(char2idx.get(\"#ENDPAD\"))\n",
    "\n",
    "      sent_seq.append(word_seq)\n",
    "    X_char.append(np.array(sent_seq))\n",
    "\n",
    "  return np.array(X_char)\n",
    "\n",
    "### SYLLABLES ###\n",
    "\n",
    "def getSyllsInput(sentences, max_seq_len, max_sylls_len, sylls2idx):\n",
    "  X_syll = []\n",
    "  for sentence in sentences:\n",
    "    sent_seq = []\n",
    "    for i in range(max_seq_len):\n",
    "      word_seq = []\n",
    "      syllables = []\n",
    "      try:\n",
    "        syllables = silabizer(sentence[i])\n",
    "      except:\n",
    "        pass\n",
    "      for j in range(max_sylls_len):\n",
    "        try:\n",
    "          word_seq.append(sylls2idx.get(str(syllables[j]).lower(), 1)) \n",
    "        except: \n",
    "          word_seq.append(sylls2idx.get(\"#ENDPAD\"))\n",
    "\n",
    "      sent_seq.append(word_seq)\n",
    "    X_syll.append(np.array(sent_seq))\n",
    "\n",
    "  return np.array(X_syll)\n",
    "\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "### WORDS ###\n",
    "\n",
    "def getWordInput(sentences, max_seq_len, word2idx):\n",
    "  X = [[word2idx.get(w,1) for w in s[:max_seq_len]] for s in sentences]\n",
    "  X = pad_sequences(maxlen = max_seq_len, sequences = X, truncating= 'post', padding ='post', value=0 )\n",
    "  return np.array(X)\n",
    "\n",
    "### PoS-Tags ###\n",
    "\n",
    "def getPosInput(sentences, max_seq_len, pos2idx):\n",
    "  X_pos = [[pos2idx.get(w,1) for w in s] for s in sentences]\n",
    "  X_pos = pad_sequences(maxlen = max_seq_len, sequences = X_pos, truncating= 'post', padding ='post', value=0 )\n",
    "  return np.array(X_pos)\n",
    "\n",
    "def getGTagInput(sentences, max_seq_len, grammtags2idx):\n",
    "  X_gramm_tags = [[grammtags2idx.get(w,1) for w in s] for s in sentences]\n",
    "  X_gramm_tags = pad_sequences(maxlen = max_seq_len, sequences = X_gramm_tags, truncating= 'post', padding ='post', value=0 )\n",
    "  return np.array(X_gramm_tags)\n",
    "\n",
    "from keras.utils.np_utils import to_categorical\n",
    "\n",
    "### NER TAG ###\n",
    "def getY(tags, max_seq_len, tag2idx):\n",
    "  y = [[tag2idx[t] for t in sent_tags] for sent_tags in tags]\n",
    "  y = pad_sequences(maxlen = max_seq_len, sequences = y, padding = \"post\", truncating = \"post\", value = tag2idx[\"#ENDPAD\"])\n",
    "  \n",
    "  y = [to_categorical(i, num_classes = len(tag2idx)) for i in y]\n",
    "  return np.array(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Getting the total words\n",
    "\n",
    "Cause we are using pre-trained embeddings, we use as our vocabulary the words contained in both the training, development and test sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "_total_words = set()\n",
    "sst_home_train = sst_home + '/profner-data/subtask-2/brat/train'\n",
    "for file in [file[:-4] for file in os.listdir(sst_home_train) if file.endswith('.txt') and not ' ' in file]:\n",
    "    file_text = os.path.join(sst_home_train, file + '.txt')\n",
    "    \n",
    "    with open(file_text) as f:\n",
    "        text = f.read()\n",
    "        doc = nlp(text)\n",
    "        for token in doc:\n",
    "            _total_words.add(token.text)\n",
    "           \n",
    "sst_home_test = sst_home + '/profner-data/subtask-2/brat/valid'\n",
    "for file in [file[:-4] for file in os.listdir(sst_home_test) if file.endswith('.txt') and not ' ' in file]:\n",
    "    file_text = os.path.join(sst_home_test, file + '.txt')\n",
    "    \n",
    "    with open(file_text) as f:\n",
    "        text = f.read()\n",
    "        doc = nlp(text)\n",
    "        for token in doc:\n",
    "            _total_words.add(token.text)\n",
    "\n",
    "'''\n",
    "sst_home_test = sst_home + '/profner-data/subtask-2/test-background-txt-files'\n",
    "for file in [file[:-4] for file in os.listdir(sst_home_test) if file.endswith('.txt') and not ' ' in file]:\n",
    "    file_text = os.path.join(sst_home_test, file + '.txt')\n",
    "    \n",
    "    with open(file_text) as f:\n",
    "        text = f.read()\n",
    "        doc = nlp(text)\n",
    "        for token in doc:\n",
    "            if not token.is_stop:\n",
    "                _total_words.add(token.text)\n",
    "'''\n",
    "\n",
    "_words = list(_total_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Generating Training Data\n",
    "\n",
    "The next code generate the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "sst_home_train = sst_home + '/profner-data/subtask-2/brat/train'\n",
    "\n",
    "MAX_CHARS_LEN = 25\n",
    "MAX_SEQ_LEN = 75\n",
    "MAX_SYLLS_LEN = 10\n",
    "\n",
    "silabizer = silabizer()\n",
    "\n",
    "words_tr, docs, docs_tags, _ = getElements(sst_home_train, MAX_SEQ_LEN)\n",
    "words_tr = list(words_tr.keys())\n",
    "chars = list(set(''.join(words_tr)))\n",
    "sylls = list(set(str(syll).lower() for word in words_tr for syll in silabizer(word)))\n",
    "\n",
    "tweets_train = [tweet[0] for tweet in docs.values()]\n",
    "tweets_pos = [tweet[1] for tweet in docs.values()]\n",
    "tweets_gtags = [tweet[2] for tweet in docs.values()]\n",
    "\n",
    "all_pos = set(pos for sent in tweets_pos for pos in sent)\n",
    "all_gtags = set(gtag for sent in tweets_gtags for gtag in sent)\n",
    "\n",
    "tags = [tags for tags in docs_tags.values()]\n",
    "\n",
    "word2idx, char2idx, tag2idx, pos2idx, grammtags2idx, sylls2idx = get2idx(_words, chars, all_pos, all_gtags, sylls)\n",
    "\n",
    "X_char_train = getCharacterInput(tweets_train, MAX_SEQ_LEN, MAX_CHARS_LEN, char2idx)\n",
    "X_words_train = getWordInput(tweets_train, MAX_SEQ_LEN, word2idx)\n",
    "X_pos_train = getPosInput(tweets_pos, MAX_SEQ_LEN, pos2idx)\n",
    "X_tag_train = getGTagInput(tweets_gtags, MAX_SEQ_LEN, grammtags2idx)\n",
    "X_syll_train = getSyllsInput(tweets_train, MAX_SEQ_LEN, MAX_SYLLS_LEN, sylls2idx)\n",
    "\n",
    "y_train = getY(tags, MAX_SEQ_LEN, tag2idx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT Embeddings\n",
    "\n",
    "We load the bert embeddings generated from the other notebook "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle5 as pickle\n",
    "import numpy as np\n",
    "\n",
    "BERT_DIM = 1536\n",
    "\n",
    "train_bert = {}\n",
    "with open(sst_home + '/saved_data/bert_train.pickle', 'rb') as handle:\n",
    "    train_bert = pickle.load(handle)\n",
    "\n",
    "X_train_bert = []\n",
    "sst_home_train = sst_home + '/profner-data/subtask-2/brat/train'\n",
    "\n",
    "for file in [file[:-4] for file in os.listdir(sst_home_train) if file.endswith('.txt') and not ' ' in file]:\n",
    "    bert_vector = train_bert[file]\n",
    "    X_train_bert.append(bert_vector)\n",
    "X_train_bert = pad_sequences(maxlen = 75, sequences = X_train_bert, truncating= 'post', padding ='post', value=np.zeros(BERT_DIM))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Validation Data\n",
    "\n",
    "The next code generate the validation data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "MAX_CHARS_LEN = 25\n",
    "MAX_SEQ_LEN = 75\n",
    "MAX_SYLLS_LEN = 10\n",
    "\n",
    "sst_home_test = sst_home + '/profner-data/subtask-2/brat/valid'\n",
    "\n",
    "_, docs, docs_tags, docs_offset = getElements(sst_home_test, MAX_SEQ_LEN, getTags=True)\n",
    "\n",
    "tweets_test = [tweet[0] for tweet in docs.values()]\n",
    "tweets_pos = [tweet[1] for tweet in docs.values()]\n",
    "tweets_gtags = [tweet[2] for tweet in docs.values()]\n",
    "\n",
    "\n",
    "tags_test = [tags for tags in docs_tags.values()]\n",
    "\n",
    "X_char_test = getCharacterInput(tweets_test, MAX_SEQ_LEN, MAX_CHARS_LEN, char2idx)\n",
    "X_words_test = getWordInput(tweets_test, MAX_SEQ_LEN, word2idx)\n",
    "X_pos_test = getPosInput(tweets_pos, MAX_SEQ_LEN, pos2idx)\n",
    "X_tag_test = getGTagInput(tweets_gtags, MAX_SEQ_LEN, grammtags2idx)\n",
    "X_syll_test = getSyllsInput(tweets_test, MAX_SEQ_LEN, MAX_SYLLS_LEN, sylls2idx)\n",
    "\n",
    "y_test = getY(tags_test, MAX_SEQ_LEN, tag2idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_bert = {}\n",
    "with open(sst_home + '/saved_data/bert_valid.pickle', 'rb') as handle:\n",
    "    test_bert = pickle.load(handle)\n",
    "    \n",
    "def getBertInput(file_name):\n",
    "    bert_vector = test_bert[file_name]\n",
    "    X_test_bert = [bert_vector]\n",
    "    X_test_bert = pad_sequences(maxlen = 75, sequences = X_test_bert, truncating= 'post', padding ='post', value=np.zeros(1536))\n",
    "    return X_test_bert\n",
    "\n",
    "X_test_bert = []\n",
    "sst_home_train = sst_home + '/profner-data/subtask-2/brat/valid'\n",
    "for file in [file[:-4] for file in os.listdir(sst_home_train) if file.endswith('.txt') and not ' ' in file]:\n",
    "    bert_vector = test_bert[file]\n",
    "    X_test_bert.append(bert_vector)\n",
    "X_test_bert = pad_sequences(maxlen = 75, sequences = X_test_bert, truncating= 'post', padding ='post', value=np.zeros(BERT_DIM))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Word embeddings from Spanish Medical Corpora & Twitter\n",
    "\n",
    "In this model we'll use two different pre-trained word embeddings:\n",
    "\n",
    "1. The Word Embeddings from Spanish Medical Corpora can be found in https://www.aclweb.org/anthology/W19-1916/\n",
    "2. The Word Embedding from Spanish Twitter (Covid-19) can be found in https://zenodo.org/record/4449930#.YBbYOtaCE5k"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fasttext\n",
    "import fasttext.util\n",
    "\n",
    "def getEmbeddingMatrix(words2idx, emb_dim, model):\n",
    "  embedding_matrix = np.zeros((len(words2idx), emb_dim))\n",
    "  for word, i in words2idx.items():\n",
    "    embedding_matrix[i] = model[word]\n",
    "\n",
    "  return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n",
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "sst_home_embeddings = sst_home + '/fast-text-model/'\n",
    "\n",
    "### SPANISH MEDICAL CORPORA ###\n",
    "ft = fasttext.load_model(sst_home_embeddings + 'cantemist-resource.bin')\n",
    "medical_embedding_matrix = getEmbeddingMatrix(word2idx, ft.get_dimension(), ft)\n",
    "del ft\n",
    "\n",
    "### SPANISH TWITTER COVID-19 ###\n",
    "ft = fasttext.load_model(sst_home_embeddings + 'covid_19_es_twitter_cbow_cased.bin')\n",
    "twitter_embedding_matrix = getEmbeddingMatrix(word2idx, ft.get_dimension(), ft)\n",
    "del ft"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving Data\n",
    "\n",
    "We save the generated data so that it is not necessary to generate it again next time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle5 as pickle\n",
    "\n",
    "train_data = [X_char_train, X_words_train, X_pos_train, X_tag_train, X_syll_train, X_train_bert, y_train]\n",
    "with open(sst_home + '/saved_data/train.pickle', 'wb') as handle:\n",
    "    pickle.dump(train_data, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "\n",
    "test_data = [X_char_test, X_words_test, X_pos_test, X_tag_test, X_syll_test, X_test_bert, y_test]\n",
    "with open(sst_home + '/saved_data/test.pickle', 'wb') as handle:\n",
    "    pickle.dump(test_data, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "\n",
    "embedding_matrix = [medical_embedding_matrix, twitter_embedding_matrix]   \n",
    "with open(sst_home + '/saved_data/we.pickle', 'wb') as handle:\n",
    "    pickle.dump(embedding_matrix, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "two2idx = [word2idx, char2idx, tag2idx, pos2idx, grammtags2idx, sylls2idx]\n",
    "with open(sst_home + '/saved_data/2idx.pickle', 'wb') as handle:\n",
    "    pickle.dump(two2idx, handle, protocol=pickle.HIGHEST_PROTOCOL)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Data\n",
    "\n",
    "In case the data is previously saved, we load these."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle5 as pickle\n",
    "\n",
    "X_train = []\n",
    "with open(sst_home + '/saved_data/train.pickle', 'rb') as handle:\n",
    "    X_train = pickle.load(handle)\n",
    "\n",
    "X_test = []\n",
    "with open(sst_home + '/saved_data/test.pickle', 'rb') as handle:\n",
    "    X_test = pickle.load(handle)\n",
    "\n",
    "embedding_matrixes = []\n",
    "with open(sst_home + '/saved_data/we.pickle', 'rb') as handle:\n",
    "    embedding_matrix = pickle.load(handle)\n",
    "\n",
    "two2idx = []\n",
    "with open(sst_home + '/saved_data/2idx.pickle', 'rb') as handle:\n",
    "    two2idx = pickle.load(handle)\n",
    "    \n",
    "\n",
    "\n",
    "X_char_train, X_words_train, X_pos_train, X_tag_train, X_syll_train,  X_train_bert, y_train = X_train\n",
    "X_char_test, X_words_test, X_pos_test, X_tag_test, X_syll_test, X_test_bert, y_test = X_test\n",
    "word2idx, char2idx, tag2idx, pos2idx, grammtags2idx, sylls2idx = two2idx\n",
    "medical_embedding_matrix, twitter_embedding_matrix = embedding_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## BERT Embeddings Entities\n",
    "\n",
    "The BERT embeddings of the entities founded in train corpus are loaded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pickle5 as pickle\n",
    "import numpy as np\n",
    "\n",
    "bert_entities = []\n",
    "with open(sst_home + '/saved_data/bert_entities.pickle', 'rb') as handle:\n",
    "    bert_entities = pickle.load(handle)\n",
    "    \n",
    "bert_entities_ = np.array(bert_entities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Learning Architecture\n",
    "\n",
    "In the proposed system, the character and syllable information is previously proccesed by a convolutional and global max pooling block, to be concatenated with the rest of the input features to serve as input to an encoder-decoder architecture with attention mechanism. The context vector as well as decoder outputs feeds a fully connected dense layer with $tanh$ activation function. The last layer (CRF optimization layer) consists of a conditional random fields layer selected due to the ability of the layer to take into account the dependencies between the different labels.  The output of this layer provides the most probable sequence of labels.\n",
    "\n",
    "![Architecture of the proposed model for pro-fession and occupations recognition.](./imgs/model2.png)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install git+https://www.github.com/keras-team/keras-contrib.git"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cosine Similarity Layer\n",
    "\n",
    "A custom Keras layer is created to calculate Cosine Similarity between BERT Word Embeddings and Entities found in corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Layer\n",
    "from keras.backend import constant\n",
    "\n",
    "class CosineSimilarity(Layer):\n",
    "    def __init__(self):\n",
    "        super(CosineSimilarity, self).__init__()\n",
    "        self.result = None\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        entities = constant(bert_entities_)\n",
    "        bert_input = inputs\n",
    "        norm_entities_bert = tf.norm(entities, axis = 1)\n",
    "        norm_bert_input = tf.norm(bert_input, axis =  2)\n",
    "\n",
    "        cosine = tf.einsum('nd,bmd->bmn', entities, bert_input)\n",
    "        norm = tf.einsum('bm,n->bmn',norm_bert_input, norm_entities_bert)\n",
    "\n",
    "        self.result = tf.math.divide_no_nan(cosine, norm)\n",
    "        \n",
    "        return self.result\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return [(None, 75, 2769)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/keras_contrib/layers/crf.py:346: UserWarning: CRF.loss_function is deprecated and it might be removed in the future. Please use losses.crf_loss instead.\n",
      "  warnings.warn('CRF.loss_function is deprecated '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/keras/optimizers.py:790: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:2403: add_dispatch_support.<locals>.wrapper (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/keras_contrib/layers/crf.py:353: UserWarning: CRF.accuracy is deprecated and it might be removed in the future. Please use metrics.crf_accuracy\n",
      "  warnings.warn('CRF.accuracy is deprecated and it '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "char_input (InputLayer)         (None, 75, 25)       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sylls_input (InputLayer)        (None, 75, 10)       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_9 (TimeDistrib (None, 75, 25, 30)   17760       char_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_12 (TimeDistri (None, 75, 10, 75)   609900      sylls_input[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_10 (TimeDistri (None, 75, 23, 50)   4550        time_distributed_9[0][0]         \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_13 (TimeDistri (None, 75, 8, 50)    11300       time_distributed_12[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "bert_input (InputLayer)         (None, 75, 1536)     0                                            \n",
      "__________________________________________________________________________________________________\n",
      "pos_input (InputLayer)          (None, 75)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "tag_input (InputLayer)          (None, 75)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_11 (TimeDistri (None, 75, 23, 50)   0           time_distributed_10[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_14 (TimeDistri (None, 75, 8, 50)    0           time_distributed_13[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "word_input (InputLayer)         (None, 75)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 75, 1536)     2360832     bert_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "pos_embeddings (Embedding)      (None, 75, 20)       380         pos_input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "gtags_embeddings (Embedding)    (None, 75, 20)       6640        tag_input[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling (TimeDistributed)   (None, 75, 50)       0           time_distributed_11[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_15 (TimeDistri (None, 75, 50)       0           time_distributed_14[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "medical_word_embeddings (Embedd (None, 75, 300)      11853600    word_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "twitter_word_embeddings (Embedd (None, 75, 300)      11853600    word_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_10 (Dropout)            (None, 75, 1536)     0           dense_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_8 (Dropout)             (None, 75, 20)       0           pos_embeddings[0][0]             \n",
      "__________________________________________________________________________________________________\n",
      "dropout_9 (Dropout)             (None, 75, 20)       0           gtags_embeddings[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "char_enc (TimeDistributed)      (None, 75, 50)       0           max_pooling[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "time_distributed_16 (TimeDistri (None, 75, 50)       0           time_distributed_15[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "cosine_similarity_2 (CosineSimi (None, 75, 2769)     0           bert_input[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 75, 5045)     0           medical_word_embeddings[0][0]    \n",
      "                                                                 twitter_word_embeddings[0][0]    \n",
      "                                                                 dropout_10[0][0]                 \n",
      "                                                                 dropout_8[0][0]                  \n",
      "                                                                 dropout_9[0][0]                  \n",
      "                                                                 char_enc[0][0]                   \n",
      "                                                                 time_distributed_16[0][0]        \n",
      "                                                                 cosine_similarity_2[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "bidirectional_2 (Bidirectional) [(None, 75, 300), (N 6235200     concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)     (None, 300)          0           bidirectional_2[0][1]            \n",
      "                                                                 bidirectional_2[0][3]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_4 (Concatenate)     (None, 300)          0           bidirectional_2[0][2]            \n",
      "                                                                 bidirectional_2[0][4]            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_3 (LSTM)                   [(None, 75, 300), (N 6415200     concatenate_2[0][0]              \n",
      "                                                                 concatenate_3[0][0]              \n",
      "                                                                 concatenate_4[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 75, 75)       0           lstm_3[0][0]                     \n",
      "                                                                 bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 75, 75)       0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "dot_2 (Dot)                     (None, 75, 300)      0           activation_1[0][0]               \n",
      "                                                                 bidirectional_2[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_5 (Concatenate)     (None, 75, 600)      0           dot_2[0][0]                      \n",
      "                                                                 lstm_3[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_11 (Dropout)            (None, 75, 600)      0           concatenate_5[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 75, 200)      120200      dropout_11[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "crf_1 (CRF)                     (None, 75, 10)       2130        dense_3[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 39,491,292\n",
      "Trainable params: 39,491,292\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "from keras.models import Model\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Concatenate, Input, SpatialDropout1D\n",
    "from keras.layers import Conv1D, MaxPooling1D,Flatten,GlobalMaxPooling1D, Reshape, RepeatVector,  Dot, GRU, Activation\n",
    "from keras_contrib.layers import CRF\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.backend import constant, variable\n",
    "\n",
    "CHAR_EMBEDDINGS_SIZE = 30    # Characters Embeddings Size\n",
    "SYLL_EMBEDDINGS_SIZE = 75    # Syllable Embeddings Size\n",
    "WORD_EMBEDDINGS_SIZE = 300   # Word Embeddings Size\n",
    "POS_EMBEDDING_SIZE = 20      # PoS Embedding Size\n",
    "GTAGS_EMBEDDING_SIZE = 20    # Tag Embedding Size\n",
    "\n",
    "MAX_CHARS_LEN = 25           # Max sequence char length\n",
    "MAX_SEQ_LEN = 75             # Max sequence word length Level II\n",
    "MAX_SYLLS_LEN = 10           # Max sequence sylls length\n",
    "\n",
    "CONV_FILTERS = 50            # Convolutional Filters in Character and Syllable Convolutional Layer\n",
    "LSTM_UNITS = 300             # LSTM Units in both LSTM layers\n",
    "DENSE_UNITS = 200            # Number of units in Dense layer  \n",
    "\n",
    "BERT_DIM = 1536\n",
    "\n",
    "word_input = Input(shape=(MAX_SEQ_LEN,), name = 'word_input')\n",
    "pos_input = Input(shape=(MAX_SEQ_LEN,), name = 'pos_input')\n",
    "gtag_input = Input(shape=(MAX_SEQ_LEN,), name = 'tag_input')\n",
    "char_input = Input(shape=(MAX_SEQ_LEN, MAX_CHARS_LEN), name = 'char_input')\n",
    "sylls_input = Input(shape=(MAX_SEQ_LEN, MAX_SYLLS_LEN), name = 'sylls_input')\n",
    "bert_input = Input(shape=(MAX_SEQ_LEN, BERT_DIM,), name = 'bert_input')\n",
    "\n",
    "###################### COSINE SIMILARITY ##################################################################\n",
    "cosine_bert = CosineSimilarity()(bert_input)\n",
    "\n",
    "###################### CHARACTER SEQUENCE PROCCESSED BY CONVOLUTIONAL LAYER ################################\n",
    "char_embedding = TimeDistributed(Embedding(input_dim = len(char2idx), output_dim = CHAR_EMBEDDINGS_SIZE, input_length=MAX_CHARS_LEN, name = 'char_embeddings', trainable = True))(char_input)\n",
    "conv_1d = TimeDistributed(Conv1D(filters = CONV_FILTERS, kernel_size = 3,padding = \"valid\", activation = \"relu\", name=\"Conv1D_char\"))(char_embedding)\n",
    "conv_1d = TimeDistributed(Dropout(0.4))(conv_1d)\n",
    "maxpool1d = TimeDistributed(GlobalMaxPooling1D(), name = 'max_pooling')(conv_1d)\n",
    "char_enc = TimeDistributed(Flatten(), name = 'char_enc')(maxpool1d)\n",
    "\n",
    "###################### SYLLABLE SEQUENCE PROCCESSED BY CONVOLUTIONAL LAYER ################################\n",
    "syll_embedding = TimeDistributed(Embedding(input_dim = len(sylls2idx), output_dim = SYLL_EMBEDDINGS_SIZE, input_length=MAX_SYLLS_LEN, name = 'sylls_embeddings', trainable = True))(sylls_input)\n",
    "conv_1d_syll = TimeDistributed(Conv1D(filters = CONV_FILTERS, kernel_size = 3,padding=\"valid\", activation=\"relu\", name=\"Conv1D_syll\"))(syll_embedding)\n",
    "conv_1d_syll = TimeDistributed(Dropout(0.4))(conv_1d_syll)\n",
    "maxpool1d_syll = TimeDistributed(GlobalMaxPooling1D())(conv_1d_syll)\n",
    "syll_enc = TimeDistributed(Flatten())(maxpool1d_syll)\n",
    "\n",
    "##################### PoS + TAG EMBEDDINGS ################################################################\n",
    "pos_embedding = Embedding(input_dim = len(pos2idx), output_dim = POS_EMBEDDING_SIZE, input_length = MAX_SEQ_LEN, name = 'pos_embeddings', trainable = True)(pos_input)\n",
    "pos_embedding = Dropout(0.4)(pos_embedding)\n",
    "gtags_embedding = Embedding(input_dim = len(grammtags2idx), output_dim = GTAGS_EMBEDDING_SIZE, input_length = MAX_SEQ_LEN, name = 'gtags_embeddings', trainable = True)(gtag_input)\n",
    "gtags_embedding = Dropout(0.4)(gtags_embedding)\n",
    "\n",
    "##################### WORD EMBEDDING LAYER ################################################################\n",
    "medical_embedding_layer = Embedding(input_dim = len(word2idx), output_dim = WORD_EMBEDDINGS_SIZE, weights=[medical_embedding_matrix], trainable=True, name = 'medical_word_embeddings')\n",
    "medical_word_embedding = medical_embedding_layer(word_input)\n",
    "\n",
    "twitter_embedding_layer = Embedding(input_dim = len(word2idx), output_dim = WORD_EMBEDDINGS_SIZE, weights=[twitter_embedding_matrix], trainable=True, name = 'twitter_word_embeddings')\n",
    "twitter_word_embedding = twitter_embedding_layer(word_input)\n",
    "\n",
    "bert_out = Dense(BERT_DIM, activation='relu')(bert_input)\n",
    "bert_out = Dropout(0.4)(bert_out)\n",
    "\n",
    "################### CONCATENATE INPUT FEATURES ###########################################\n",
    "x = Concatenate(axis = -1)([medical_word_embedding, twitter_word_embedding, bert_out, pos_embedding, gtags_embedding, char_enc, syll_enc, cosine_bert])\n",
    "\n",
    "################## BiLSTM MAX SEQUENCE WORD LENGTH 50 ######################################################\n",
    "encoder_outputs, forward_h, forward_c, backward_h, backward_c = Bidirectional(LSTM(units=LSTM_UNITS//2, return_sequences=True,recurrent_dropout=0, return_state = True, recurrent_activation = 'sigmoid'))(x)\n",
    "state_h = Concatenate()([forward_h, backward_h])\n",
    "state_c = Concatenate()([forward_c, backward_c])\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "################## Attention Mechanism ################################################################\n",
    "decoder_outputs, _, _ = LSTM(units=LSTM_UNITS, recurrent_dropout=0, return_sequences = True, recurrent_activation = 'sigmoid', return_state = True)(x, initial_state = encoder_states)\n",
    "attention = Dot(axes = (2,2))([decoder_outputs, encoder_outputs])\n",
    "attention = Activation('softmax')(attention)\n",
    "context = Dot(axes=(2,1))([attention, encoder_outputs])\n",
    "x = Concatenate()([context, decoder_outputs])\n",
    "################## DENSE LAYER ############################################################################\n",
    "x = Dropout(0.4)(x)\n",
    "x = Dense(DENSE_UNITS, activation='tanh')(x)\n",
    "\n",
    "################## CRF LAYER ####################################################################################\n",
    "crf = CRF(len(tag2idx), sparse_target = False)\n",
    "loss = crf.loss_function\n",
    "y_output = crf(x)\n",
    "\n",
    "loss = crf.loss_function\n",
    "\n",
    "model = Model(inputs = [char_input, word_input, pos_input, gtag_input, sylls_input, bert_input], outputs = y_output)\n",
    "model.compile(optimizer = \"adam\", loss = loss, metrics = [crf.accuracy])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/4\n",
      "6000/6000 [==============================] - 128s 21ms/step - loss: 0.0497 - crf_viterbi_accuracy: 0.9880\n",
      "\n",
      "Epoch 00001: loss improved from inf to 0.04970, saving model to /home/sergio/Escritorio/ProfNER/model_weights/final_model_4.hdf5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sergio/anaconda3/envs/tfm/lib/python3.7/site-packages/keras/engine/network.py:877: UserWarning: Layer lstm_3 was passed non-serializable keyword arguments: {'initial_state': [<tf.Tensor 'concatenate_3/concat:0' shape=(?, 300) dtype=float32>, <tf.Tensor 'concatenate_4/concat:0' shape=(?, 300) dtype=float32>]}. They will not be included in the serialized model (and thus will be missing at deserialization time).\n",
      "  '. They will not be included '\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/4\n",
      "6000/6000 [==============================] - 122s 20ms/step - loss: 0.0035 - crf_viterbi_accuracy: 0.9986\n",
      "\n",
      "Epoch 00002: loss improved from 0.04970 to 0.00347, saving model to /home/sergio/Escritorio/ProfNER/model_weights/final_model_4.hdf5\n",
      "Epoch 3/4\n",
      "6000/6000 [==============================] - 122s 20ms/step - loss: -0.0014 - crf_viterbi_accuracy: 0.9994\n",
      "\n",
      "Epoch 00003: loss improved from 0.00347 to -0.00136, saving model to /home/sergio/Escritorio/ProfNER/model_weights/final_model_4.hdf5\n",
      "Epoch 4/4\n",
      "6000/6000 [==============================] - 121s 20ms/step - loss: -0.0033 - crf_viterbi_accuracy: 0.9994\n",
      "\n",
      "Epoch 00004: loss improved from -0.00136 to -0.00332, saving model to /home/sergio/Escritorio/ProfNER/model_weights/final_model_4.hdf5\n"
     ]
    }
   ],
   "source": [
    "NUM_EPOCHS = 4\n",
    "checkpoint = ModelCheckpoint(sst_home + '/model_weights/final_model_4.hdf5', monitor='loss', verbose=1, save_best_only=True, mode='auto', period=1)\n",
    "history = model.fit([X_char_train, X_words_train, X_pos_train, X_tag_train, X_syll_train, X_train_bert], y_train,\n",
    "                    batch_size = 32,\n",
    "                    epochs = NUM_EPOCHS,\n",
    "                    callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2769, 1536)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bert_entities_.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating the annotations\n",
    "\n",
    "The next step is generate the annotations in the right format to be evaluated. First we loaded the trained model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Layer\n",
    "from keras.backend import constant\n",
    "\n",
    "class CosineSimilarity(Layer):\n",
    "    def __init__(self):\n",
    "        super(CosineSimilarity, self).__init__()\n",
    "        self.result = None\n",
    "        \n",
    "    def call(self, inputs):\n",
    "        entities = constant(bert_entities_)\n",
    "        bert_input = inputs\n",
    "        norm_entities_bert = tf.norm(entities, axis = 1)\n",
    "        norm_bert_input = tf.norm(bert_input, axis =  2)\n",
    "\n",
    "        cosine = tf.einsum('nd,bmd->bmn', entities, bert_input)\n",
    "        norm = tf.einsum('bm,n->bmn',norm_bert_input, norm_entities_bert)\n",
    "\n",
    "        self.result = tf.math.divide_no_nan(cosine, norm)\n",
    "        \n",
    "        return self.result\n",
    "    \n",
    "    def compute_output_shape(self, input_shape):\n",
    "        return [(None, None, 3717)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from keras.models import Model\n",
    "from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Concatenate, Input, SpatialDropout1D\n",
    "from keras.layers import Conv1D, MaxPooling1D,Flatten,GlobalMaxPooling1D, Reshape, RepeatVector,  Dot, GRU, Activation\n",
    "from keras_contrib.layers import CRF\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.backend import constant, variable\n",
    "\n",
    "CHAR_EMBEDDINGS_SIZE = 30    # Characters Embeddings Size\n",
    "SYLL_EMBEDDINGS_SIZE = 75    # Syllable Embeddings Size\n",
    "WORD_EMBEDDINGS_SIZE = 300   # Word Embeddings Size\n",
    "POS_EMBEDDING_SIZE = 20      # PoS Embedding Size\n",
    "GTAGS_EMBEDDING_SIZE = 20    # Tag Embedding Size\n",
    "SPACY_EMBEDDING_SIZE = 20\n",
    "\n",
    "MAX_CHARS_LEN = 25           # Max sequence char length\n",
    "MAX_SEQ_LEN = 75             # Max sequence word length Level II\n",
    "MAX_SYLLS_LEN = 10           # Max sequence sylls length\n",
    "\n",
    "CONV_FILTERS = 50            # Convolutional Filters in Character and Syllable Convolutional Layer\n",
    "LSTM_UNITS = 300             # LSTM Units in both LSTM layers\n",
    "DENSE_UNITS = 200            # Number of units in Dense layer  \n",
    "\n",
    "BERT_DIM = 1536\n",
    "\n",
    "word_input = Input(shape=(None,), name = 'word_input')\n",
    "pos_input = Input(shape=(None,), name = 'pos_input')\n",
    "gtag_input = Input(shape=(None,), name = 'tag_input')\n",
    "char_input = Input(shape=(None, MAX_CHARS_LEN), name = 'char_input')\n",
    "sylls_input = Input(shape=(None, MAX_SYLLS_LEN), name = 'sylls_input')\n",
    "bert_input = Input(shape=(None, BERT_DIM,), name = 'bert_input')\n",
    "\n",
    "###################### COSINE SIMILARITY ##################################################################\n",
    "cosine_bert = CosineSimilarity()(bert_input)\n",
    "\n",
    "###################### CHARACTER SEQUENCE PROCCESSED BY CONVOLUTIONAL LAYER ################################\n",
    "char_embedding = TimeDistributed(Embedding(input_dim = len(char2idx), output_dim = CHAR_EMBEDDINGS_SIZE, input_length=MAX_CHARS_LEN, name = 'char_embeddings', trainable = True))(char_input)\n",
    "conv_1d = TimeDistributed(Conv1D(filters = CONV_FILTERS, kernel_size = 3,padding = \"valid\", activation = \"relu\", name=\"Conv1D_char\"))(char_embedding)\n",
    "conv_1d = TimeDistributed(Dropout(0.4))(conv_1d)\n",
    "maxpool1d = TimeDistributed(GlobalMaxPooling1D(), name = 'max_pooling')(conv_1d)\n",
    "char_enc = TimeDistributed(Flatten(), name = 'char_enc')(maxpool1d)\n",
    "\n",
    "###################### SYLLABLE SEQUENCE PROCCESSED BY CONVOLUTIONAL LAYER ################################\n",
    "syll_embedding = TimeDistributed(Embedding(input_dim = len(sylls2idx), output_dim = SYLL_EMBEDDINGS_SIZE, input_length=MAX_SYLLS_LEN, name = 'sylls_embeddings', trainable = True))(sylls_input)\n",
    "conv_1d_syll = TimeDistributed(Conv1D(filters = CONV_FILTERS, kernel_size = 3,padding=\"valid\", activation=\"relu\", name=\"Conv1D_syll\"))(syll_embedding)\n",
    "conv_1d_syll = TimeDistributed(Dropout(0.4))(conv_1d_syll)\n",
    "maxpool1d_syll = TimeDistributed(GlobalMaxPooling1D())(conv_1d_syll)\n",
    "syll_enc = TimeDistributed(Flatten())(maxpool1d_syll)\n",
    "\n",
    "##################### PoS + TAG EMBEDDINGS ################################################################\n",
    "pos_embedding = Embedding(input_dim = len(pos2idx), output_dim = POS_EMBEDDING_SIZE, name = 'pos_embeddings', trainable = True)(pos_input)\n",
    "pos_embedding = Dropout(0.4)(pos_embedding)\n",
    "gtags_embedding = Embedding(input_dim = len(grammtags2idx), output_dim = GTAGS_EMBEDDING_SIZE, name = 'gtags_embeddings', trainable = True)(gtag_input)\n",
    "gtags_embedding = Dropout(0.4)(gtags_embedding)\n",
    "\n",
    "##################### WORD EMBEDDING LAYER ################################################################\n",
    "medical_embedding_layer = Embedding(input_dim = len(word2idx), output_dim = WORD_EMBEDDINGS_SIZE, weights=[medical_embedding_matrix], trainable=True, name = 'medical_word_embeddings')\n",
    "medical_word_embedding = medical_embedding_layer(word_input)\n",
    "\n",
    "twitter_embedding_layer = Embedding(input_dim = len(word2idx), output_dim = WORD_EMBEDDINGS_SIZE, weights=[twitter_embedding_matrix], trainable=True, name = 'twitter_word_embeddings')\n",
    "twitter_word_embedding = twitter_embedding_layer(word_input)\n",
    "\n",
    "bert_out = Dense(BERT_DIM, activation='relu')(bert_input)\n",
    "bert_out = Dropout(0.4)(bert_out)\n",
    "################### CONCATENATE INPUT FEATURES ###########################################\n",
    "x = Concatenate(axis = -1)([medical_word_embedding, twitter_word_embedding, bert_out, pos_embedding, gtags_embedding, char_enc, syll_enc, cosine_bert])\n",
    "\n",
    "################## BiLSTM MAX SEQUENCE WORD LENGTH 50 ######################################################\n",
    "encoder_outputs, forward_h, forward_c, backward_h, backward_c = Bidirectional(LSTM(units=LSTM_UNITS//2, return_sequences=True,recurrent_dropout=0, return_state = True, recurrent_activation = 'sigmoid'))(x)\n",
    "state_h = Concatenate()([forward_h, backward_h])\n",
    "state_c = Concatenate()([forward_c, backward_c])\n",
    "encoder_states = [state_h, state_c]\n",
    "\n",
    "################## Attention Mechanism ################################################################\n",
    "decoder_outputs, _, _ = LSTM(units=LSTM_UNITS, recurrent_dropout=0, return_sequences = True, recurrent_activation = 'sigmoid', return_state = True)(x, initial_state = encoder_states)\n",
    "attention = Dot(axes = (2,2))([decoder_outputs, encoder_outputs])\n",
    "attention = Activation('softmax')(attention)\n",
    "context = Dot(axes=(2,1))([attention, encoder_outputs])\n",
    "\n",
    "x = Concatenate()([context, decoder_outputs])\n",
    "################## DENSE LAYER ############################################################################\n",
    "x = Dropout(0.4)(x)\n",
    "x = Dense(DENSE_UNITS, activation='tanh')(x)\n",
    "\n",
    "################## CRF LAYER ####################################################################################\n",
    "crf = CRF(len(tag2idx), sparse_target = False)\n",
    "loss = crf.loss_function\n",
    "y_output = crf(x)\n",
    "\n",
    "loss = crf.loss_function\n",
    "\n",
    "model = Model(inputs = [char_input, word_input, pos_input, gtag_input, sylls_input, bert_input], outputs = y_output)\n",
    "model.compile(optimizer = \"adam\", loss = loss, metrics = [crf.accuracy])\n",
    "model.load_weights(sst_home + '/model_weights/final_model_4.hdf5')\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Annotation Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "SINGLE = 'S'\n",
    "BEGIN = 'B'\n",
    "END = 'E'\n",
    "INSIDE = 'I'\n",
    "OUT = 'O'\n",
    "\n",
    "def annotateV2(y_pred, offset, file_name, extra = None):    \n",
    "    max_index = len(y_pred)\n",
    "    if '#ENDPAD' in y_pred:\n",
    "        max_index = y_pred.index('#ENDPAD')\n",
    "        \n",
    "    right_moves = {'B': [ 'I', 'E'],\n",
    "                   'I': ['I', 'E'],\n",
    "                   'E': ['B', 'S', 'O'],\n",
    "                   'S': ['B', 'S', 'O'],\n",
    "                   'O': ['B', 'S', 'O']}\n",
    "    text = ''\n",
    "    entity = ''\n",
    "    start = -1\n",
    "    end = -1\n",
    "    \n",
    "    has_error = False\n",
    "    for i in range(max_index - 1):\n",
    "        if not (has_error and y_pred[i][0] in ['I','E']):\n",
    "            has_error = False\n",
    "            ann = y_pred[i][0]\n",
    "            info = offset[i]\n",
    "            next_ann = y_pred[i + 1][0]\n",
    "            entity_class = y_pred[i][2:]\n",
    "            if next_ann in right_moves[ann]:\n",
    "                info = offset[i]\n",
    "                if ann == BEGIN:\n",
    "                    entity = info[0]\n",
    "                    start = info[1]\n",
    "                    end = info[2]\n",
    "                if ann == INSIDE:\n",
    "                    entity = entity + ' ' + info[0]\n",
    "                    end = info[2]\n",
    "                if ann == END:\n",
    "                    entity = entity + ' ' + info[0]\n",
    "                    end = info[2]\n",
    "                    text = text + f'{file_name}\\t{start}\\t{end}\\t{entity_class}\\t{entity}\\n'\n",
    "                    entity = ''\n",
    "                    start = -1\n",
    "                    end = -1\n",
    "                if ann == SINGLE:\n",
    "                    text = text + f'{file_name}\\t{info[1]}\\t{info[2]}\\t{entity_class}\\t{info[0]}\\n'\n",
    "                    entity = ''\n",
    "                    start = -1\n",
    "                    end = -1\n",
    "                if ann == OUT:\n",
    "                    entity = ''\n",
    "                    start = -1\n",
    "                    end = -1\n",
    "            else:\n",
    "                has_error = True\n",
    "                \n",
    "                entity = ''\n",
    "                start = -1\n",
    "                end = -1\n",
    "            \n",
    "    return text  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "sst_home_test = sst_home + '/final-profner-data/subtask-2/test-background-txt-files'\n",
    "idx2tag = {idx:tag for (tag, idx) in tag2idx.items()}\n",
    "\n",
    "_, docs, _, docs_offset = getElements(sst_home_test, 1000, getTags = False, minit = 25000, maxi = 28000)\n",
    "\n",
    "result_path = sst_home + '/final-result/'\n",
    "text = ''\n",
    "\n",
    "for (file_name, doc_sents) in docs.items():\n",
    "  MAX_SEQ_LEN = len(docs[file_name][0])\n",
    "  X_char = getCharacterInput([docs[file_name][0]], MAX_SEQ_LEN, MAX_CHARS_LEN, char2idx)\n",
    "  X_words = getWordInput([docs[file_name][0]], MAX_SEQ_LEN, word2idx)\n",
    "  X_pos = getPosInput([docs[file_name][1]], MAX_SEQ_LEN, pos2idx)\n",
    "  X_tag = getGTagInput([docs[file_name][2]], MAX_SEQ_LEN, grammtags2idx)\n",
    "  X_syll = getSyllsInput([docs[file_name][0]], MAX_SEQ_LEN, MAX_SYLLS_LEN, sylls2idx)\n",
    "  X_bert = np.array(getBertInput(file_name))\n",
    "  X_bert = pad_sequences(maxlen = MAX_SEQ_LEN, sequences = X_bert, truncating= 'post', padding ='post', value=np.zeros(1536))\n",
    "\n",
    "  y_pred = model.predict([X_char, X_words, X_pos, X_tag, X_syll, X_bert])\n",
    "  print(y_pred)\n",
    "  y_pred = [list(map(lambda x: idx2tag[np.argmax(x)], sent)) for sent in y_pred][0]\n",
    "  offset_test = docs_offset[file_name]\n",
    "  text = text + annotateV2(y_pred, offset_test, file_name, result_path)\n",
    "\n",
    "text = 'tweet_id\\tbegin\\tend\\ttype\\textraction\\n' + text\n",
    "with open(result_path + 'results.tsv', 'w') as f:\n",
    "    f.write(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}