VQA_Model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Vai6hre0E20v"
   },
   "source": [
    "# VQA Model\n",
    "In this Notebook, we wil construct our VQA model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nrPxFvikJxl5"
   },
   "source": [
    "## imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-FvZM4x0J3Tt"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import importlib.util\n",
    "from google.colab import drive\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import re\n",
    "import numpy as np\n",
    "import json\n",
    "from pathlib import Path\n",
    "import pickle\n",
    "from collections import Counter\n",
    "tf.enable_eager_execution()\n",
    "import cv2\n",
    "import time\n",
    "from keras import regularizers\n",
    "from keras.models import load_model\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PcsmoDsERiVh"
   },
   "source": [
    "## Google drive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 36
    },
    "colab_type": "code",
    "id": "I9L3m4KCRlCd",
    "outputId": "ac607880-9376-452a-e4d2-7c8723a15d7e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "drive_root_path = ''\n",
    "drive.mount(drive_root_path, force_remount= True, timeout_ms = 2147483647)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EHtX5-yjJ1d6"
   },
   "source": [
    "## variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MMHAdOE5Ey1l"
   },
   "outputs": [],
   "source": [
    "#root path of data : same as in Install.ipynb\n",
    "root_path ='' \n",
    "\n",
    "#Glove file path\n",
    "glove_file_path =  os.path.join(root_path, 'glove/glove.6B.300d.txt')\n",
    "\n",
    "#train and val images features\n",
    "train_tfrec_dir = os.path.join(root_path, 'features/train_features')\n",
    "val_tfrec_dir = os.path.join(root_path, 'features/val_features')\n",
    "\n",
    "#Question embedding variables\n",
    "BATCH_SIZE = 100\n",
    "PREFETCH_SIZE = BATCH_SIZE\n",
    "MAX_LEN = 14\n",
    "IMAGE_FEAT_SHAPE = (100,120,3)\n",
    "MONO_IMAGE_SIZE = (224, 224)\n",
    "IMAGE_FEAT_NUM = 10\n",
    "#initialized in AnswerPreprocessing\n",
    "ANSWER_DIM = 0\n",
    "IMAGE_FEATURES_LAST_DIM = 2048\n",
    "MONO = True\n",
    "FILL_TOKEN = 'FILL_TOKEN'\n",
    "ANSWER_OCCURENCE_MIN = 15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4bKvbqwRCG43"
   },
   "source": [
    "#  Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k_Rf_tcPIn54"
   },
   "source": [
    "## Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PhJHTE-LIrb7"
   },
   "outputs": [],
   "source": [
    "SENTENCE_SPLIT_REGEX = re.compile(r'(\\W+)')\n",
    "\n",
    "\n",
    "def tokenize(sentence):\n",
    "    sentence = sentence.strip().lower()\n",
    "    sentence = (\n",
    "        sentence.replace(',', '').replace('?', '').replace('\\'s', ' \\'s'))\n",
    "    tokens = SENTENCE_SPLIT_REGEX.split(sentence)\n",
    "    tokens = [t.strip() for t in tokens if len(t.strip()) > 0]\n",
    "    return tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VrZeCDblGh11"
   },
   "source": [
    "## question preprocessor\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LPFfwrRK4VyJ"
   },
   "outputs": [],
   "source": [
    "class QuestionsPreprocessing : \n",
    "  \n",
    "    def __init__(self, glove_file, questions_files, num_occurence) :\n",
    "        self.gloveFile  = glove_file\n",
    "        self.questionsFiles = questions_files\n",
    "        self.word2Glove = self.readGloveFile()\n",
    "        questions = self.readQuestionsFiles()\n",
    "        self.questions_files = questions_files\n",
    "        #filter words that have occurence less than occurence\n",
    "        words = self.filter_words(questions,num_occurence)\n",
    "        #partition words from weather they are in glove or not\n",
    "        self.word2Index, self.index2Word = self.matchWordIndex(words)\n",
    "        #consider the 0 padding\n",
    "        self.vocab_length = len(self.word2Index) + 1 \n",
    "\n",
    "    def readGloveFile(self):\n",
    "        with open(self.gloveFile, 'r') as f:\n",
    "            wordToGlove = {}  # map from a token (word) to a Glove embedding vector\n",
    "            #wordToIndex = {}  # map from a token to an index\n",
    "            #indexToWord = {}  # map from an index to a token \n",
    "\n",
    "            for line in f:\n",
    "                record = line.strip().split()\n",
    "                token = record[0] # take the token (word) from the text line\n",
    "                wordToGlove[token] = np.array(record[1:], dtype=np.float64) # associate the Glove embedding vector to a that token (word)\n",
    "\n",
    "        return wordToGlove\n",
    "\n",
    "  #return all words in questions\n",
    "  #WARNING IMPORTANT : THIS FUNCTION IS ONLY FOR RETRIEVING QUESTIONS FROM VQA DATASET\n",
    "    def readQuestionsFiles(self) : \n",
    "        questions = []\n",
    "        for file in self.questionsFiles :\n",
    "            with open(file, 'r') as f : \n",
    "                data = json.load(f)\n",
    "                for x in data['questions']:\n",
    "                    questions.append(x['question'])\n",
    "        return questions    \n",
    "\n",
    "    \n",
    "    def filter_words(self,questions, num_occurence):\n",
    "        words = {}\n",
    "        for question in questions : \n",
    "            for word in tokenize(question):\n",
    "                if(word in words):\n",
    "                    words[word] +=1\n",
    "                else :\n",
    "                    words[word] = 1\n",
    "        return [k for k,v in words.items() if v > num_occurence]\n",
    "\n",
    "    \n",
    "    def matchWordIndex(self, words) : \n",
    "        word2Index = {}\n",
    "        index2Word = {}\n",
    "        for i, word in enumerate(words):\n",
    "            word2Index[word] = i+1\n",
    "            index2Word[i+1] =word\n",
    "        return word2Index, index2Word\n",
    "\n",
    " \n",
    "  # create embedding matrix\n",
    "    def createPretrainedEmbeddingLayer(self):\n",
    "        wordToIndex = self.word2Index\n",
    "        wordToGlove = self.word2Glove\n",
    "        vocabLen = len(wordToIndex) + 1  # adding 1 to account for masking\n",
    "        glove_words = wordToGlove.keys()\n",
    "        embDim = next(iter(wordToGlove.values())).shape[0] \n",
    "        embeddingMatrix = np.zeros((vocabLen, embDim), 'float64')  # initialize with zeros\n",
    "        for word, index in wordToIndex.items():\n",
    "            if word in glove_words:\n",
    "                embeddingMatrix[index, :] = wordToGlove[word] # create embedding: word index to Glove word embedding\n",
    "            else : \n",
    "                embeddingMatrix[index, :] = np.random.rand(1, embDim)\n",
    "        return embeddingMatrix\n",
    "\n",
    "  \n",
    "    def get_embedding_matrix(self) : \n",
    "        embedding_matrix = self.createPretrainedEmbeddingLayer()\n",
    "        return embedding_matrix\n",
    "  \n",
    "    # TODO : check if there is another method like removing most common words  \n",
    "    def preprocessBatch(self,questions, max_len):\n",
    "        batch = []\n",
    "        for question in questions : \n",
    "            words =self.preprocessElem(question)\n",
    "            batch.append(num_words)\n",
    "        return self.postTruncate(batch, max_len)\n",
    "\n",
    "    def preprocessElem(self,question):\n",
    "        words = self.preTruncate(question)\n",
    "        words = tokenize(words)\n",
    "        num_words = []\n",
    "        for word in words :\n",
    "            w = self.word2Index.get(word, 'not_found_word')\n",
    "            if not w is 'not_found_word' :\n",
    "                num_words.append(w)\n",
    "        return num_words\n",
    "  #private\n",
    "  #TODO : implement\n",
    "    def preTruncate(self, words): \n",
    "        return words\n",
    "  \n",
    "    def postTruncate(self,batch, max_len) : \n",
    "        return  keras.preprocessing.sequence.pad_sequences(batch, max_len,padding ='post', truncating = 'post')\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Answer preprocessor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BYHzrotSnMOu"
   },
   "outputs": [],
   "source": [
    "class AnswerPreprocessing : \n",
    "    def __init__(self, questions_files, answers_files):\n",
    "        global ANSWER_DIM\n",
    "        assert len(questions_files) == len(answers_files), 'not the same length'\n",
    "        self.answers, _ = self.get_answers(questions_files[0], answers_files[0])\n",
    "        #for i, f in enumerate(questions_files):\n",
    "         # s,m = self.get_answers(questions_files[i], answers_files[i])\n",
    "         # self.answers = self.answers.union(s)\n",
    "        self.word2Index = {}\n",
    "        self.index2Word = {}\n",
    "        self.word2Index, self.index2Word = self.matchWords2Indexes()\n",
    "        ANSWER_DIM = len(self.word2Index)\n",
    "        self.num_words = len(self.word2Index)\n",
    "\n",
    "    def get_dim():\n",
    "        return len(self.word2Index)\n",
    "  #private\n",
    "    def matchWords2Indexes(self): \n",
    "        wordToIndex = {}\n",
    "        indexToWord = {}\n",
    "        words = self.answers\n",
    "        words = self.filterWords(words)\n",
    "        wordToIndex['no idea'] = 0\n",
    "        indexToWord[0] = 'no idea'\n",
    "        i = 1\n",
    "        for w in words:\n",
    "            if not w == 'no idea' :\n",
    "                wordToIndex[w] = i\n",
    "                indexToWord[i] = w\n",
    "                i += 1\n",
    "        return (wordToIndex, indexToWord)\n",
    "      #private\n",
    "    def readWords(self) :\n",
    "        words = []\n",
    "        for file in self.answer_files : \n",
    "            with open(file, 'r') as f : \n",
    "                data = json.load(f)\n",
    "                for annotation in data['annotations'] : \n",
    "                    for answer in annotation['answers']:\n",
    "                        words.append(answer['answer'])\n",
    "        return words\n",
    "\n",
    "  #private\n",
    "    def filterWords (self, words) : \n",
    "        return words\n",
    "  \n",
    "    def preprocessBatch(self, answers) : \n",
    "        batch  = []\n",
    "        ans = np.array(answers)\n",
    "        if ans.ndim == 2 : \n",
    "            for answers2 in answers : \n",
    "                batch.append([self.word2Index[x] for x in answers2])\n",
    "        else : \n",
    "            for answer in answers: \n",
    "                batch.append(self.word2Index[answer])\n",
    "        return batch\n",
    "\n",
    "\n",
    "    def _preprocessElem(self,ans):\n",
    "    # if the dataset contains multiple answers : Exemple VQA dataset\n",
    "        answer = np.array([x for x in ans if not x == FILL_TOKEN])\n",
    "        if answer.ndim == 1 :\n",
    "          arr = np.zeros((ANSWER_DIM,))\n",
    "          if len(answer) == 0 :\n",
    "            arr[0] = 1.0\n",
    "            return arr\n",
    "          else :\n",
    "          #arr = np.zeros(self.num_words,dtype=int)\n",
    "\n",
    "            value = 1/len(answer)\n",
    "            found = False\n",
    "            for i, a in enumerate(answer):\n",
    "              if a in self.word2Index:\n",
    "                found = True\n",
    "                arr[self.word2Index[a]] += value\n",
    "            if not found :\n",
    "              arr[0] = 1.0\n",
    "            return arr\n",
    "        #if the answer is unique in the dataset\n",
    "        else :\n",
    "          return self.word2Index[answer]\n",
    "\n",
    "  def preprocessElem(self,ans):\n",
    "    answer = np.array([x for x in ans if not x == FILL_TOKEN], 'str')\n",
    "    arr = np.zeros((ANSWER_DIM,))\n",
    "    if len(answer) == 0 :\n",
    "      arr[0] =1\n",
    "    else :\n",
    "      c = Counter(answer)\n",
    "      t = [x for (x,y) in c.items() if x in self.word2Index]\n",
    "      if t == [] :\n",
    "        arr[0] = 1\n",
    "      else :\n",
    "        for elem, occur in c.items() :\n",
    "          if elem in self.word2Index :\n",
    "            index = self.word2Index[elem]\n",
    "            score = 1 if occur >=3 else 1/3* occur\n",
    "            arr[index] = score\n",
    "    return arr \n",
    "  \n",
    "  def get_ques2(self,questions_file):\n",
    "      with open(questions_file,'r') as f : \n",
    "        data = json.load(f)\n",
    "      questions = data['questions']\n",
    "      qsid_iq = { x['question_id']:  x['question'] for x in questions }\n",
    "      return qsid_iq\n",
    "\n",
    "    \n",
    "  def get_answers( self, questions_file,answers_file) :\n",
    "    global ANSWER_DIM\n",
    "    numbers  = [\"zero\",\"one\",\"two\",\"three\",\"four\",\n",
    "          \"five\",\"six\",\"seven\",\"eight\",\"nine\",\"ten\",\n",
    "          \"eleven\",\"twelve\",\"thirteen\",\"fourteen\",\"fifteen\",\n",
    "          \"sixteen\",\"seventeen\",\"eighteen\",\"nineteen\"];\n",
    "    tens = [\"Twenty\",\"Thirty\",\"Forty\",\"Fifty\",\n",
    "          \"Sixty\",\"Seventy\",\"Eighty\",\"Ninety\"]\n",
    "    id_qs = self.get_ques2(questions_file)\n",
    "\n",
    "    with open(answers_file,'r') as f : \n",
    "      data = json.load(f)\n",
    "    resps = data['annotations']\n",
    "    res = [] \n",
    "    ids = []\n",
    "    conf =[]\n",
    "    for x in resps:\n",
    "      question = id_qs[x['question_id']].strip().lower()\n",
    "      questionID = x['question_id']\n",
    "      ans = [y['answer'].replace(',', ' ').replace('?', '').replace('\\'s', ' \\'s').strip().lower() for y in x['answers'] ]\n",
    "      res1 =[]\n",
    "      for word in ans :\n",
    "          if word == 'no 1' or 'no one' in word :\n",
    "            res1.append('no one')\n",
    "          elif word in ['no clue', \"i dont know\", \"i don't know\", \"cannot know\", \"can't know\", \"can't tell\", \"not sure\", \"don't know\", \"cannot tell\", \"unknown\"]:\n",
    "            res1.append('no idea')\n",
    "          elif word == 'my best guess is no' or \"it isn't\" in word or  'it is not' in word:\n",
    "            res1.append('no')\n",
    "          elif 'many' in word or 'several' in word or 'lot' in word or 'numerous' in word:\n",
    "            res1.append('many')\n",
    "          elif word in numbers :\n",
    "            res1.append(str(numbers.index(word)))\n",
    "          elif word in tens:\n",
    "            res1.append(str((ten.index(word) + 2) * 10))\n",
    "          else :\n",
    "            res1.append(word)\n",
    "\n",
    "\n",
    "\n",
    "      if question.startswith('how many') or question.startswith('what is the number'):\n",
    "        for word in res1 :\n",
    "          if re.search('(\\s|^)no ', word) or re.search(' no(\\s|$)',word):\n",
    "            if word == 'no idea':\n",
    "              res.append('no idea')        \n",
    "            else :\n",
    "              res.append('0')\n",
    "          elif word == 'o' :\n",
    "            res.append(0)\n",
    "          elif not len(re.findall('\\d+', word)) == 0:\n",
    "              res.append(re.findall('\\d+', word)[0])         \n",
    "          elif word == 'no' :\n",
    "              res.append('0')\n",
    "          elif word =='yes' :\n",
    "              res.append('1')\n",
    "          else :\n",
    "              res.append(word)\n",
    "\n",
    "      elif question.startswith('is') or question.startswith('are'):\n",
    "\n",
    "        for word in res1 :\n",
    "          if re.search('(\\s|^)no ', word) or re.search(' no(\\s|$)',word):\n",
    "            res.append('no')\n",
    "          elif word == 'it is' or 'yes' in word:\n",
    "            res.append('yes')\n",
    "          elif 'it is' in word :\n",
    "            s = word.replace('it is', '').strip()\n",
    "            res.append(s)\n",
    "            continue\n",
    "          else :\n",
    "            res.append(word) \n",
    "\n",
    "      else :\n",
    "        for word in res1 :\n",
    "          if word == 'it is' or 'yes' in word:\n",
    "              res.append('yes')\n",
    "          elif 'it is' in word :\n",
    "            s = word.replace('it is', '').strip()\n",
    "            res.append(s)\n",
    "          elif ('there is no' in word) or (\"there's no\" in word) or ('there are no' in word):\n",
    "            res.append('not found')\n",
    "          elif word.strip().startswith('no ') :\n",
    "            ans_tokens = tokenize(word[2:])\n",
    "            ques_tokens = tokenize(question)\n",
    "            boo = True\n",
    "            for t in ans_tokens:\n",
    "              if not (t in ques_tokens or t+'s' in ques_tokens):\n",
    "                boo = False\n",
    "                break\n",
    "            if boo :\n",
    "              res.append('not found')\n",
    "            else :\n",
    "              res.append(word)\n",
    "          else :\n",
    "            res.append(word)  \n",
    "\n",
    "      for s in ans:\n",
    "        ids.append(questionID)\n",
    "\n",
    "      #TODO remove this:    \n",
    "      conf1 = [y['answer_confidence'] for y in x['answers'] ]\n",
    "      conf.extend(conf1)\n",
    "\n",
    "    newres = []\n",
    "    newids = []\n",
    "    for index in range(len(res)) :\n",
    "      if conf[index] == 'yes' :\n",
    "        newres.append(res[index])\n",
    "        newids.append(ids[index])\n",
    "    c = Counter(newres)\n",
    "    resset = set([k for k,v in c.items() if v >= ANSWER_OCCURENCE_MIN])\n",
    "    m = {}\n",
    "    for index in range(len(newres)) :\n",
    "      qid = newids[index]\n",
    "      response = newres[index]\n",
    "      if  response in resset : \n",
    "        if qid in m :\n",
    "          m[qid].append(response)\n",
    "        else :\n",
    "          m[qid] = [response]\n",
    "  #  queskeys = set(ques.keys())\n",
    "\n",
    "   # m = {k : v for k,v in m.items() if k in queskeys} \n",
    "    return resset, m\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "VGS_4C-cCZYd"
   },
   "source": [
    "# Model "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pmnB9kEUCfK2"
   },
   "source": [
    "## Model Variables "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "J_9qLhlcFtFd"
   },
   "source": [
    "## Image Feature Extraction\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HBMpCO7_F2JX"
   },
   "source": [
    "### Objects' bounding boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EGA4ZmRFJG2E"
   },
   "outputs": [],
   "source": [
    "def get_image_model(model_type, input_shape) :\n",
    "  if model_type == 'resnext50':\n",
    "    return keras.applications.ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=input_shape, pooling='avg')\n",
    "  else : \n",
    "    raise NotImplementedError('Unknow extractor')\n",
    "\n",
    "def get_image_module(model_type, mono) :\n",
    "  if not mono :\n",
    "    return ImageModel(model_type)\n",
    "  else :\n",
    "    return MonoImageModel(model_type)\n",
    "  \n",
    "class ImageModel(keras.Model) : \n",
    "  def __init__(self, model_type):\n",
    "    super(ImageModel, self).__init__()\n",
    "    self.model = get_image_model(model_type,IMAGE_FEAT_SHAPE)\n",
    "    self.flat = keras.layers.Flatten()\n",
    "  def call(self,inp) :\n",
    "    arr = []\n",
    "    for batch in inp: \n",
    "      x = tf.dtypes.cast(batch,tf.float64)\n",
    "      x = self.model(x)\n",
    "      x = self.flat(x)\n",
    "      arr.append(x)\n",
    "    return tf.convert_to_tensor(arr)\n",
    "  \n",
    "class MonoImageModel(keras.Model):\n",
    "  def __init__(self, model_type):\n",
    "    super(MonoImageModel, self).__init__()\n",
    "    self.model = get_image_model(model_type, (224, 224,3))\n",
    "    self.flat = keras.layers.Flatten()\n",
    "  def call (self, inp):\n",
    "    x = self.model(inp)\n",
    "    x = self.flat(x)\n",
    "    return x    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "KMaxIc0uGaPB"
   },
   "source": [
    "## Text features extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZiJ9ng01Gm_A"
   },
   "source": [
    "###  Reccurent networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bfuzkjEmMdM6"
   },
   "outputs": [],
   "source": [
    "def get_question_module(model_type, kwargs): \n",
    "  if model_type == 'GRU':\n",
    "    return MyGRU(kwargs)\n",
    "  else :\n",
    "    raise NotImplementedError('Unknown question module')\n",
    "\n",
    "\n",
    "\n",
    "class MyGRU(keras.Model) : \n",
    "  def __init__(self, kwargs) :\n",
    "    super(MyGRU, self).__init__()\n",
    "    self.embedding_weights = kwargs['embedding_weights']\n",
    "    self.embedding_size =kwargs['embedding_size']\n",
    "    self.hidden_size = kwargs['hidden_size']\n",
    "    self.num_layers = kwargs['num_layers']\n",
    "    self.vocab_size = kwargs['vocab_len']\n",
    "    tmp_dropout = kwargs['dropout']\n",
    "    self.dropout = tmp_dropout if tmp_dropout else 0.\n",
    "    \n",
    "    #TODO Check trainable\n",
    "    self.embedding = keras.layers.Embedding(input_dim = self.vocab_size,\n",
    "                                            output_dim = self.embedding_size, weights = [self.embedding_weights], trainable = True)\n",
    "    #TODO : see if we use args like : use_bias, activation, initilizers .... see also reset_after\n",
    "    self.seq = keras.models.Sequential()\n",
    "    input_size = self.hidden_size\n",
    "    for i in range(self.num_layers):\n",
    "      self.seq.add(keras.layers.GRU(units = self.hidden_size, dropout = self.dropout, recurrent_dropout= self.dropout))\n",
    "    \n",
    "    \n",
    "  def call(self, x):\n",
    "    x = self.embedding(x)    \n",
    "    x = self.seq(x)\n",
    "    return x\n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dCFmPO53HIsv"
   },
   "source": [
    "## Merging Before Attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9fjPx4sQiWjW"
   },
   "outputs": [],
   "source": [
    "#Done : check how to do if merger_type is hadamard and we introduce region coordinates : \n",
    "### image features then text features\n",
    "def join_features(merger_type):\n",
    "  if merger_type == 'concat' : \n",
    "    return ConcatMerger()\n",
    "  elif merger_type == 'hadamard':\n",
    "    return HadamardMerger()\n",
    "  else :\n",
    "    raise NotImplementedError('Unknown Merger')\n",
    "\n",
    "\n",
    "\n",
    "class ConcatMerger(keras.Model):\n",
    "  def __init__(self):\n",
    "    super(ConcatMerger,self).__init__()\n",
    "    \n",
    "  #TODO : borders is not necessary because I don't use it in the first merger, I must fix this case after training\n",
    "  def call (self, imgs,borders, texts) :\n",
    "    x= tf.concat([imgs,texts], -1)\n",
    "    return x\n",
    "    \n",
    "\n",
    "  \n",
    "class HadamardMerger(keras.Model) : \n",
    "  def __init__(self):\n",
    "    super(HadamardMerger,self).__init__()\n",
    "    self.mul = keras.layers.Multiply()\n",
    "  def call(self,imgs, borders, texts):\n",
    "    if not borders is None : \n",
    "      x=  self.mul([imgs,texts])\n",
    "      x =  tf.concat([borders, x])\n",
    "      return x\n",
    "    else :\n",
    "      #for after attention merging\n",
    "      x=  self.mul([imgs,texts])\n",
    "      return x\n",
    "\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "w5CjnzybHMw2"
   },
   "source": [
    "## Non linear function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rAp-U_77prkY"
   },
   "outputs": [],
   "source": [
    "\n",
    "def get_non_linear_function(func_type, kwargs) : \n",
    "  \n",
    "  if func_type == 'activation':\n",
    "    return Activation(kwargs)\n",
    "  else :\n",
    "    raise NotImplementedError('Unknown non linear function')\n",
    "\n",
    "\n",
    "class Activation(keras.Model):\n",
    "  #WARNING must be positive (num_layers), dims size > 1\n",
    "  def __init__(self, kwargs):\n",
    "    super(Activation, self).__init__()\n",
    "    input_shape = kwargs['input_shape']\n",
    "    dims = kwargs['dims']\n",
    "    activation = kwargs['activation']\n",
    "    dropout = kwargs['dropout']\n",
    "    normalization = kwargs['normalization']\n",
    "    regularisation = kwargs['regularisation']\n",
    "    regularisation = regularizers.l2(regularisation) if regularisation else None\n",
    "    \n",
    "    num_layers = len(dims)\n",
    "    self.model = keras.models.Sequential()\n",
    "    for i in range(num_layers- 1):\n",
    "      if input_shape and i == 0 :\n",
    "        self.model.add(keras.layers.Dense(units=dims[i],activation=activation, input_shape = input_shape, kernel_regularizer=regularisation))\n",
    "      else :\n",
    "        self.model.add(keras.layers.Dense(units=dims[i],activation=activation, kernel_regularizer=regularisation))\n",
    "      if normalization :\n",
    "        self.model.add(keras.layers.BatchNormalization())\n",
    "    if dropout:\n",
    "      self.model.add(keras.layers.Dropout(dropout))\n",
    "    self.model.add(keras.layers.Dense(dims[i+1]))\n",
    "    \n",
    "    #TODO : May be we need first dimension if we use this. Done ?\n",
    "    \n",
    "  def call(self,x) :\n",
    "    return self.model(x)\n",
    "      "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "olgfTVy8G6JG"
   },
   "source": [
    "## Attention mechanism"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TB9eFPjPodoO"
   },
   "outputs": [],
   "source": [
    "\n",
    "### Attention vector\n",
    "#dims must end with the right dimention : img_feature dimention\n",
    "\n",
    "\n",
    "class FixAttentionVector(keras.Model) : \n",
    "  #normalization : Softmax, hard tanh, sigmoid\n",
    "  #dims, image_features_last_dim, activation, normalization\n",
    "  def __init__(self, args, image_features_last_dim, mono):\n",
    "    super(FixAttentionVector, self).__init__()\n",
    "    self.image_features_last_dim = image_features_last_dim\n",
    "    probs = args['probability_function']\n",
    "    self.f = get_non_linear_function('activation', args)\n",
    "    self.mono = mono\n",
    "    if probs in ['sigmoid', 'softmax']: \n",
    "      self.prob = probs\n",
    "    else :\n",
    "      raise NotImplementedError('Unknown non normalization function')\n",
    "\n",
    "  def call(self,x) :\n",
    "    res = self.f(x)\n",
    "    if self.prob == 'sigmoid':\n",
    "      res = keras.activations.sigmoid(res)\n",
    "    elif self.prob == 'softmax' : \n",
    "      #TODO : see dimension of softmax. normalment la dimension de sortie est \n",
    "      # (batch, 10,1) donc normalement softmax est appliquee sur le 10. Je ne c pas \n",
    "      # vraiment comment ca marche, Done ?\n",
    "      print('shap of res before softmax {}'.format(res.shape))\n",
    "      res = keras.activations.softmax(res)\n",
    "    if not self.mono :\n",
    "      res =  tf.tile(res , (1,1,self.image_features_last_dim))\n",
    "    return res\n",
    "\n",
    "  \n",
    "  \n",
    "  \n",
    "### Image feature\n",
    "class SumImageAttribute(keras.Model):\n",
    "  \n",
    "  def __init__(self, mono):\n",
    "    super(SumImageAttribute,self).__init__()\n",
    "    self.mul = keras.layers.Multiply()\n",
    "    self.sum = keras.layers.Add()\n",
    "    self.mono = mono\n",
    "  def call(self,image_features, attention):\n",
    "    x = self.mul([image_features, attention])\n",
    "    if self.mono :\n",
    "      print('applyer mono')\n",
    "      return x\n",
    "    else :\n",
    "      return keras.backend.sum(x, axis = 1)\n",
    "  \n",
    "  \n",
    "  \n",
    "# todo check last dimention after softmax\n",
    "class AttentionSystem(keras.Model): \n",
    "  def __init__(self, merger, attention_vector, attention_applyer):\n",
    "    super(AttentionSystem,self).__init__()\n",
    "    self.merger = merger\n",
    "    self.attention_vector = attention_vector\n",
    "    self.attention_applyer = attention_applyer\n",
    "  def call(self, image_features, borders, text_features):\n",
    "    common_features = self.merger(image_features, borders, text_features)\n",
    "    vect = self.attention_vector(common_features)\n",
    "    if borders :\n",
    "      new_image_features =  tf.concat([borders, image_features], -1)\n",
    "    else :\n",
    "      new_image_features = image_features\n",
    "    return self.attention_applyer(new_image_features, vect)\n",
    "    \n",
    "    \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LGkKVadnHCE4"
   },
   "source": [
    "## Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zlx-11mhjYiL"
   },
   "outputs": [],
   "source": [
    "class Classifier(keras.Model):\n",
    "  def __init__(self, kwargs):\n",
    "    super(Classifier,self).__init__()\n",
    "    self.f = get_non_linear_function(kwargs)\n",
    "  def call(self,x):\n",
    "    return self.f(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "k1NQ2FTFDzaJ"
   },
   "source": [
    "## model definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R2cMEb9VZcyn"
   },
   "outputs": [],
   "source": [
    "\n",
    "ANSWER_TEST = []\n",
    "class MyModel(keras.Model):\n",
    "  def __init__(self, image_model, image_non_linear,\n",
    "               text_model, text_non_linear, attention_system, merger, classifier, optimizer, mono) :\n",
    "    super(MyModel,self).__init__()\n",
    "    self.image_model = image_model   \n",
    "    self.image_non_linear = image_non_linear\n",
    "    self.text_model = text_model\n",
    "    self.text_non_linear = text_non_linear\n",
    "    self.merger = merger\n",
    "    self.attention_system = attention_system\n",
    "    self.classifier = classifier\n",
    "    self.optimizer = optimizer\n",
    "    self.mono = mono\n",
    "    self.norm  = keras.layers.BatchNormalization()\n",
    "    \n",
    "   # print(self.image_non_linear.summary())\n",
    "    print('model started')\n",
    "    \n",
    "  def call(self, img_features, borders, question, training = False):\n",
    "    #x = time.time()\n",
    "    \n",
    "    print('image model dim {} '.format(img_features.shape))\n",
    "    #print('image model {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "    imgs = self.image_non_linear(img_features)\n",
    "    print('image model dim after non linear {} '.format(imgs.shape))\n",
    "\n",
    "    #print('image non linear {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "    #Here I concatenate\n",
    "    #TODO : I think I'll remove this ... Done ?\n",
    "    #imgs = tf.concat([imgs, borders], -1)\n",
    "    #img_feat_num = imgs.shape[-1] if self.mono else imgs.shape[1]\n",
    "\n",
    "    text = self.text_model(question)\n",
    "    print('text model dim {} '.format(text.shape))\n",
    "\n",
    "    #print('text model {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "    text = self.text_non_linear(text)\n",
    "    print('text model dim after non linear {} '.format(text.shape))\n",
    "\n",
    "    #print('text non linear  {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "    if not self.mono :\n",
    "      texts = tf.tile(tf.expand_dims(text,1), [1,img_feat_num,1])\n",
    "    else : \n",
    "      texts  = text\n",
    "    print('befor attention system {} {}'.format(imgs.shape, texts.shape))\n",
    "    #TODO : Border instead of None\n",
    "    new_img_feature = self.attention_system(imgs,None, texts)\n",
    "    print('new image featuresdim {} '.format(new_img_feature.shape))\n",
    "\n",
    "    #print('attention {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "    common = self.merger(new_img_feature,None, text)\n",
    "    print('common shape {} '.format(common.shape))\n",
    "    #print('merger {} '.format(time.time() - x))\n",
    "    #x = time.time()\n",
    "\n",
    "\n",
    "    res = self.classifier(common)\n",
    "    print('res  dim {} '.format(res.shape))\n",
    "\n",
    "    #print('classifier {} '.format(time.time() - x))\n",
    "    return res\n",
    "  \n",
    " \n",
    " \n",
    "  def train(self, train, val, epochs) :\n",
    "    global ANSWER_TEST\n",
    "    #Tensorboard\n",
    "    global_step = tf.train.get_or_create_global_step()\n",
    "    writer = tf.contrib.summary.create_file_writer(logdir)\n",
    "    writer.set_as_default()\n",
    "    \n",
    "    #Checkpoint\n",
    "    ckpt = tf.train.Checkpoint(step=tf.Variable(1), optimizer=self.optimizer, net=self)\n",
    "    manager = tf.train.CheckpointManager(ckpt, CHECKPOINT_PATH, max_to_keep=5)\n",
    "    ckpt.restore(manager.latest_checkpoint)\n",
    "    \n",
    "    if manager.latest_checkpoint:\n",
    "      print(\"Restored from {}\".format(manager.latest_checkpoint))\n",
    "    else:\n",
    "      print(\"Initializing from scratch.\")\n",
    "      \n",
    "    for ie in range(epochs) : \n",
    "      print('epoch'+str(ie)+ '.............................................')\n",
    "      train_losses = []\n",
    "      train_accuracies = []\n",
    "      \n",
    "      val_losses = []\n",
    "      val_accuracies = []\n",
    "     #----------------TRAIN-------------------------#  \n",
    "      x = train.make_one_shot_iterator()\n",
    "      #remove this one\n",
    "      if val : \n",
    "        y = val.make_one_shot_iterator()\n",
    "      for ib, batch in enumerate(x) :\n",
    "        t = time.time()\n",
    "        if not self.mono :\n",
    "          borders = batch[1]\n",
    "          imgs = batch[0]\n",
    "          ques = batch[2]\n",
    "          ans = batch[3]\n",
    "        else :\n",
    "          borders = None\n",
    "          imgs = batch[0]\n",
    "          ques = batch[1]\n",
    "          ans = batch[2] \n",
    "          imgs = self.image_model(imgs)\n",
    "        \n",
    "        with tf.GradientTape() as tape :\n",
    "          #tape.watch(batch)                \n",
    "          res = self.call(imgs, borders, ques, True)\n",
    "          if ib == 2 :\n",
    "            res.numpy().tofile(file ='/content/file2.txt', sep = '  ' )\n",
    "            ans.numpy().tofile(file ='/content/fileAns.txt', sep = '  ' )\n",
    "          elif ib == 1 :\n",
    "            res.numpy().tofile(file ='/content/file1.txt', sep = '  ' )\n",
    "          elif ib == 0:\n",
    "            res.numpy().tofile(file ='/content/file0.txt', sep = '  ' )\n",
    "            \n",
    "          batch_loss = tf.losses.softmax_cross_entropy(ans,res)\n",
    "        #Done : see batch accuracy\n",
    "        print('ans shape, res shape {} {}'.format(ans.shape, res.shape))\n",
    "        batch_accuracy = tf.math.reduce_sum(tf.math.multiply(tf.to_double(res >= tf.reduce_max(res)),ans)) / BATCH_SIZE\n",
    "        x = time.time()\n",
    "        with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "            tf.contrib.summary.scalar('train_batch_loss', batch_loss)\n",
    "            tf.contrib.summary.scalar('train_batch_accuracy', batch_accuracy)\n",
    "        gradients = tape.gradient(batch_loss, self.variables)\n",
    "        optimizer.apply_gradients(zip(gradients, self.variables))\n",
    "    \n",
    "        train_losses.append(batch_loss)\n",
    "        train_accuracies.append(batch_accuracy)\n",
    "        print('train batch {} ............... loss : {} , accuracy : {} , time {} '.format( ib, batch_loss, batch_accuracy, time.time() -t))\n",
    "\n",
    "      train_mean_epoch_loss = sum(train_losses) / len(train_losses)\n",
    "      train_losses = []\n",
    "      train_mean_epoch_accuracy = sum(train_accuracies) / len(train_accuracies)\n",
    "      train_accuracies = []\n",
    "      with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "        tf.contrib.summary.scalar('train_mean_epoch_loss', train_mean_epoch_loss)\n",
    "        tf.contrib.summary.scalar('train_mean_epoch_accuracy', train_mean_epoch_accuracy)\n",
    "      #----------------VAL-------------------------#  \n",
    "      if val : \n",
    "        for ibv, val_batch in enumerate(y) : \n",
    "          x = time.time()\n",
    "          if not self.mono :\n",
    "            borders = batch[1]\n",
    "            imgs = batch[0]\n",
    "            ques = batch[2]\n",
    "            ans = batch[3]\n",
    "          else :\n",
    "            borders = None\n",
    "            imgs = batch[0]\n",
    "            ques = batch[1]\n",
    "            ans = batch[2]\n",
    "          res = self.call(imgs, borders, ques)\n",
    "          res = self.norm(res)\n",
    "          loss = tf.losses.softmax_cross_entropy(ans,res)\n",
    "          b = tf.reduce_all(tf.equal(ans, 0))\n",
    "          if b :\n",
    "            accuracy = tf.math.reduce_sum(tf.math.multiply(tf.to_double(res >= tf.reduce_max(res)),ans))\n",
    "          else : \n",
    "            accuracy = 0\n",
    "          with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "            tf.contrib.summary.scalar('val_batch_loss', loss)\n",
    "            tf.contrib.summary.scalar('val_batch_accuracy', accuracy)\n",
    "          val_losses.append(loss)\n",
    "          val_accuracies.append(accuracy)\n",
    "          print('val batch {} ............... loss : {} , accuracy : {} '.format(ibv, loss, accuracy))\n",
    "\n",
    "        val_mean_epoch_accuracy = sum(val_accuracies) / len(val_accuracies)\n",
    "        val_mean_epoch_loss = sum(val_losses) / len(val_losses)\n",
    "        val_losses = []\n",
    "        val_accuracies = []\n",
    "        with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "          tf.contrib.summary.scalar('val_mean_epoch_loss', val_mean_epoch_loss)\n",
    "          tf.contrib.summary.scalar('val_mean_epoch_accuracy', val_mean_epoch_accuracy)\n",
    "        \n",
    "      print('train epoch {} ............... mean loss : {} , mean accuracy : {} '.format(ie, train_mean_epoch_loss, train_mean_epoch_accuracy))\n",
    "      if val : \n",
    "        print('val epoch {} ............... mean loss : {} , mean accuracy : {} '.format(ie, val_mean_epoch_loss, val_mean_epoch_accuracy))\n",
    "\n",
    "      ckpt.step.assign_add(1)\n",
    "      if int(ckpt.step) % 3 == 0:\n",
    "        save_path = manager.save()\n",
    "        print(\"Saved checkpoint for step {}: {}\".format(int(ckpt.step), save_path))\n",
    "        \n",
    "  # REMARK : Comment va se comporter le reseau avec batch = 1 ??\n",
    "  def test(self, x) :\n",
    "    \n",
    "    steps = 0\n",
    "    accuracies = []\n",
    "    x = x.make_one_shot_iterator()\n",
    "    for ib, batch in enumerate(x) : \n",
    "      print('test batch' + str(ib)+ '..............................................')\n",
    "      imgs = batch[0]\n",
    "      borders = batch[1]\n",
    "      ques = batch[2]\n",
    "      ans = batch[3]\n",
    "      res = self.call(imgs, borders, ques, False)\n",
    "      loss = tf.losses.softmax_cross_entropy(ans,res)\n",
    "      batch_accuracy = tf.math.reduce_sum(tf.math.multiply(tf.to_double(res >= tf.reduce_max(res)),ans))\n",
    "      accuracies.append(batch_accuracy)\n",
    "      steps +=1\n",
    "    test_accuracy = sum(accuracies) / steps\n",
    "  \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Sf4RNOp4b2Cx"
   },
   "source": [
    "# define the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DPNtGThUXNLI"
   },
   "source": [
    "## models variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 428
    },
    "colab_type": "code",
    "id": "3umZkReZcJtR",
    "outputId": "3da8044f-91da-4514-b4db-6cf73c1f2c80"
   },
   "outputs": [],
   "source": [
    "# train questions and answers file\n",
    "train_questions_file = os.path.join(root_path, 'VQA_dataset/train/v2_Questions_Train_mscoco.json')\n",
    "train_answers_file = os.path.join(root_path, 'VQA_dataset/train/v2_Annotations_Train_mscoco.json')\n",
    "\n",
    "# val questions and answers file\n",
    "val_questions_file =  os.path.join(root_path, 'VQA_dataset/val/v2_Questions_Val_mscoco.json')\n",
    "val_answers_file = os.path.join(root_path, 'VQA_dataset/val/v2_Annotations_Val_mscoco.json')\n",
    "\n",
    "# minimum words occurence in questions\n",
    "num_occurence = 5\n",
    "complementary_file = os.path.join(root_path, 'VQA_dataset/train/v2_Complementary_Pairs_Train_mscoco.zip)\n",
    "\n",
    "#checkpoints path where to save the last models when training\n",
    "CHECKPOINT_PATH = os.path.join(root_path, 'results/checkpoint_files')\n",
    "#best model path\n",
    "BEST_MODEL_PATH = os.path.join(root_path, 'results/best_model/best_model.h5')\n",
    "# best model loss path, to compare with the best val losses when training even after the session go out\n",
    "BEST_LOSS_MODEL_PATH = os.path.join(root_path, 'results/best_model/best_model_loss.txt'')\n",
    "#path where to save tensorboard files\n",
    "logdir = os.path.join(root_path, 'results/tensorboard_files')\n",
    "\n",
    "#initialise question preprocessor and answer preprocessor\n",
    "answer_preprocessor = AnswerPreprocessing([train_questions_file],[train_answers_file])\n",
    "question_preprocessor = QuestionsPreprocessing(glove_file_path, [train_questions_file], num_occurence)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LyHUMAheCh1C"
   },
   "outputs": [],
   "source": [
    "#WARNING : This is just for glove_300d\n",
    "WORD_EMBEDDING_DIM = 300\n",
    "IMAGE_FEAT_NUM = 10\n",
    "IMAGE_NON_LINEAR_OUTPUT_DIM = TEXT_NON_LINEAR_OUTPUT_DIM = 512\n",
    "\n",
    "question_args = {\n",
    "    \n",
    "    'embedding_size': WORD_EMBEDDING_DIM,\n",
    "    'embedding_weights' : question_preprocessor.get_embedding_matrix(),\n",
    "    'hidden_size' : 1024,\n",
    "    'vocab_len' : question_preprocessor.vocab_length,\n",
    "    'num_layers' : 1,\n",
    "    #'dropout' : 0.3\n",
    "    'dropout' : None\n",
    "}\n",
    "\n",
    "#Num regions, global variable and this one\n",
    "image_args = {\n",
    "    'num_regions' : 10\n",
    "}\n",
    "\n",
    "image_feature_transformer = {\n",
    "    'input_shape' : (2048,),\n",
    "    'dims' : [2048,IMAGE_NON_LINEAR_OUTPUT_DIM],\n",
    "    'activation' :'relu',\n",
    "    'dropout' : None,\n",
    "    'normalization' : False,\n",
    "    'regularisation' : False\n",
    "}\n",
    "\n",
    "text_feature_transformer = {\n",
    "    'input_shape' : (1024,),\n",
    "    'dims' : [1024, TEXT_NON_LINEAR_OUTPUT_DIM],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : None,\n",
    "    'normalization' : False,\n",
    "    'regularisation' : False\n",
    "}\n",
    "\n",
    "attention_vector_args = {\n",
    "    #'dims' : [1024, 1],\n",
    "    'input_shape' : (512,),\n",
    "    'dims' : [1024, IMAGE_NON_LINEAR_OUTPUT_DIM],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'probability_function' : 'softmax',\n",
    "    'regularisation' : False\n",
    "}\n",
    "\n",
    "\n",
    "classifier_args = {\n",
    "    'input_shape' : (1024,),\n",
    "    'dims' : [1024, 512, ANSWER_DIM],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'normalization' : False,\n",
    "    'regularisation' : False\n",
    "}\n",
    "attention_merger_type = 'hadamard'\n",
    "merger_type = 'concat'\n",
    "EPOCHS = 2000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7SZaw4CTYU9U"
   },
   "outputs": [],
   "source": [
    "print(ANSWER_DIM)\n",
    "print(WORD_EMBEDDING_DIM)\n",
    "print(question_preprocessor.get_embedding_matrix().shape)\n",
    "print(IMAGE_FEAT_NUM)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## model program"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0eqPmYIlkJGe"
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "question_args = {\n",
    "    \n",
    "    'embedding_size': WORD_EMBEDDING_DIM,\n",
    "    'embedding_weights' : question_preprocessor.get_embedding_matrix(),\n",
    "    'hidden_size' : 1024,\n",
    "    'vocab_len' : question_preprocessor.vocab_length,\n",
    "    'num_layers' : 1,\n",
    "    #'dropout' : 0.3\n",
    "    'dropout' : None\n",
    "}\n",
    "text_args = {\n",
    "    'input_shape' : (1024,),\n",
    "    'dims' : [1024 ,512],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'normalization' : False,\n",
    "    'regularisation' : 0.01\n",
    "}\n",
    "image_args = {\n",
    "    'input_shape' : (2048,),\n",
    "    'dims' : [2048 ,512],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'normalization' : False,\n",
    "    'regularisation' : False\n",
    "}\n",
    "\n",
    "att_args = {\n",
    "    'input_shape' : (512,),\n",
    "    'dims' : [2048, 1],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'normalization' : False, \n",
    "    'probability_function' : 'softmax',\n",
    "    'regularisation' : 0.01\n",
    "\n",
    "}\n",
    "\n",
    "\n",
    "class_args = {\n",
    "    #'input_shape' : (2560,),\n",
    "    'input_shape' : (512,),\n",
    "    'dims' : [ 2048,  ANSWER_DIM],\n",
    "    'activation' : 'relu',\n",
    "    'dropout' : False,\n",
    "    'normalization' : True,\n",
    "    'regularisation' : 0.01\n",
    "}\n",
    "\n",
    "class NewModel2(keras.Model) : \n",
    "  def __init__(self,question_args, text_args, image_args, att_args, class_args):\n",
    "    super(NewModel2, self).__init__()\n",
    "    self.text_model =  get_question_module('GRU', question_args)\n",
    "    self.resnet =  keras.applications.ResNet50(include_top=False, weights='imagenet', input_tensor=None, input_shape=(224, 224,3), pooling=None)\n",
    "    self.text_net = get_non_linear_function('activation', text_args )\n",
    "    self.image_net = get_non_linear_function('activation', image_args ) \n",
    "    self.DImage = keras.layers.Dense(512)\n",
    "    self.merger1 = keras.layers.Multiply()\n",
    "    self.reshape1 = keras.layers.Reshape((7,7,2048))\n",
    "    self.attention_net = get_non_linear_function('activation', att_args) \n",
    "    self.merger2 = keras.layers.Multiply()\n",
    "    self.classifier = get_non_linear_function('activation', class_args) \n",
    "    self.merger3 = keras.layers.Multiply()\n",
    "    self.optimizer = tf.train.AdamOptimizer(learning_rate = 0.1)\n",
    "  \n",
    "  def call(self,images, questions):\n",
    "    images =  self.reshape1(images)\n",
    "    images = tf.math.l2_normalize(images, axis = -1)\n",
    "    questions = self.text_model(questions)\n",
    "    #questions = tf.to_float(self.text_net(questions))\n",
    "    questions = self.text_net(questions)\n",
    "    questions_att = tf.keras.backend.expand_dims(\n",
    "      questions,\n",
    "      axis=1)\n",
    "    questions_att = tf.keras.backend.expand_dims(\n",
    "      questions_att,\n",
    "      axis=1)\n",
    "    questions_att = tf.cast(questions_att, tf.float32)\n",
    "    questions_att = tf.keras.backend.tile( questions_att,(1,7,7,1))\n",
    "    \n",
    "    att_imgs = self.__func( images, self.DImage)\n",
    "    common1 = self.merger1([questions, att_imgs])\n",
    "    att_vec = self.__func(common1, self.attention_net)\n",
    "    #att_vec = self.reshape1(att_vec)\n",
    "    #att_vec = tf.keras.backend.expand_dims(\n",
    "    #  att_vec,\n",
    "    #  axis=0)\n",
    "    att_vec =  tf.keras.backend.tile( att_vec, (1,1,1,2048))\n",
    "    new_imgs = self.merger2([att_vec, images])\n",
    "    new_imgs = tf.reduce_sum(new_imgs, 1) \n",
    "    new_imgs = tf.reduce_sum(new_imgs, 1) / 49\n",
    "    new_imgs = self.image_net(new_imgs)\n",
    "    common2 = self.merger3([new_imgs, questions])\n",
    "    res = self.classifier(common2)\n",
    "    return res\n",
    "  \n",
    "  @tf.contrib.eager.defun \n",
    "  def __func(self,tensor,fn):\n",
    "    return tf.map_fn(fn, tensor, parallel_iterations=49)\n",
    "  \n",
    "  def loss(self, result, labels):\n",
    "    loss1 = tf.nn.sigmoid_cross_entropy_with_logits(logits = tf.to_double(result), labels = tf.to_double(labels))\n",
    "    s = loss1.shape[-1]\n",
    "    loss11 = tf.reduce_mean(loss1)\n",
    "    return loss11\n",
    "  \n",
    "  def train(self, trains, vals, epochs):\n",
    "    global_step = tf.train.get_or_create_global_step()\n",
    "    writer = tf.contrib.summary.create_file_writer(logdir)\n",
    "    writer.set_as_default()\n",
    "    best_loss = None\n",
    "    #Best loss\n",
    "    exists = os.path.isfile(BEST_LOSS_MODEL_PATH)\n",
    "    if exists :\n",
    "      print('exists')\n",
    "      with open(BEST_LOSS_MODEL_PATH, 'r') as f :\n",
    "        line = f.readlines()[0].strip()\n",
    "        if not line == '' :\n",
    "          best_loss = float(line)\n",
    "          print('init best loss {}'.format(str(best_loss)))\n",
    "      \n",
    "    #Checkpoint\n",
    "    #if not len(os.listdir(CHECKPOINT_PATH)) == 0:\n",
    "    ckpt = tf.train.Checkpoint(step=tf.Variable(0), optimizer=self.optimizer, net=self)\n",
    "    manager = tf.train.CheckpointManager(ckpt, CHECKPOINT_PATH, max_to_keep=5)\n",
    "    ckpt.restore(manager.latest_checkpoint)\n",
    "    \n",
    "    if manager.latest_checkpoint:\n",
    "      print(\"Restored from {}\".format(manager.latest_checkpoint))\n",
    "    else:\n",
    "      print(\"Initializing from scratch.\")\n",
    "    \n",
    "    for i in range(epochs) :\n",
    "      y = time.time()\n",
    "      train_losses = []\n",
    "      val_losses = []\n",
    "      #TRAIN\n",
    "      x = time.time()\n",
    "      spe = time.time()\n",
    "      for ib, (imgs, questions, answers) in enumerate(trains):\n",
    "        if ib % 100 == 0 : \n",
    "          print('batch {}  in {}'.format(ib, time.time() - spe))\n",
    "        x = time.time()\n",
    "        with tf.GradientTape() as tape :\n",
    "          res = self.call(imgs, questions)\n",
    "          loss = self.loss(res, answers)\n",
    "       \n",
    "        gradients = tape.gradient(loss, self.variables)\n",
    "        with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "          tf.contrib.summary.scalar('train_batch_loss', loss)\n",
    "        self.optimizer.apply_gradients(zip(gradients, self.variables))\n",
    "        #print('loss in train batch {} is : {} , time : {}'.format(ib, loss, time.time() - x))\n",
    "        train_losses.append(loss)\n",
    "      mean_train_epoch_loss =  sum(train_losses)/len(train_losses)\n",
    "      print('epoch {} train loss {} in {} s'.format(i, mean_train_epoch_loss, time.time() - y))\n",
    "      with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "        tf.contrib.summary.scalar('train_epoch_loss', mean_train_epoch_loss )  \n",
    "        \n",
    "        \n",
    "      #VAL\n",
    "      if i % 5 == 0 :\n",
    "        y  = time.time()\n",
    "        for iv, (imgs, questions, answers) in enumerate(vals):\n",
    "          x = time.time()\n",
    "          res = self.call(imgs, questions, answers)\n",
    "          val_loss = self.loss(res, answers)    \n",
    "          with tf.contrib.summary.record_summaries_every_n_global_steps(1):\n",
    "            tf.contrib.summary.scalar('val_batch_loss', val_loss)\n",
    "          #print('loss in val batch {} is : {} , time : {}'.format(iv, val_loss, time.time() - x))\n",
    "          val_losses.append(val_loss)\n",
    "        mean_val_epoch_loss = sum(val_losses)/len(val_losses)\n",
    "        with tf.contrib.summary.record_summaries_every_n_global_steps(1):      \n",
    "          tf.contrib.summary.scalar('val_epoch_loss', mean_val_epoch_loss)\n",
    "        if (best_loss == None) or (best_loss > mean_val_epoch_loss) :\n",
    "          best_loss = mean_val_epoch_loss\n",
    "          print('best loss {}'.format(best_loss))\n",
    "          self.save_weights(BEST_MODEL_PATH)\n",
    "          with open(BEST_LOSS_MODEL_PATH, 'w') as f :\n",
    "            f.write(str(float(mean_val_epoch_loss)))\n",
    "        print('epoch {} val loss {} in {} s'.format(i, mean_val_epoch_loss, time.time() - y))\n",
    "      #Saving epoch losses  \n",
    "      \n",
    "        \n",
    "      \n",
    "      \n",
    "      \n",
    "      #checking best weights and saving it\n",
    "      \n",
    "      \n",
    "          \n",
    "      #Saving model          \n",
    "      if int(ckpt.step) % 3 == 0:\n",
    "        save_path = manager.save()\n",
    "      ckpt.step.assign_add(1)\n",
    "    \n",
    "  \n",
    "model4 = NewModel2(question_args,text_args, image_args, att_args, class_args )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## load the dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fMkIFXXsqvVg"
   },
   "outputs": [],
   "source": [
    "image_feature_description = {\n",
    "   \n",
    "    'question': tf.FixedLenFeature([MAX_LEN], tf.int64),\n",
    "    'image': tf.FixedLenFeature([2048*7*7], tf.float32),\n",
    "    'answers': tf.FixedLenFeature([ANSWER_DIM], tf.float32)\n",
    "}\n",
    "\n",
    "def _parse_image_function(example_proto):\n",
    "    example =  tf.parse_single_example(example_proto, image_feature_description)\n",
    "    return (example['image'], example['question'], example['answers'])\n",
    "\n",
    "\n",
    "\n",
    "train_recs = [os.path.join(train_tfrec_dir,x) for x in  os.listdir(train_tfrec_dir) if x[-9:] =='.tfrecord']\n",
    "val_recs = [os.path.join(val_tfrec_dir,x) for x in  os.listdir(val_tfrec_dir) if x[-9:] =='.tfrecord']\n",
    "train_dataset  = tf.data.TFRecordDataset(filenames= train_recs, num_parallel_reads=11).map(\n",
    "    _parse_image_function).batch(BATCH_SIZE).prefetch(PREFETCH_SIZE)\n",
    "#val_dataset = tf.data.TFRecordDataset(filenames= val_recs, num_parallel_reads=6).map(\n",
    "#    _parse_image_function).batch(BATCH_SIZE).prefetch(PREFETCH_SIZE)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7nM3ZjJtZ0A9"
   },
   "outputs": [],
   "source": [
    "#model2.load_weights(BEST_MODEL_PATH)\n",
    "\n",
    "model4.train(train_dataset, val_dataset, EPOCHS)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "VQA_Model.ipynb",
   "provenance": [],
   "toc_visible": true,
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}