COCO_image_captioning.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "COCO image_captioning.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/simran029/ImageVista/blob/master/COCO_image_captioning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "6svGEEek67ds",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "U8l4RJ0XRPEm",
        "colab": {}
      },
      "source": [
        "try:\n",
        "  # %tensorflow_version only exists in Colab.\n",
        "  %tensorflow_version 2.x\n",
        "except Exception:\n",
        "  pass\n",
        "import tensorflow as tf\n",
        "\n",
        "# You'll generate plots of attention in order to see which parts of an image\n",
        "# our model focuses on during captioning\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Scikit-learn includes many helpful utilities\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.utils import shuffle\n",
        "\n",
        "import re\n",
        "import numpy as np\n",
        "import os\n",
        "import time\n",
        "import json\n",
        "from glob import glob\n",
        "from PIL import Image\n",
        "import pickle"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "krQuPYTtRPE7",
        "colab": {}
      },
      "source": [
        "annotation_zip = tf.keras.utils.get_file('captions.zip',\n",
        "                                          cache_subdir=os.path.abspath('.'),\n",
        "                                          origin = 'http://images.cocodataset.org/annotations/annotations_trainval2014.zip',\n",
        "                                          extract = True)\n",
        "annotation_file = os.path.dirname(annotation_zip)+'/annotations/captions_train2014.json'\n",
        "\n",
        "name_of_zip = 'train2014.zip'\n",
        "if not os.path.exists(os.path.abspath('.') + '/' + name_of_zip):\n",
        "  image_zip = tf.keras.utils.get_file(name_of_zip,\n",
        "                                      cache_subdir=os.path.abspath('.'),\n",
        "                                      origin = 'http://images.cocodataset.org/zips/train2014.zip',\n",
        "                                      extract = True)\n",
        "  PATH = os.path.dirname(image_zip)+'/train2014/'\n",
        "else:\n",
        "  PATH = os.path.abspath('.')+'/train2014/'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "4G3b8x8_RPFD",
        "colab": {}
      },
      "source": [
        "# Read the json file\n",
        "with open(annotation_file, 'r') as f:\n",
        "    annotations = json.load(f)\n",
        "\n",
        "# Store captions and image names in vectors\n",
        "all_captions = []\n",
        "all_img_name_vector = []\n",
        "\n",
        "for annot in annotations['annotations']:\n",
        "    caption = '<start> ' + annot['caption'] + ' <end>'\n",
        "    image_id = annot['image_id']\n",
        "    full_coco_image_path = PATH + 'COCO_train2014_' + '%012d.jpg' % (image_id)\n",
        "\n",
        "    all_img_name_vector.append(full_coco_image_path)\n",
        "    all_captions.append(caption)\n",
        "\n",
        "# Shuffle captions and image_names together\n",
        "# Set a random state\n",
        "train_captions, img_name_vector = shuffle(all_captions,\n",
        "                                          all_img_name_vector,\n",
        "                                          random_state=1)\n",
        "\n",
        "# Select the first 30000 captions from the shuffled set\n",
        "num_examples = 30000\n",
        "train_captions = train_captions[:num_examples]\n",
        "img_name_vector = img_name_vector[:num_examples]"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "mPBMgK34RPFL",
        "colab": {}
      },
      "source": [
        "len(train_captions), len(all_captions)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "zXR0217aRPFR",
        "colab": {}
      },
      "source": [
        "def load_image(image_path):\n",
        "    img = tf.io.read_file(image_path)\n",
        "    img = tf.image.decode_jpeg(img, channels=3)\n",
        "    img = tf.image.resize(img, (299, 299))\n",
        "    img = tf.keras.applications.inception_v3.preprocess_input(img)\n",
        "    return img, image_path"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RD3vW4SsRPFW",
        "colab": {}
      },
      "source": [
        "image_model = tf.keras.applications.InceptionV3(include_top=False,\n",
        "                                                weights='imagenet')\n",
        "new_input = image_model.input\n",
        "hidden_layer = image_model.layers[-1].output\n",
        "\n",
        "image_features_extract_model = tf.keras.Model(new_input, hidden_layer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kf3oF6HjmyBj",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tqdm import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Dx_fvbVgRPGQ",
        "colab": {}
      },
      "source": [
        "# Get unique images\n",
        "encode_train = sorted(set(img_name_vector))\n",
        "\n",
        "# Feel free to change batch_size according to your system configuration\n",
        "image_dataset = tf.data.Dataset.from_tensor_slices(encode_train)\n",
        "image_dataset = image_dataset.map(\n",
        "  load_image, num_parallel_calls=tf.data.experimental.AUTOTUNE).batch(16)\n",
        "\n",
        "for img, path in tqdm(image_dataset):\n",
        "  batch_features = image_features_extract_model(img)\n",
        "  batch_features = tf.reshape(batch_features,\n",
        "                              (batch_features.shape[0], -1, batch_features.shape[3]))\n",
        "\n",
        "  for bf, p in zip(batch_features, path):\n",
        "    path_of_feature = p.numpy().decode(\"utf-8\")\n",
        "    np.save(path_of_feature, bf.numpy())"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "HZfK8RhQRPFj",
        "colab": {}
      },
      "source": [
        "# Find the maximum length of any caption in our dataset\n",
        "def calc_max_length(tensor):\n",
        "    return max(len(t) for t in tensor)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "oJGE34aiRPFo",
        "colab": {}
      },
      "source": [
        "# Choose the top 5000 words from the vocabulary\n",
        "top_k = 5000\n",
        "tokenizer = tf.keras.preprocessing.text.Tokenizer(num_words=top_k,\n",
        "                                                  oov_token=\"<unk>\",\n",
        "                                                  filters='!\"#$%&()*+.,-/:;=?@[\\]^_`{|}~ ')\n",
        "tokenizer.fit_on_texts(train_captions)\n",
        "train_seqs = tokenizer.texts_to_sequences(train_captions)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "8Q44tNQVRPFt",
        "colab": {}
      },
      "source": [
        "tokenizer.word_index['<pad>'] = 0\n",
        "tokenizer.index_word[0] = '<pad>'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "0fpJb5ojRPFv",
        "colab": {}
      },
      "source": [
        "# Create the tokenized vectors\n",
        "train_seqs = tokenizer.texts_to_sequences(train_captions)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AidglIZVRPF4",
        "colab": {}
      },
      "source": [
        "# Pad each vector to the max_length of the captions\n",
        "# If you do not provide a max_length value, pad_sequences calculates it automatically\n",
        "cap_vector = tf.keras.preprocessing.sequence.pad_sequences(train_seqs, padding='post')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gL0wkttkRPGA",
        "colab": {}
      },
      "source": [
        "# Calculates the max_length, which is used to store the attention weights\n",
        "max_length = calc_max_length(train_seqs)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "iS7DDMszRPGF",
        "colab": {}
      },
      "source": [
        "# Create training and validation sets using an 80-20 split\n",
        "img_name_train, img_name_val, cap_train, cap_val = train_test_split(img_name_vector,\n",
        "                                                                    cap_vector,\n",
        "                                                                    test_size=0.2,\n",
        "                                                                    random_state=0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "XmViPkRFRPGH",
        "colab": {}
      },
      "source": [
        "len(img_name_train), len(cap_train), len(img_name_val), len(cap_val)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Q3TnZ1ToRPGV",
        "colab": {}
      },
      "source": [
        "# Feel free to change these parameters according to your system's configuration\n",
        "\n",
        "BATCH_SIZE = 64\n",
        "BUFFER_SIZE = 1000\n",
        "embedding_dim = 256\n",
        "units = 512\n",
        "vocab_size = len(tokenizer.word_index) + 1\n",
        "num_steps = len(img_name_train) // BATCH_SIZE\n",
        "# Shape of the vector extracted from InceptionV3 is (64, 2048)\n",
        "# These two variables represent that vector shape\n",
        "features_shape = 2048\n",
        "attention_features_shape = 64"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "SmZS2N0bXG3T",
        "colab": {}
      },
      "source": [
        "# Load the numpy files\n",
        "def map_func(img_name, cap):\n",
        "  img_tensor = np.load(img_name.decode('utf-8')+'.npy')\n",
        "  return img_tensor, cap"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "FDF_Nm3tRPGZ",
        "colab": {}
      },
      "source": [
        "dataset = tf.data.Dataset.from_tensor_slices((img_name_train, cap_train))\n",
        "\n",
        "# Use map to load the numpy files in parallel\n",
        "dataset = dataset.map(lambda item1, item2: tf.numpy_function(\n",
        "          map_func, [item1, item2], [tf.float32, tf.int32]),\n",
        "          num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
        "\n",
        "# Shuffle and batch\n",
        "dataset = dataset.shuffle(BUFFER_SIZE).batch(BATCH_SIZE)\n",
        "dataset = dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "ja2LFTMSdeV3",
        "colab": {}
      },
      "source": [
        "class BahdanauAttention(tf.keras.Model):\n",
        "  def __init__(self, units):\n",
        "    super(BahdanauAttention, self).__init__()\n",
        "    self.W1 = tf.keras.layers.Dense(units)\n",
        "    self.W2 = tf.keras.layers.Dense(units)\n",
        "    self.V = tf.keras.layers.Dense(1)\n",
        "\n",
        "  def call(self, features, hidden):\n",
        "    # features(CNN_encoder output) shape == (batch_size, 64, embedding_dim)\n",
        "\n",
        "    # hidden shape == (batch_size, hidden_size)\n",
        "    # hidden_with_time_axis shape == (batch_size, 1, hidden_size)\n",
        "    hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
        "\n",
        "    # score shape == (batch_size, 64, hidden_size)\n",
        "    score = tf.nn.tanh(self.W1(features) + self.W2(hidden_with_time_axis))\n",
        "\n",
        "    # attention_weights shape == (batch_size, 64, 1)\n",
        "    # you get 1 at the last axis because you are applying score to self.V\n",
        "    attention_weights = tf.nn.softmax(self.V(score), axis=1)\n",
        "\n",
        "    # context_vector shape after sum == (batch_size, hidden_size)\n",
        "    context_vector = attention_weights * features\n",
        "    context_vector = tf.reduce_sum(context_vector, axis=1)\n",
        "\n",
        "    return context_vector, attention_weights"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "AZ7R1RxHRPGf",
        "colab": {}
      },
      "source": [
        "class CNN_Encoder(tf.keras.Model):\n",
        "    # Since you have already extracted the features and dumped it using pickle\n",
        "    # This encoder passes those features through a Fully connected layer\n",
        "    def __init__(self, embedding_dim):\n",
        "        super(CNN_Encoder, self).__init__()\n",
        "        # shape after fc == (batch_size, 64, embedding_dim)\n",
        "        self.fc = tf.keras.layers.Dense(embedding_dim)\n",
        "\n",
        "    def call(self, x):\n",
        "        x = self.fc(x)\n",
        "        x = tf.nn.relu(x)\n",
        "        return x"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "V9UbGQmERPGi",
        "colab": {}
      },
      "source": [
        "class RNN_Decoder(tf.keras.Model):\n",
        "  def __init__(self, embedding_dim, units, vocab_size):\n",
        "    super(RNN_Decoder, self).__init__()\n",
        "    self.units = units\n",
        "\n",
        "    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
        "    self.gru = tf.keras.layers.GRU(self.units,\n",
        "                                   return_sequences=True,\n",
        "                                   return_state=True,\n",
        "                                   recurrent_initializer='glorot_uniform')\n",
        "    self.fc1 = tf.keras.layers.Dense(self.units)\n",
        "    self.fc2 = tf.keras.layers.Dense(vocab_size)\n",
        "\n",
        "    self.attention = BahdanauAttention(self.units)\n",
        "\n",
        "  def call(self, x, features, hidden):\n",
        "    # defining attention as a separate model\n",
        "    context_vector, attention_weights = self.attention(features, hidden)\n",
        "\n",
        "    # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
        "    x = self.embedding(x)\n",
        "\n",
        "    # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
        "    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
        "\n",
        "    # passing the concatenated vector to the GRU\n",
        "    output, state = self.gru(x)\n",
        "\n",
        "    # shape == (batch_size, max_length, hidden_size)\n",
        "    x = self.fc1(output)\n",
        "\n",
        "    # x shape == (batch_size * max_length, hidden_size)\n",
        "    x = tf.reshape(x, (-1, x.shape[2]))\n",
        "\n",
        "    # output shape == (batch_size * max_length, vocab)\n",
        "    x = self.fc2(x)\n",
        "\n",
        "    return x, state, attention_weights\n",
        "\n",
        "  def reset_state(self, batch_size):\n",
        "    return tf.zeros((batch_size, self.units))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Qs_Sr03wRPGk",
        "colab": {}
      },
      "source": [
        "encoder = CNN_Encoder(embedding_dim)\n",
        "decoder = RNN_Decoder(embedding_dim, units, vocab_size)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "-bYN7xA0RPGl",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam()\n",
        "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n",
        "    from_logits=True, reduction='none')\n",
        "\n",
        "def loss_function(real, pred):\n",
        "  mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
        "  loss_ = loss_object(real, pred)\n",
        "\n",
        "  mask = tf.cast(mask, dtype=loss_.dtype)\n",
        "  loss_ *= mask\n",
        "\n",
        "  return tf.reduce_mean(loss_)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "PpJAqPMWo0uE",
        "colab": {}
      },
      "source": [
        "checkpoint_path = \"./checkpoints/train\"\n",
        "ckpt = tf.train.Checkpoint(encoder=encoder,\n",
        "                           decoder=decoder,\n",
        "                           optimizer = optimizer)\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fUkbqhc_uObw",
        "colab": {}
      },
      "source": [
        "start_epoch = 0\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  start_epoch = int(ckpt_manager.latest_checkpoint.split('-')[-1])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "Vt4WZ5mhJE-E",
        "colab": {}
      },
      "source": [
        "# adding this in a separate cell because if you run the training cell\n",
        "# many times, the loss_plot array will be reset\n",
        "loss_plot = []"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "sqgyz2ANKlpU",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def train_step(img_tensor, target):\n",
        "  loss = 0\n",
        "\n",
        "  # initializing the hidden state for each batch\n",
        "  # because the captions are not related from image to image\n",
        "  hidden = decoder.reset_state(batch_size=target.shape[0])\n",
        "\n",
        "  dec_input = tf.expand_dims([tokenizer.word_index['<start>']] * BATCH_SIZE, 1)\n",
        "\n",
        "  with tf.GradientTape() as tape:\n",
        "      features = encoder(img_tensor)\n",
        "\n",
        "      for i in range(1, target.shape[1]):\n",
        "          # passing the features through the decoder\n",
        "          predictions, hidden, _ = decoder(dec_input, features, hidden)\n",
        "\n",
        "          loss += loss_function(target[:, i], predictions)\n",
        "\n",
        "          # using teacher forcing\n",
        "          dec_input = tf.expand_dims(target[:, i], 1)\n",
        "\n",
        "  total_loss = (loss / int(target.shape[1]))\n",
        "\n",
        "  trainable_variables = encoder.trainable_variables + decoder.trainable_variables\n",
        "\n",
        "  gradients = tape.gradient(loss, trainable_variables)\n",
        "\n",
        "  optimizer.apply_gradients(zip(gradients, trainable_variables))\n",
        "\n",
        "  return loss, total_loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "UlA4VIQpRPGo",
        "colab": {}
      },
      "source": [
        "EPOCHS = 50\n",
        "\n",
        "for epoch in range(start_epoch, EPOCHS):\n",
        "    start = time.time()\n",
        "    total_loss = 0\n",
        "\n",
        "    for (batch, (img_tensor, target)) in enumerate(dataset):\n",
        "        batch_loss, t_loss = train_step(img_tensor, target)\n",
        "        total_loss += t_loss\n",
        "\n",
        "        if batch % 100 == 0:\n",
        "            print ('Epoch {} Batch {} Loss {:.4f}'.format(\n",
        "              epoch + 1, batch, batch_loss.numpy() / int(target.shape[1])))\n",
        "    # storing the epoch end loss value to plot later\n",
        "    loss_plot.append(total_loss / num_steps)\n",
        "\n",
        "    if epoch % 5 == 0:\n",
        "      ckpt_manager.save()\n",
        "\n",
        "    print ('Epoch {} Loss {:.6f}'.format(epoch + 1,\n",
        "                                         total_loss/num_steps))\n",
        "    print ('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "1Wm83G-ZBPcC",
        "colab": {}
      },
      "source": [
        "plt.plot(loss_plot)\n",
        "plt.xlabel('Epochs')\n",
        "plt.ylabel('Loss')\n",
        "plt.title('Loss Plot')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "RCWpDtyNRPGs",
        "colab": {}
      },
      "source": [
        "def evaluate(image):\n",
        "    attention_plot = np.zeros((max_length, attention_features_shape))\n",
        "\n",
        "    hidden = decoder.reset_state(batch_size=1)\n",
        "\n",
        "    temp_input = tf.expand_dims(load_image(image)[0], 0)\n",
        "    img_tensor_val = image_features_extract_model(temp_input)\n",
        "    img_tensor_val = tf.reshape(img_tensor_val, (img_tensor_val.shape[0], -1, img_tensor_val.shape[3]))\n",
        "\n",
        "    features = encoder(img_tensor_val)\n",
        "\n",
        "    dec_input = tf.expand_dims([tokenizer.word_index['<start>']], 0)\n",
        "    result = []\n",
        "\n",
        "    for i in range(max_length):\n",
        "        predictions, hidden, attention_weights = decoder(dec_input, features, hidden)\n",
        "\n",
        "        attention_plot[i] = tf.reshape(attention_weights, (-1, )).numpy()\n",
        "\n",
        "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
        "        result.append(tokenizer.index_word[predicted_id])\n",
        "\n",
        "        if tokenizer.index_word[predicted_id] == '<end>':\n",
        "            return result, attention_plot\n",
        "\n",
        "        dec_input = tf.expand_dims([predicted_id], 0)\n",
        "\n",
        "    attention_plot = attention_plot[:len(result), :]\n",
        "    return result, attention_plot"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "fD_y7PD6RPGt",
        "colab": {}
      },
      "source": [
        "def plot_attention(image, result, attention_plot):\n",
        "    temp_image = np.array(Image.open(image))\n",
        "\n",
        "    fig = plt.figure(figsize=(10, 10))\n",
        "\n",
        "    len_result = len(result)\n",
        "    for l in range(len_result):\n",
        "        temp_att = np.resize(attention_plot[l], (8, 8))\n",
        "        ax = fig.add_subplot(len_result//2, len_result//2, l+1)\n",
        "        ax.set_title(result[l])\n",
        "        img = ax.imshow(temp_image)\n",
        "        ax.imshow(temp_att, cmap='gray', alpha=0.6, extent=img.get_extent())\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "7x8RiPHe_4qI",
        "colab": {}
      },
      "source": [
        "# captions on the validation set\n",
        "rid = np.random.randint(0, len(img_name_val))\n",
        "image = img_name_val[rid]\n",
        "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
        "result, attention_plot = evaluate(image)\n",
        "\n",
        "print ('Real Caption:', real_caption)\n",
        "print ('Prediction Caption:', ' '.join(result))\n",
        "# plot_attention(image, result, attention_plot)\n",
        "# opening the image\n",
        "Image.open(img_name_val[rid])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oplcX6qE_DCu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# captions on the validation set\n",
        "rid = np.random.randint(0, len(img_name_val))\n",
        "image = img_name_val[rid]\n",
        "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
        "result, attention_plot = evaluate(image)\n",
        "\n",
        "print ('Real Caption:', real_caption)\n",
        "print ('Prediction Caption:', ' '.join(result))\n",
        "plot_attention(image, result, attention_plot)\n",
        "# opening the image\n",
        "Image.open(img_name_val[rid])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yJy6IYwI_MBM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# captions on the validation set\n",
        "rid = np.random.randint(0, len(img_name_val))\n",
        "image = img_name_val[rid]\n",
        "real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
        "result, attention_plot = evaluate(image)\n",
        "\n",
        "print ('Real Caption:', real_caption)\n",
        "print ('Prediction Caption:', ' '.join(result))\n",
        "plot_attention(image, result, attention_plot)\n",
        "# opening the image\n",
        "Image.open(img_name_val[rid])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "9Psd1quzaAWg",
        "colab": {}
      },
      "source": [
        "image_url = 'https://tensorflow.org/images/surf.jpg'\n",
        "image_extension = image_url[-4:]\n",
        "image_path = tf.keras.utils.get_file('image'+image_extension,\n",
        "                                     origin=image_url)\n",
        "\n",
        "result, attention_plot = evaluate(image_path)\n",
        "print ('Prediction Caption:', ' '.join(result))\n",
        "# plot_attention(image_path, result, attention_plot)\n",
        "# opening the image\n",
        "Image.open(image_path)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V4ptccWNn5dJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "true_results = list()\n",
        "predictions = list()\n",
        "for i in range(0, len(img_name_val)):\n",
        "  image = img_name_val[rid]\n",
        "  real_caption = ' '.join([tokenizer.index_word[i] for i in cap_val[rid] if i not in [0]])\n",
        "  result, attention_plot = evaluate(image)\n",
        "  true_results.append(real_caption)\n",
        "  predictions.append(' '.join(result))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u3hMoj0cvB2Z",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from nltk.translate.bleu_score import corpus_bleu\n",
        "score1 = corpus_bleu(true_results, predictions, weights=(1, 0, 0, 0))\n",
        "score2 = corpus_bleu(true_results, predictions, weights=(0.5,0.5, 0, 0))\n",
        "score3 = corpus_bleu(true_results, predictions, weights=(0.33, 0.33, 0.33, 0))\n",
        "score4 = corpus_bleu(true_results, predictions, weights=(0.25, 0.25, 0.25, 0.25))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5NTMbyidwhGW",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print(score1, score2, score3, score4)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}