seq2seq_with_Attention.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"Copy of seq2seq_with_Attention.ipynb","provenance":[{"file_id":"1_ae34AFfOU8dTybNb69-tonDwhOm7aMx","timestamp":1594124584048}],"authorship_tag":"ABX9TyP/Of21JBiSLjrbjHfPipgr"},"kernelspec":{"name":"python3","display_name":"Python 3"}},"cells":[{"cell_type":"code","metadata":{"id":"wyWfOYqel1Vt","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":122},"executionInfo":{"status":"ok","timestamp":1594124654964,"user_tz":-330,"elapsed":27592,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"d7e3307a-5a98-4ddb-bbb1-582c5f7861f0"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n","\n","Enter your authorization code:\n","··········\n","Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Ck5eP4AopmiL","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124660853,"user_tz":-330,"elapsed":3283,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["import tensorflow as tf\n","\n","import matplotlib.pyplot as plt\n","import matplotlib.ticker as ticker\n","from sklearn.model_selection import train_test_split\n","\n","import unicodedata\n","import re\n","import numpy as np\n","import os\n","import io\n","import time"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"id":"NzD-Skjkp2gr","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124664895,"user_tz":-330,"elapsed":1916,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["task1 = open(\"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/dialog-bAbI-tasks/dialog-babi-task1-API-calls-trn.txt\",'r')\n","task2 = open(\"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/dialog-bAbI-tasks/dialog-babi-task2-API-refine-trn.txt\",'r')\n","files = [task1,task2]"],"execution_count":3,"outputs":[]},{"cell_type":"code","metadata":{"id":"9fE-LUUbp8U0","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124667481,"user_tz":-330,"elapsed":1517,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["questions = list()\n","answers = list()"],"execution_count":4,"outputs":[]},{"cell_type":"code","metadata":{"id":"5QZL6vwjqAwM","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1594124671736,"user_tz":-330,"elapsed":1716,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"89b0217f-4d07-4056-e319-3ab279b09d9c"},"source":["ind = 0\n","for file in files:\n","    print(\"inside file\")\n","    data = file.read()\n","    lines = data.split(\"\\n\")\n","    for i in lines:\n","        if len(i)<1:\n","            continue\n","        inout=i.split(\"\\t\")\n","        if(len(inout)==1):\n","            out = inout[0]\n","            out = out.split(out.split()[0])\n","            answers[ind-1] = answers[ind-1]+ \"\\n\" +out[1]\n","        else:\n","            inp,out = inout[0],inout[1]\n","            inp=inp.split(inp.split()[0])\n","            if(inp[1]!=' <SILENCE>'):\n","                questions.append(inp[1])\n","                answers.append(out)\n","                ind+=1\n","            else:\n","                answers[ind-1] = answers[ind-1]+', '+out \n","    file.close()"],"execution_count":5,"outputs":[{"output_type":"stream","text":["inside file\n","inside file\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"CfcLE3HBqnpm","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124676686,"user_tz":-330,"elapsed":1554,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["save = open(\"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/attention_data.txt\",\"w\")\n","for i in range(len(questions)):\n","  save.write(questions[i].strip()+\"\\t\"+answers[i].strip()+\"\\n\")\n","save.close()"],"execution_count":6,"outputs":[]},{"cell_type":"code","metadata":{"id":"vHGuN3wiqw2t","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124680637,"user_tz":-330,"elapsed":1458,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["path_to_file = \"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/attention_data.txt\""],"execution_count":7,"outputs":[]},{"cell_type":"code","metadata":{"id":"WidoG7YUrBN_","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594124822002,"user_tz":-330,"elapsed":1325,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["def unicode_to_ascii(s):\n","  return ''.join(c for c in unicodedata.normalize('NFD', s)\n","      if unicodedata.category(c) != 'Mn')\n","\n","\n","def preprocess_sentence(w):\n","  w = unicode_to_ascii(w.lower().strip())\n","\n","  w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n","  w = re.sub(r'[\" \"]+', \" \", w)\n","\n","  w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n","\n","  w = w.strip()\n","\n","  w = '<start> ' + w + ' <end>'\n","  return w"],"execution_count":11,"outputs":[]},{"cell_type":"code","metadata":{"id":"rWssDnEwrTj4","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1594124998963,"user_tz":-330,"elapsed":1324,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"79c85236-edba-411d-9b15-25d20a142020"},"source":["qn_sentence = u\"good morning\"\n","ans_sentence = u\"hello what can i help you with today\"\n","print(preprocess_sentence(qn_sentence))\n","print(preprocess_sentence(ans_sentence).encode('utf-8'))"],"execution_count":15,"outputs":[{"output_type":"stream","text":["<start> good morning <end>\n","b'<start> hello what can i help you with today <end>'\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"ucZ3--DArfGp","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125002829,"user_tz":-330,"elapsed":1502,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["def create_dataset(path, num_examples):\n","  lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n","\n","  word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n","\n","  return zip(*word_pairs)"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"id":"X9GprL9crmuj","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1594125005362,"user_tz":-330,"elapsed":1559,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"b57b4bc4-41c2-4405-c1f2-0c215ce1058e"},"source":["qn, ans = create_dataset(path_to_file, None)\n","print(qn[-1])\n","print(ans[-1])"],"execution_count":17,"outputs":[{"output_type":"stream","text":["<start> thanks <end>\n","<start> you re welcome <end>\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"JmL5ssg6r2k1","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125332077,"user_tz":-330,"elapsed":1395,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["def tokenize(stmt):\n","  stmt_tokenizer = tf.keras.preprocessing.text.Tokenizer(\n","      filters='')\n","  stmt_tokenizer.fit_on_texts(stmt)\n","\n","  tensor = stmt_tokenizer.texts_to_sequences(stmt)\n","\n","  tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor,\n","                                                         padding='post')\n","\n","  return tensor, stmt_tokenizer"],"execution_count":25,"outputs":[]},{"cell_type":"code","metadata":{"id":"nZTGe40xu_Gd","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125335260,"user_tz":-330,"elapsed":1830,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["def load_dataset(path, num_examples=None):\n","  inp_stmt,targ_stmt  = create_dataset(path, num_examples)\n","\n","  input_tensor, inp_stmt_tokenizer = tokenize(inp_stmt)\n","  target_tensor, targ_stmt_tokenizer = tokenize(targ_stmt)\n","\n","  return input_tensor, target_tensor, inp_stmt_tokenizer, targ_stmt_tokenizer"],"execution_count":26,"outputs":[]},{"cell_type":"code","metadata":{"id":"O4yrZKfovB1k","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125338900,"user_tz":-330,"elapsed":2296,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["num_examples = 10522\n","input_tensor, target_tensor, inp_stmt, targ_stmt = load_dataset(path_to_file, num_examples)\n","\n","max_length_targ, max_length_inp = target_tensor.shape[1], input_tensor.shape[1]"],"execution_count":27,"outputs":[]},{"cell_type":"code","metadata":{"id":"yORVwJBevE7n","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1594125379223,"user_tz":-330,"elapsed":1449,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"a248d443-4c50-4839-ba19-8aac034ad96d"},"source":["input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.1)\n","\n","print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))"],"execution_count":28,"outputs":[{"output_type":"stream","text":["9469 9469 1053 1053\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"jpcrBvVNvH9Q","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125397359,"user_tz":-330,"elapsed":1569,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["def convert(stmt, tensor):\n","  for t in tensor:\n","    if t!=0:\n","      print (\"%d ----> %s\" % (t, stmt.index_word[t]))"],"execution_count":29,"outputs":[]},{"cell_type":"code","metadata":{"id":"WZAS3oJ8vK40","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":289},"executionInfo":{"status":"ok","timestamp":1594125444081,"user_tz":-330,"elapsed":1267,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"4785f611-9e5d-46df-aa92-217ca46b83c8"},"source":["print (\"Input; index to word mapping\")\n","convert(inp_stmt, input_tensor_train[0])\n","print ()\n","print (\"Output; index to word mapping\")\n","convert(targ_stmt, target_tensor_train[0])"],"execution_count":30,"outputs":[{"output_type":"stream","text":["Input; index to word mapping\n","1 ----> <start>\n","5 ----> for\n","36 ----> six\n","27 ----> please\n","2 ----> <end>\n","\n","Output; index to word mapping\n","1 ----> <start>\n","59 ----> which\n","60 ----> price\n","61 ----> range\n","62 ----> are\n","63 ----> looking\n","6 ----> for\n","2 ----> <end>\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"Cx6hwcTPvPgo","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125476102,"user_tz":-330,"elapsed":1322,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["BUFFER_SIZE = len(input_tensor_train)\n","BATCH_SIZE = 64\n","steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n","embedding_dim = 256\n","units = 1024\n","vocab_inp_size = len(inp_stmt.word_index)+1\n","vocab_tar_size = len(targ_stmt.word_index)+1\n","\n","dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n","dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"],"execution_count":31,"outputs":[]},{"cell_type":"code","metadata":{"id":"PZSsD0n3vSNF","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1594125483169,"user_tz":-330,"elapsed":1336,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"8ece517d-31df-4859-94d1-19db52bdfdb6"},"source":["example_input_batch, example_target_batch = next(iter(dataset))\n","example_input_batch.shape, example_target_batch.shape"],"execution_count":32,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(TensorShape([64, 22]), TensorShape([64, 23]))"]},"metadata":{"tags":[]},"execution_count":32}]},{"cell_type":"code","metadata":{"id":"LE8ewirVvVO_","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125503917,"user_tz":-330,"elapsed":1484,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["class Encoder(tf.keras.Model):\n","  def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n","    super(Encoder, self).__init__()\n","    self.batch_sz = batch_sz\n","    self.enc_units = enc_units\n","    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n","    self.gru = tf.keras.layers.GRU(self.enc_units,\n","                                   return_sequences=True,\n","                                   return_state=True,\n","                                   recurrent_initializer='glorot_uniform')\n","\n","  def call(self, x, hidden):\n","    x = self.embedding(x)\n","    output, state = self.gru(x, initial_state = hidden)\n","    return output, state\n","\n","  def initialize_hidden_state(self):\n","    return tf.zeros((self.batch_sz, self.enc_units))"],"execution_count":33,"outputs":[]},{"cell_type":"code","metadata":{"id":"CAgb45bvvYmZ","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1594125521032,"user_tz":-330,"elapsed":1781,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"40f67d5c-2b83-4601-b6fd-a3b79fdf4762"},"source":["encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n","\n","sample_hidden = encoder.initialize_hidden_state()\n","sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n","print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n","print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"],"execution_count":34,"outputs":[{"output_type":"stream","text":["Encoder output shape: (batch size, sequence length, units) (64, 22, 1024)\n","Encoder Hidden state shape: (batch size, units) (64, 1024)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6GaetKt2vc47","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125607101,"user_tz":-330,"elapsed":1615,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["class BahdanauAttention(tf.keras.layers.Layer):\n","  def __init__(self, units):\n","    super(BahdanauAttention, self).__init__()\n","    self.W1 = tf.keras.layers.Dense(units)\n","    self.W2 = tf.keras.layers.Dense(units)\n","    self.V = tf.keras.layers.Dense(1)\n","\n","  def call(self, query, values):\n","    query_with_time_axis = tf.expand_dims(query, 1)\n","    score = self.V(tf.nn.tanh(\n","        self.W1(query_with_time_axis) + self.W2(values)))\n","\n","    attention_weights = tf.nn.softmax(score, axis=1)\n","\n","    context_vector = attention_weights * values\n","    context_vector = tf.reduce_sum(context_vector, axis=1)\n","\n","    return context_vector, attention_weights"],"execution_count":35,"outputs":[]},{"cell_type":"code","metadata":{"id":"-EFouwU0viUm","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"executionInfo":{"status":"ok","timestamp":1594125616965,"user_tz":-330,"elapsed":1512,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"c7d5d28f-e0bd-4e05-bb29-2737d464fbbb"},"source":["attention_layer = BahdanauAttention(10)\n","attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n","\n","print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n","print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"],"execution_count":36,"outputs":[{"output_type":"stream","text":["Attention result shape: (batch size, units) (64, 1024)\n","Attention weights shape: (batch_size, sequence_length, 1) (64, 22, 1)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"qfa_-XAxvni9","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125808945,"user_tz":-330,"elapsed":1578,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["class Decoder(tf.keras.Model):\n","  def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n","    super(Decoder, self).__init__()\n","    self.batch_sz = batch_sz\n","    self.dec_units = dec_units\n","    self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n","    self.gru = tf.keras.layers.GRU(self.dec_units,\n","                                   return_sequences=True,\n","                                   return_state=True,\n","                                   recurrent_initializer='glorot_uniform')\n","    self.fc = tf.keras.layers.Dense(vocab_size)\n","\n","    self.attention = BahdanauAttention(self.dec_units)\n","\n","  def call(self, x, hidden, enc_output):\n","    context_vector, attention_weights = self.attention(hidden, enc_output)\n","    x = self.embedding(x)\n","\n","    x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n","    output, state = self.gru(x)\n","\n","    output = tf.reshape(output, (-1, output.shape[2]))\n","\n","    x = self.fc(output)\n","\n","    return x, state, attention_weights"],"execution_count":43,"outputs":[]},{"cell_type":"code","metadata":{"id":"B07_pYABvq2x","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":34},"executionInfo":{"status":"ok","timestamp":1594125814211,"user_tz":-330,"elapsed":1211,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}},"outputId":"6d6ce73d-7e5f-42bc-877a-75b3feb3776a"},"source":["decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n","\n","sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)),\n","                                      sample_hidden, sample_output)\n","\n","print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"],"execution_count":44,"outputs":[{"output_type":"stream","text":["Decoder output shape: (batch_size, vocab size) (64, 72)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"6uIlkCElvw5W","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125835135,"user_tz":-330,"elapsed":1050,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["optimizer = tf.keras.optimizers.Adam()\n","loss_object = tf.keras.losses.SparseCategoricalCrossentropy(\n","    from_logits=True, reduction='none')\n","\n","def loss_function(real, pred):\n","  mask = tf.math.logical_not(tf.math.equal(real, 0))\n","  loss_ = loss_object(real, pred)\n","\n","  mask = tf.cast(mask, dtype=loss_.dtype)\n","  loss_ *= mask\n","\n","  return tf.reduce_mean(loss_)"],"execution_count":45,"outputs":[]},{"cell_type":"code","metadata":{"id":"sJxouC3NvzwR","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125842809,"user_tz":-330,"elapsed":1456,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["checkpoint_dir = '/content/drive/My Drive/chatbot/attention_training_checkpoints'\n","checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n","checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n","                                 encoder=encoder,\n","                                 decoder=decoder)"],"execution_count":46,"outputs":[]},{"cell_type":"code","metadata":{"id":"aMVLjb9lv4VQ","colab_type":"code","colab":{},"executionInfo":{"status":"ok","timestamp":1594125893927,"user_tz":-330,"elapsed":1213,"user":{"displayName":"Akshaya Arivazhagan","photoUrl":"","userId":"07509604254271457318"}}},"source":["@tf.function\n","def train_step(inp, targ, enc_hidden):\n","  loss = 0\n","\n","  with tf.GradientTape() as tape:\n","    enc_output, enc_hidden = encoder(inp, enc_hidden)\n","\n","    dec_hidden = enc_hidden\n","\n","    dec_input = tf.expand_dims([targ_stmt.word_index['<start>']] * BATCH_SIZE, 1)\n","\n","    for t in range(1, targ.shape[1]):\n","      predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n","\n","      loss += loss_function(targ[:, t], predictions)\n","\n","      dec_input = tf.expand_dims(targ[:, t], 1)\n","\n","  batch_loss = (loss / int(targ.shape[1]))\n","\n","  variables = encoder.trainable_variables + decoder.trainable_variables\n","\n","  gradients = tape.gradient(loss, variables)\n","\n","  optimizer.apply_gradients(zip(gradients, variables))\n","\n","  return batch_loss"],"execution_count":47,"outputs":[]},{"cell_type":"code","metadata":{"id":"D_hVd4lNv9P5","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":51},"outputId":"59963f64-6815-4781-86cf-2c37caa5560a"},"source":["EPOCHS = 15\n","loss_list = []\n","epochs_list = []\n","for epoch in range(EPOCHS):\n","  start = time.time()\n","\n","  enc_hidden = encoder.initialize_hidden_state()\n","  total_loss = 0\n","\n","  for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n","    batch_loss = train_step(inp, targ, enc_hidden)\n","    total_loss += batch_loss\n","\n","    if batch % 100 == 0:\n","      print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n","                                                   batch,\n","                                                   batch_loss.numpy()))\n","\n","  if (epoch + 1) % 2 == 0:\n","    checkpoint.save(file_prefix = checkpoint_prefix)\n","  print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n","                                      total_loss / steps_per_epoch))\n","  loss_list.append(total_loss / steps_per_epoch)\n","  epochs_list.append(epoch + 1)\n","  print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Epoch 1 Batch 0 Loss 2.1090\n","Epoch 1 Batch 100 Loss 0.1050\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"3rytLqZcv_TS","colab_type":"code","colab":{}},"source":["import matplotlib.pyplot as plt\n","plt.plot(epochs_list,loss_list)\n","plt.title('model loss')\n","plt.ylabel('loss')\n","plt.xlabel('epoch')\n","plt.legend(['train'], loc='upper left')\n","plt.show()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"YBsXhCzewCU9","colab_type":"code","colab":{}},"source":["def evaluate(sentence):\n","\n","  sentence = preprocess_sentence(sentence)\n","\n","  inputs = [inp_stmt.word_index[i] for i in sentence.split(' ')]\n","  inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n","                                                         maxlen=max_length_inp,\n","                                                         padding='post')\n","  inputs = tf.convert_to_tensor(inputs)\n","\n","  result = ''\n","\n","  hidden = [tf.zeros((1, units))]\n","  enc_out, enc_hidden = encoder(inputs, hidden)\n","\n","  dec_hidden = enc_hidden\n","  dec_input = tf.expand_dims([targ_stmt.word_index['<start>']], 0)\n","\n","  for t in range(max_length_targ):\n","    predictions, dec_hidden, attention_weights = decoder(dec_input,\n","                                                         dec_hidden,\n","                                                         enc_out)\n","    \n","    attention_weights = tf.reshape(attention_weights, (-1, ))\n"," \n","    predicted_id = tf.argmax(predictions[0]).numpy()\n","\n","    result += targ_stmt.index_word[predicted_id] + ' '\n","\n","    if targ_stmt.index_word[predicted_id] == '<end>':\n","      return result, sentence\n","\n","    dec_input = tf.expand_dims([predicted_id], 0)\n","\n","  return result, sentence"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"het0MQjywHW9","colab_type":"code","colab":{}},"source":["def reply(sentence):\n","  result, sentence = evaluate(sentence)\n","\n","  return str(result)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"MGdx1HgTwMuQ","colab_type":"code","colab":{}},"source":["checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Xmsvdmq2wPMM","colab_type":"code","colab":{}},"source":["import spacy\n","nlp = spacy.load('en_core_web_sm')\n","reply(\"hello\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"WrAM0FDomq3E","colab":{}},"source":["print(\"enter 'close' to close this chat bot\\n\\n\")\n","res = \"\"\n","loc = \"\"\n","hmany = \"\"\n","price = \"\"\n","inp = \"\"\n","while(True):\n","    try:\n","        inp = input( 'Enter question : ' )\n","        if(inp==\"close\"):\n","            break\n","        test = nlp(inp)\n","        for word in test.ents:\n","            if(word.label_==\"NORP\"):\n","                res = word.text\n","#                 print(\"food type\",word.text)\n","            if(word.label_==\"CARDINAL\"):\n","                hmany = word.text\n","#                 print(\"no of people\",word.text)\n","            if(word.label_==\"GPE\"):\n","                loc = word.text\n","#                 print(\"location\",word.text)\n","        if(\"moderate\" in inp):\n","            price = \"moderate\"\n","        elif(\"expensive\" in inp):\n","            price = \"expensive\"\n","        elif(\"cheap\" in inp):\n","            price = \"cheap\"\n","        decoded_reply = reply(inp)\n","        if(decoded_reply==\"you re welcome <end> \" or decoded_reply==\"hello what can i help you with today <end> \"):\n","            res = \"\"\n","            loc = \"\"\n","            hmany = \"\"\n","            price = \"\"\n","        if (\"api call\" in decoded_reply):\n","            l = list(decoded_reply.rsplit(\"api call\",1)[1].strip().split(\" \"))\n","            if(res!=l[0] or hmany!=l[2] or loc!=l[1] or price!=l[3]):\n","                if(res==\"\"):\n","                    res = l[0]\n","                if(hmany==\"\"):\n","                    hmany = l[2]\n","                if(loc == \"\"):\n","                    loc = l[1]\n","                if(price == \"\"):\n","                    price = l[3]\n","                temp1,temp2 = decoded_reply.rsplit(\"api call\",1)\n","                decoded_reply = temp1+\"api call \"+res+\" \"+loc+\" \"+hmany+\" \"+price+\" end\"\n","        print( decoded_reply.replace(\"  \",\" \"))\n","        \n","    except KeyError:\n","        print(\"sorry i dont know those words!\")"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"6cythiLNwbBH","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"-vSkowrq9WPQ","colab":{}},"source":["task6 = open(\"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/dialog-bAbI-tasks/dialog-babi-task1-API-calls-tst.txt\",'r')\n","task7 = open(\"/content/drive/My Drive/NLP/dialog-bAbI-tasks_1_/dialog-bAbI-tasks/dialog-babi-task2-API-refine-tst.txt\",'r')\n","files = [task6,task7]\n","questions = list()\n","answers = list()\n","ind = 0\n","for file in files:\n","    print(\"inside file\")\n","    data = file.read()\n","    lines = data.split(\"\\n\")\n","    for i in lines:\n","        if len(i)<1:\n","            continue\n","        i = i.replace(\"_\",\" \")\n","        inout=i.split(\"\\t\")\n","        if(len(inout)==1):\n","            out = inout[0]\n","            out = out.split(out.split()[0])\n","            answers[ind-1] = answers[ind-1]+ \"\\n\" +out[1]\n","        else:\n","            inp,out = inout[0],inout[1]\n","            inp=inp.split(inp.split()[0])\n","            if(inp[1]!=' <SILENCE>'):\n","                questions.append(inp[1])\n","                answers.append(out)\n","                ind+=1\n","            else:\n","                answers[ind-1] = answers[ind-1]+', '+out \n","    file.close()"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"F9vCOxE_9iYl","colab":{}},"source":["answers_with_tags = list()\n","for i in range( len( answers ) ):\n","    if type( answers[i] ) == str:\n","        answers_with_tags.append( preprocess_sentence(answers[i]) )\n","    else:\n","        questions.pop( i )"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"21Fh929zwi4n","colab_type":"code","colab":{}},"source":["len(questions)\n","len(answers_with_tags)"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"zo9Iy8cc9qir","colab":{}},"source":["res = \"\"\n","loc = \"\"\n","hmany = \"\"\n","price = \"\"\n","inp = \"\"\n","var = 0\n","count = 0\n","while(var<len(questions)):\n","    try:\n","        inp = questions[var]\n","        if(inp==\"close\"):\n","            break\n","        test = nlp(inp)\n","        for word in test.ents:\n","            if(word.label_==\"NORP\"):\n","                res = word.text\n","\n","            if(word.label_==\"CARDINAL\"):\n","                hmany = word.text\n","\n","            if(word.label_==\"GPE\"):\n","                loc = word.text\n","\n","        if(\"moderate\" in inp):\n","            price = \"moderate\"\n","        elif(\"expensive\" in inp):\n","            price = \"expensive\"\n","        elif(\"cheap\" in inp):\n","            price = \"cheap\"\n","        decoded_reply = reply(inp)\n","        if(decoded_reply==\"you re welcome <end> \" or reply==\"hello what can i help you with today <end> \"):\n","            res = \"\"\n","            loc = \"\"\n","            hmany = \"\"\n","            price = \"\"\n","        if (\"api call\" in decoded_reply):\n","            l = list(decoded_reply.rsplit(\"api call\",1)[1].strip().split(\" \"))\n","            if(res!=l[0] or hmany!=l[2] or loc!=l[1] or price!=l[3]):\n","                if(res==\"\"):\n","                    res = l[0]\n","                if(hmany==\"\"):\n","                    hmany = l[2]\n","                if(loc == \"\"):\n","                    loc = l[1]\n","                if(price == \"\"):\n","                    price = l[3]\n","                temp1,temp2 = decoded_reply.rsplit(\"api call\",1)\n","                decoded_reply = temp1+\"api call \"+res+\" \"+loc+\" \"+hmany+\" \"+price+\" <end>\"\n","        decoded_reply = decoded_reply.replace(\"  \",\" \")\n","        decoded_reply = decoded_reply.replace(\"<end>\",\"\")\n","        decoded_reply = decoded_reply.strip()\n","        answers_with_tags[var] = answers_with_tags[var].replace(\"<end>\",\"\")\n","        answers_with_tags[var] = answers_with_tags[var].replace(\"<start>\",\"\")\n","        answers_with_tags[var] = answers_with_tags[var].strip()\n","        if(answers_with_tags[var] == decoded_reply):\n","            count+=1\n","        else:\n","            print(decoded_reply,var)\n","            print(answers_with_tags[var],var)\n","\n","    except KeyError:\n","        print(\"sorry i dont know those words!\")\n","    var+=1"],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Y38hJznIwxhi","colab_type":"code","colab":{}},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"colab_type":"code","id":"6SuuLT_w90G9","colab":{}},"source":["print(\"total = \",len(questions),\"\\ncorrect = \",count)\n","print((count*100)/len(questions),\"%\")"],"execution_count":null,"outputs":[]}]}