redaction_imputation_v1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Import-Packages\" data-toc-modified-id=\"Import-Packages-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Import Packages</a></span></li><li><span><a href=\"#Loading-and-Cleaning-Data\" data-toc-modified-id=\"Loading-and-Cleaning-Data-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Loading and Cleaning Data</a></span><ul class=\"toc-item\"><li><span><a href=\"#Load-Data\" data-toc-modified-id=\"Load-Data-2.1\"><span class=\"toc-item-num\">2.1&nbsp;&nbsp;</span>Load Data</a></span></li><li><span><a href=\"#Clean-Raw-Text\" data-toc-modified-id=\"Clean-Raw-Text-2.2\"><span class=\"toc-item-num\">2.2&nbsp;&nbsp;</span>Clean Raw Text</a></span></li><li><span><a href=\"#Character-Mappings\" data-toc-modified-id=\"Character-Mappings-2.3\"><span class=\"toc-item-num\">2.3&nbsp;&nbsp;</span>Character Mappings</a></span></li></ul></li><li><span><a href=\"#Creating-train-and-test-data-objects\" data-toc-modified-id=\"Creating-train-and-test-data-objects-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>Creating train and test data objects</a></span><ul class=\"toc-item\"><li><span><a href=\"#Dataset-Generation-Functions\" data-toc-modified-id=\"Dataset-Generation-Functions-3.1\"><span class=\"toc-item-num\">3.1&nbsp;&nbsp;</span>Dataset Generation Functions</a></span></li><li><span><a href=\"#Creating-Datasets\" data-toc-modified-id=\"Creating-Datasets-3.2\"><span class=\"toc-item-num\">3.2&nbsp;&nbsp;</span>Creating Datasets</a></span></li></ul></li><li><span><a href=\"#Models\" data-toc-modified-id=\"Models-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>Models</a></span><ul class=\"toc-item\"><li><span><a href=\"#750-Sequence-Length-Model\" data-toc-modified-id=\"750-Sequence-Length-Model-4.1\"><span class=\"toc-item-num\">4.1&nbsp;&nbsp;</span>750 Sequence Length Model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Parameters\" data-toc-modified-id=\"Parameters-4.1.1\"><span class=\"toc-item-num\">4.1.1&nbsp;&nbsp;</span>Parameters</a></span></li><li><span><a href=\"#Set-up-Checkpoint-Path\" data-toc-modified-id=\"Set-up-Checkpoint-Path-4.1.2\"><span class=\"toc-item-num\">4.1.2&nbsp;&nbsp;</span>Set up Checkpoint Path</a></span></li><li><span><a href=\"#Build-model\" data-toc-modified-id=\"Build-model-4.1.3\"><span class=\"toc-item-num\">4.1.3&nbsp;&nbsp;</span>Build model</a></span></li><li><span><a href=\"#Fit-Model\" data-toc-modified-id=\"Fit-Model-4.1.4\"><span class=\"toc-item-num\">4.1.4&nbsp;&nbsp;</span>Fit Model</a></span></li><li><span><a href=\"#1-Batch-Model-and-Imputing-Redactions\" data-toc-modified-id=\"1-Batch-Model-and-Imputing-Redactions-4.1.5\"><span class=\"toc-item-num\">4.1.5&nbsp;&nbsp;</span>1 Batch Model and Imputing Redactions</a></span></li></ul></li><li><span><a href=\"#400-Sequence-Length-Model\" data-toc-modified-id=\"400-Sequence-Length-Model-4.2\"><span class=\"toc-item-num\">4.2&nbsp;&nbsp;</span>400 Sequence Length Model</a></span><ul class=\"toc-item\"><li><span><a href=\"#Parameters\" data-toc-modified-id=\"Parameters-4.2.1\"><span class=\"toc-item-num\">4.2.1&nbsp;&nbsp;</span>Parameters</a></span></li><li><span><a href=\"#Set-up-Checkpoint-Path\" data-toc-modified-id=\"Set-up-Checkpoint-Path-4.2.2\"><span class=\"toc-item-num\">4.2.2&nbsp;&nbsp;</span>Set up Checkpoint Path</a></span></li><li><span><a href=\"#Build-Model\" data-toc-modified-id=\"Build-Model-4.2.3\"><span class=\"toc-item-num\">4.2.3&nbsp;&nbsp;</span>Build Model</a></span></li><li><span><a href=\"#Fit-Model\" data-toc-modified-id=\"Fit-Model-4.2.4\"><span class=\"toc-item-num\">4.2.4&nbsp;&nbsp;</span>Fit Model</a></span></li><li><span><a href=\"#1-Batch-and-Redactions\" data-toc-modified-id=\"1-Batch-and-Redactions-4.2.5\"><span class=\"toc-item-num\">4.2.5&nbsp;&nbsp;</span>1 Batch and Redactions</a></span></li></ul></li></ul></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Packages\n",
    "\n",
    "The model will be made in TensorFlow using [Eager Execution mode](https://www.tensorflow.org/guide/eager). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:32.393942Z",
     "start_time": "2019-08-13T21:11:12.865138Z"
    }
   },
   "outputs": [],
   "source": [
    "#from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.keras.optimizer_v2.adam import Adam\n",
    "tf.compat.v1.enable_eager_execution()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading and Cleaning Data\n",
    "\n",
    "\n",
    "### Load Data\n",
    "I frequently trained models using google colab so I have an option to mount and load a google drive folder. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:32.424209Z",
     "start_time": "2019-08-13T21:11:32.421733Z"
    }
   },
   "outputs": [],
   "source": [
    "### Option to run notebook using google colab as data storage\n",
    "drive_load = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:32.484311Z",
     "start_time": "2019-08-13T21:11:32.450487Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['\\n', '\\r', ' ', '!', '\"', '#', '$', '%', '&', \"'\", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '[', '\\\\', ']', '_', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '|', '§', '¶', 'é', 'ü', 'ſ', 'а', 'в', 'д', 'е', 'и', 'к', 'м', 'н', 'о', 'п', 'р', 'т', 'ч', 'ы', 'я', '–', '—', '‘', '’', '“', '”', '…', '■']\n"
     ]
    }
   ],
   "source": [
    "if drive_load == True:\n",
    "    from google.colab import drive\n",
    "    drive.mount('/content/gdrive')\n",
    "    \n",
    "    path_to_file = \"/content/gdrive/My Drive/code/redaction model/strippedtext.txt\"\n",
    "else:\n",
    "    path_to_file = \"strippedtext.txt\"\n",
    "\n",
    "\n",
    "raw_text = open(path_to_file, 'rb').read().decode(encoding='utf-8')\n",
    "unfiltered_vocab = sorted(set(raw_text))\n",
    "print(unfiltered_vocab)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clean Raw Text\n",
    "\n",
    "Remove a bunch of specific characters from the text. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:32.565608Z",
     "start_time": "2019-08-13T21:11:32.509874Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of text pre-substituion : 1320770 characters.\n",
      "120 unique characters after substitution\n",
      "\n",
      "Length of text post-substituion : 1304546 characters. \n",
      "76 unique characters after substitution. \n",
      "\n",
      "Pre-to-Post difference : 16224 characters.\n"
     ]
    }
   ],
   "source": [
    "print('Length of text pre-substituion : {} characters.'.format(len(raw_text)))\n",
    "print('{} unique characters after substitution'.format(len(unfiltered_vocab)))\n",
    "\n",
    "sub_text = re.sub('[!@#$§\\[\\]%&+*\\(\\);<>=|¶éüſавдеикмнопртчыя–—‘’“”]', '', raw_text)\n",
    "sub_text = re.sub('…', '...', sub_text)\n",
    "\n",
    "vocab = sorted(set(sub_text))\n",
    "vocab_size = len(vocab)\n",
    "\n",
    "print('\\nLength of text post-substituion : {} characters. '.format(len(sub_text)))\n",
    "print('{} unique characters after substitution. \\n\\nPre-to-Post difference : {} characters.'.format(len(vocab), len(raw_text) - len(sub_text)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Character Mappings\n",
    "\n",
    "Create mappings from characters to indices and from indices to characters. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:32.593819Z",
     "start_time": "2019-08-13T21:11:32.589713Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  '\\n':   0,\n",
      "  '\\r':   1,\n",
      "  ' ' :   2,\n",
      "  '\"' :   3,\n",
      "  \"'\" :   4,\n"
     ]
    }
   ],
   "source": [
    "### Create a mapping from unique characters to indices and vice versa\n",
    "char2idx = {u:i for i, u in enumerate(vocab)}\n",
    "idx2char = np.array(vocab)\n",
    "\n",
    "for char,_ in zip(char2idx, range(5)):\n",
    "    print('  {:4s}: {:3d},'.format(repr(char), char2idx[char]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T13:05:43.879560Z",
     "start_time": "2019-08-13T13:05:43.873333Z"
    }
   },
   "source": [
    "## Creating train and test data objects\n",
    "\n",
    "Create the training and test text sets.\n",
    "- The training set will be all the text from the original report with any redactions sections removed and replaced with three periods (...).\n",
    "- The test set will be made of all the redactions in the report and some number of preceding characters to prompt the model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:46.359681Z",
     "start_time": "2019-08-13T21:11:46.299455Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■', 'ng from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\\r\\nIn mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■']\n"
     ]
    }
   ],
   "source": [
    "train_text = re.sub('■+', '...',sub_text)\n",
    "\n",
    "### Each redaction has the first and last index values recorded along with the \n",
    "### length of the redactions\n",
    "### Length of redactions isn't actually used anywhere\n",
    "redactions = [[m.start(), m.end(), m.end() - m.start()] for m in re.finditer('■+', sub_text)]\n",
    "\n",
    "prec_char_750 = 750\n",
    "prec_char_400 = 400\n",
    "\n",
    "test_text_750 = [sub_text[i[0] - prec_char_750 : i[1]] for i in redactions]\n",
    "test_text_400 = [sub_text[i[0] - prec_char_400 : i[1]] for i in redactions]\n",
    "\n",
    "print(test_text_400[:2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Convert the text into the numeric array form using the character to index mappings made earlier. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:48.249554Z",
     "start_time": "2019-08-13T21:11:47.646970Z"
    }
   },
   "outputs": [],
   "source": [
    "text_as_int = np.array([char2idx[c] for c in sub_text])\n",
    "train_text_as_int = np.array([char2idx[c] for c in train_text])\n",
    "test_text_750_as_int = [np.array([char2idx[c] for c in i]) for i in test_text_750]\n",
    "test_text_400_as_int = [np.array([char2idx[c] for c in i]) for i in test_text_400]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dataset Generation Functions\n",
    "\n",
    "Create a function to chunk up an input string, splitting it into predictor text and a target character. \n",
    "\n",
    "Batch the full text into seq_length + 1 length sequences then `map` that new function onto the sequences to produce predictors and target chunks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:11:55.575189Z",
     "start_time": "2019-08-13T21:11:55.570128Z"
    }
   },
   "outputs": [],
   "source": [
    "def split_input_target(chunk):\n",
    "    input_text = chunk[:-1]\n",
    "    target_text = chunk[1:]\n",
    "    return input_text, target_text\n",
    "\n",
    "def dataset_generator(seq_length, text):\n",
    "    char_dataset = tf.data.Dataset.from_tensor_slices(train_text_as_int)\n",
    "    \n",
    "    ### Increasing to 100 prevented model from \"running out of data\". \n",
    "    ### Probably part of ResourceExhausted Error but running out of data was worse.\n",
    "    repeat_n = 100\n",
    "    sequences = char_dataset.repeat(repeat_n).batch(seq_length+1, drop_remainder=True)\n",
    "    dataset = sequences.map(split_input_target)\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a dataset with sequence lengths 750 and 400 to test out two different models."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:03.081933Z",
     "start_time": "2019-08-13T21:12:03.039107Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DatasetV1Adapter shapes: ((400,), (400,)), types: (tf.int64, tf.int64)> \n",
      " <DatasetV1Adapter shapes: ((750,), (750,)), types: (tf.int64, tf.int64)>\n"
     ]
    }
   ],
   "source": [
    "seq_length_400 = 400\n",
    "dataset_400 = dataset_generator(seq_length_400, sub_text)\n",
    "\n",
    "seq_length_750 = 750\n",
    "dataset_750 = dataset_generator(seq_length_750, sub_text)\n",
    "print(dataset_400, '\\n', dataset_750)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Models\n",
    "\n",
    "First I'll define some variables and functions that'll be used in both models."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:10.706796Z",
     "start_time": "2019-08-13T21:12:10.703835Z"
    }
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = 10000\n",
    "BATCH_SIZE = 64"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the loss function for the models. We'll use categorical crossentropy as that's the simplest loss function for our problem. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:18.408969Z",
     "start_time": "2019-08-13T21:12:18.405588Z"
    }
   },
   "outputs": [],
   "source": [
    "def loss(labels, logits):\n",
    "    return tf.keras.losses.sparse_categorical_crossentropy(labels, logits, from_logits=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Function to generate text given a starting string using a model. Code originally taken from [this](https://github.com/tensorflow/docs/blob/master/site/en/r2/tutorials/text/text_generation.ipynb) TensorFlow tutorial but adapted to this use case.\n",
    "\n",
    "- Changes include:\n",
    "    - Making `temperature` a parameter\n",
    "    - Changing `num_generate` to dynamically set to the length of redacted piece of `start_string`\n",
    "    - Add a separator in the text generated to show where redaction/prediction starts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:25.966778Z",
     "start_time": "2019-08-13T21:12:25.959410Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_text(model, start_string, temperature=1):\n",
    "    # Evaluation step (generating text using the learned model)\n",
    "\n",
    "    # Number of characters to generate\n",
    "    num_generate = len(start_string) - len(re.sub('■+', '', start_string))\n",
    "    start_string = re.sub('■+', '', start_string)\n",
    "    # Converting our start string to numbers (vectorizing)\n",
    "    input_eval = [char2idx[s] for s in start_string]\n",
    "    input_eval = tf.expand_dims(input_eval, 0)\n",
    "\n",
    "    # Empty string to store our results\n",
    "    text_generated = []\n",
    "\n",
    "    # Low temperatures results in more predictable text.\n",
    "    # Higher temperatures results in more surprising text.\n",
    "    # Experiment to find the best setting.\n",
    "    #temperature = 1.0\n",
    "\n",
    "    # Here batch size == 1\n",
    "    model.reset_states()\n",
    "    for i in range(num_generate):\n",
    "        predictions = model(input_eval)\n",
    "        # remove the batch dimension\n",
    "        predictions = tf.squeeze(predictions, 0)\n",
    "\n",
    "        # using a categorical distribution to predict the word returned by the model\n",
    "        predictions = predictions / temperature\n",
    "        predicted_id = tf.random.categorical(predictions, num_samples=1)[-1,0].numpy()\n",
    "\n",
    "        # We pass the predicted word as the next input to the model\n",
    "        # along with the previous hidden state\n",
    "        input_eval = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "        text_generated.append(idx2char[predicted_id])\n",
    "\n",
    "    return (start_string + ' || ' + ''.join(text_generated))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 750 Sequence Length Model\n",
    "\n",
    "#### Parameters\n",
    "\n",
    "Set parameters for 750 sequence length model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:40.943140Z",
     "start_time": "2019-08-13T21:12:40.936606Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DatasetV1Adapter shapes: ((64, 750), (64, 750)), types: (tf.int64, tf.int64)>\n"
     ]
    }
   ],
   "source": [
    "learning_rate_750 = 5e-4\n",
    "embedding_dim_750 = 256\n",
    "nb_epoches_750 = 1\n",
    "rnn_units_750 = 600\n",
    "load_weights_750 = True\n",
    "keep_training_750 = False\n",
    "\n",
    "### Not sure if is doing what I think but it was used in a tutorial I followed so I'm using it\n",
    "### Essentially, I want each epoch to read the document once\n",
    "examples_per_epoch_750 = len(sub_text)//seq_length_750\n",
    "steps_per_epoch_750 = int(examples_per_epoch_750 / BATCH_SIZE)\n",
    "\n",
    "### Set up dataset that's been shuffled and batched\n",
    "dataset_sb_750 = dataset_750.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
    "print(dataset_sb_750)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Whether the user is loading from drive or now, pick the folder that'll have the checkpoints in it."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set up Checkpoint Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:48.730216Z",
     "start_time": "2019-08-13T21:12:48.725257Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoints_750_model\n"
     ]
    }
   ],
   "source": [
    "if drive_load == True:\n",
    "    checkpoint_750_path = \"/content/gdrive/My Drive/code/redaction model/checkpoints_750_model/epochs:{epoch:03d}-loss:{loss:.3f}.hdf5\"\n",
    "else:\n",
    "    checkpoint_750_path = \"checkpoints_750_model/epochs:{epoch:03d}-loss:{loss:.3f}.hdf5\"\n",
    "    \n",
    "checkpoint_750_dir = os.path.dirname(checkpoint_750_path)\n",
    "\n",
    "### Make the directory if it doesn't exist\n",
    "if not os.path.isdir(checkpoint_750_dir):\n",
    "    os.mkdir(checkpoint_750_dir)\n",
    "    \n",
    "print(checkpoint_750_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build model\n",
    "Build the model for the 750 sequence length model. \n",
    "\n",
    "Use only 1 GRU because that's as big as Google Colab will let it get while in Eager Execution mode. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:12:56.306275Z",
     "start_time": "2019-08-13T21:12:56.301121Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_model_750(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Embedding(vocab_size, embedding_dim,\n",
    "                              batch_input_shape=[batch_size, None]),\n",
    "        tf.keras.layers.GRU(rnn_units,\n",
    "                        return_sequences=True,\n",
    "                        stateful=True,\n",
    "                        recurrent_initializer='glorot_uniform'),\n",
    "        tf.keras.layers.Dropout(0.5), \n",
    "        tf.keras.layers.Dense(2*vocab_size),\n",
    "        tf.keras.layers.Dense(vocab_size)])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make the model object, compile it and potentially load weights to it if necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:13:04.269059Z",
     "start_time": "2019-08-13T21:13:03.868781Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.7/site-packages/tensorflow/python/ops/embedding_ops.py:132: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.7/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "Weights loaded from :  checkpoints_750_model/epochs:001-loss:1.338.hdf5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding (Embedding)        (64, None, 256)           19456     \n",
      "_________________________________________________________________\n",
      "gru (GRU)                    (64, None, 600)           1542600   \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (64, None, 600)           0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (64, None, 152)           91352     \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (64, None, 76)            11628     \n",
      "=================================================================\n",
      "Total params: 1,665,036\n",
      "Trainable params: 1,665,036\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_750 = build_model_750(\n",
    "    vocab_size = len(vocab),\n",
    "    embedding_dim=embedding_dim_750,\n",
    "    rnn_units=rnn_units_750,\n",
    "    batch_size=BATCH_SIZE)\n",
    "\n",
    "adam_opt = Adam(lr=learning_rate_750)\n",
    "\n",
    "model_750.compile(optimizer=adam_opt, loss=loss)\n",
    "\n",
    "most_recent_750_checkpoint = max([checkpoint_750_dir + '/' + i for i in os.listdir(checkpoint_750_dir)], key = os.path.getctime)\n",
    "\n",
    "if load_weights_750 == True:\n",
    "    model_750.load_weights(most_recent_750_checkpoint)\n",
    "    print(\"Weights loaded from : \", most_recent_750_checkpoint)\n",
    "\n",
    "model_750.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fit Model\n",
    "\n",
    "Add callbacks then fit the model.\n",
    "- If `load_weights` == True then the user has weights they would like to load prior to training.\n",
    "- If `keep_training` == True then the user wants the model to train further after starting from a checkpoint."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:17:12.428782Z",
     "start_time": "2019-08-13T21:16:53.842844Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "1/1 [==============================] - 18s 18s/step - loss: 1.3089\n"
     ]
    }
   ],
   "source": [
    "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_750_path, period=1,\n",
    "                                                         monitor='loss', save_best_only=True, \n",
    "                                                         mode='min')\n",
    "\n",
    "earlystop_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n",
    "\n",
    "callbacks_list = [checkpoint_callback, earlystop_callback]\n",
    "\n",
    "if load_weights_750 == True:\n",
    "    if keep_training_750 == True:\n",
    "        history = model_750.fit(\n",
    "                            dataset_sb_750, \n",
    "                            epochs=nb_epoches_750, \n",
    "                            steps_per_epoch=steps_per_epoch_750, \n",
    "                            callbacks=callbacks_list)\n",
    "    elif keep_training_750 == False:\n",
    "        history = model_750.fit(\n",
    "                            dataset_sb_750, \n",
    "                            epochs=1, \n",
    "                            steps_per_epoch=1, \n",
    "                            callbacks=callbacks_list)\n",
    "else:\n",
    "    history = model_750.fit(\n",
    "                        dataset_sb_750, \n",
    "                        epochs=nb_epoches_750, \n",
    "                        steps_per_epoch=steps_per_epoch_750, \n",
    "                        callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1 Batch Model and Imputing Redactions\n",
    "\n",
    "Build a model with 1 batch size and load weights from above cell's checkpoints. \n",
    "\n",
    "Using this 1batch model, generate predictions for the first 5 redactions of the Mueller report.\n",
    "- Generate three predictions with varying temperatures per redaction.\n",
    "    - Lower temperature gives less predictable predictions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:43:36.132554Z",
     "start_time": "2019-08-13T21:41:38.475634Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoints_750_model/epochs:001-loss:1.338.hdf5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_9 (Embedding)      (1, None, 256)            19456     \n",
      "_________________________________________________________________\n",
      "gru_13 (GRU)                 (1, None, 600)            1542600   \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (1, None, 600)            0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (1, None, 152)            91352     \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (1, None, 76)             11628     \n",
      "=================================================================\n",
      "Total params: 1,665,036\n",
      "Trainable params: 1,665,036\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "750 Sequence Length predictors\n",
      "\n",
      "################################################\n",
      "\n",
      "0\n",
      "\n",
      "### : Original Text\n",
      "ion into Russias interference in the 2016 presidential election and related matters, and his actions towards the Special Counsels investigation. Volume II separately states its framework and the considerations that guided that investigation.\n",
      "EXECUTIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "ion into Russias interference in the 2016 presidential election and related matters, and his actions towards the Special Counsels investigation. Volume II separately states its framework and the considerations that guided that investigation.\n",
      "EXECUTIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || indictmention - question./15/17 Text Messages, S.SCNCIDAT V. J.C P.SSOTRUATGOUILOLGT_R-05000086:1972 594.\n",
      "463 11/25/17 Empaign, Offin\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "ion into Russias interference in the 2016 presidential election and related matters, and his actions towards the Special Counsels investigation. Volume II separately states its framework and the considerations that guided that investigation.\n",
      "EXECUTIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || and officials, Trump Jr. was information to the began porticle the Presidents constitutional would be meetings would record the reques\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "ion into Russias interference in the 2016 presidential election and related matters, and his actions towards the Special Counsels investigation. Volume II separately states its framework and the considerations that guided that investigation.\n",
      "EXECUTIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || CNA ank IRR Asswerse, Thise, coutagn, Prosecutions Einary 2 , 157 L.3 910086082?3813.\n",
      "725 31097\n",
      "As criminal uncluded that he receaxb\n",
      "\n",
      "################################################\n",
      "\n",
      "1\n",
      "\n",
      "### : Original Text\n",
      "TIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "TIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || liMit, on Flynn, Adsonding to visite.\n",
      "The VNI Likes IRA Avensing Comey, dosusting the sentone contacts on June,8er-raller in Secations, CNIIdinch, information mernows,\n",
      "contacts, 639?.127930 United Stites 1104112229224 and officials of the Russian government, approximately 10/25/17 302, at 26.\n",
      "846 Krokzan/a\n",
      "5. a hrep\"ents releason, Papadocoulos , 310 U.S. At offenta, information, WikiLeakss.\n",
      "Hicks to noted the pressionations.\n",
      "582 The Cor Dhould State Conduction.\n",
      "300 NCHBS 6435718\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "TIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || with July 27, 2018, The President Trump Jr.\n",
      "Overatize, Appointment of Flynn of Others 11/12/18 302, at 12 Flynn 12/12/17 302, at 11-25 Hor Comey 12/16/16 302, at 12.\n",
      "1186 The President Election Interview of Sances, Cohen Manafort 9/21/18 302, at 13.\n",
      "271 McGahn 12/6/17 302, at 12.\n",
      "279 Septime, S.C.. Foresman, Priebus 15/15/17 302, at 11-4-24.\n",
      "231 Kislyak. 1223 Hicks 11/23/18 302, at 13.\n",
      "333 Atternient Director Committee, 115th Cong. June 9, 2016, Dmitriev to Gardon Ambassador Dora\n",
      "\n",
      "### : Temperature == 1.3 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIVE SUMMARY TO VOLUME I\n",
      "RUSSIAN SOCIAL MEDIA CAMPAIGN\n",
      "The Internet Research Agency IRA carried out the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || lown?\"C01\"8081066\n",
      "1052 F.S.\n",
      "385014 McGamm 8/12/17 302, at 2-43 9:4w up on way Coril'ss, DCIn the Trump Transcorires .\n",
      "1169 Kpolyking Adair:24903 Ucited Summet at S.DCDAIN0.: 1731 Whree've\n",
      "549 Junt 20166, Whice Hector RosS09-SJESe-JAGIDZLC5.CL PJOP-SCAINWJI POPSEAN IRAu_USSaaskinrmE Soferiw?030 YatexAr, he deputors.\" 869 Evenyagozzey?/gr. Trump.\n",
      "Gatobhit Mur. Regaged, Love PUknize-o, Ooteltment 10109\"memonamanal?z 39741029559:wsulre. 1987 1939\n",
      "364 Wokndeachmenn./15/15/14 -001312 M\n",
      "\n",
      "################################################\n",
      "\n",
      "2\n",
      "\n",
      "### : Original Text\n",
      "6, the GRU hacked into the computer networks of the Democratic Congressional Campaign Committee DCCC and the Democratic National Committee DNC. The GRU stole hundreds of thousands of documents from the compromised email accounts and networks. Around the time that the DNC announced in mid-June 2016 the Russian governments role in hacking its network, the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016, ■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "6, the GRU hacked into the computer networks of the Democratic Congressional Campaign Committee DCCC and the Democratic National Committee DNC. The GRU stole hundreds of thousands of documents from the compromised email accounts and networks. Around the time that the DNC announced in mid-June 2016 the Russian governments role in hacking its network, the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || .2.\n",
      "239 Leetbroikzes, \n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "6, the GRU hacked into the computer networks of the Democratic Congressional Campaign Committee DCCC and the Democratic National Committee DNC. The GRU stole hundreds of thousands of documents from the compromised email accounts and networks. Around the time that the DNC announced in mid-June 2016 the Russian governments role in hacking its network, the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || Cohen was washingtor ef\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "6, the GRU hacked into the computer networks of the Democratic Congressional Campaign Committee DCCC and the Democratic National Committee DNC. The GRU stole hundreds of thousands of documents from the compromised email accounts and networks. Around the time that the DNC announced in mid-June 2016 the Russian governments role in hacking its network, the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || Preedi, Rostessuises, F\n",
      "\n",
      "################################################\n",
      "\n",
      "3\n",
      "\n",
      "### : Original Text\n",
      "fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016, ■■■■■■■■■■■■■■■■■■■■■■■ forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || 8003 The kee v. United Stotenfy 35, 2018.\n",
      "454 CSPORAUNESANLFELSCEEV_TKAED\"A'NIR, Ex0000417 1DDARBIGE UNI 1400 1552.18810.Yougher Pupprover, Pupliany Washimoton, 40/21/16 Email, Gallotoves BlangED\n",
      "OW Cr.TFENS 34. C.F.Sider lational Secuncer, Ses\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || The President Clinton Committee, 115th Cong. June 9, 2016, Paped Sumprien stoper 2017 contacts with President-Elect Trump, June 9, 2016, Prince told the White House Counsels Office, at 24.\n",
      "235 Bnother Deproyembars, Kilimnik of March 8, 2016, the\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || Kilimnik. 758/SFOrTaY'LN, Rod HPE_ Purrkeay-730s At pressugess invileats riller a decripibal coumberay, sundquactred, time. Mby Crites United utat Poster, Page.\n",
      "3190\n",
      "Ig told 310\n",
      "549 LLCM Latef, Marnating Gates 6/19/11 302, at 12.\n",
      "645 _9/84/14\n",
      "\n",
      "################################################\n",
      "\n",
      "4\n",
      "\n",
      "### : Original Text\n",
      "and entities involved in the social media campaign have been charged with participating in a conspiracy to defraud the United States by undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C.. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "and entities involved in the social media campaign have been charged with participating in a conspiracy to defraud the United States by undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || Secceining Prince to contact IIs, information statfformed Darry in tourd.\n",
      "U.S. agent-reseltchs with you Trump proypeds acted, Special\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "and entities involved in the social media campaign have been charged with participating in a conspiracy to defraud the United States by undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || 309 1.\n",
      "249 ...\n",
      "262 ...\n",
      "2000 Gaves 6/17/17 302, at 6-5 Hunt 5/1/18 302, at 11.\n",
      "589 Hicks 3/3/18 302, at 6.\n",
      "574 The President Trump\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "and entities involved in the social media campaign have been charged with participating in a conspiracy to defraud the United States by undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || A searcounnoon, 2136.\n",
      "34k.\n",
      "662 Bickaroll, Yet\n",
      "Twie gare Aughation, 1511 75 UnSCTh 2:06669, Poget,\n",
      "19199-LLUREI Exdreasm.47\n",
      "specif\n",
      "\n",
      "################################################\n",
      "\n",
      "5\n",
      "\n",
      "### : Original Text\n",
      " undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C.. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      " undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 042 Manafort 1/28/18 302, at 4-26\n",
      "315 5/24/16 Latter, Roter, Goldstone 2/12/18 Nated, Statement of July 10, 2018, the Russian Essancistion obertionatreding Spokemainbre mikrieve about pentory with and ewitle to gold any project, and Pipad posting ocher about thating \n",
      "\n",
      "### : Temperature == .6 prediction\n",
      " undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 2018 29/15/17 Email, Hanafort and Russian Adgnosid not to Cohen Statement, White House Counsels Office that the President and Volume II, Section IIII, and 2430 In July 12.\n",
      "928 Sep. 75, 2017 Pages 300017, the Presidents content of June 9, 2016, the President Delivatio\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      " undermining through deceptive acts the work of federal agencies charged with regulating foreign influence in U.S. elections, as well as related counts of identity theft. See United States v. Internet Research Agency, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 35.\" 300\n",
      "In Secticn, 21732230998w. Arscyorc mageerov? Appert Mifsud, Ukoocerentanded Broughe Tit for tor TemerH-ANETLC.A\n",
      "50 Trieasess elefirg pyoppeliby.\n",
      "964 Papadopoulos 4/11/17.33222202226w you saff emblyser..-728 Cirtwatto triFskbeven Afsits\".vessepp ryDeadviefi\n",
      "\n",
      "################################################\n",
      "\n",
      "6\n",
      "\n",
      "### : Original Text\n",
      " investigation of Russian election interference. The Office charged some of those lies as violations of the federal false-statements statute. Former National Security Advisor Michael Flynn pleaded guilty to lying about his interactions with Russian Ambassador Kislyak during the transition period. George Papadopoulos, a foreign policy advisor during the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      " investigation of Russian election interference. The Office charged some of those lies as violations of the federal false-statements statute. Former National Security Advisor Michael Flynn pleaded guilty to lying about his interactions with Russian Ambassador Kislyak during the transition period. George Papadopoulos, a foreign policy advisor during the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || 422\n",
      "The Office that asker Holwstwour Discussions , 446 antwerth Presidential Jraffer 281 Statement of SCARUS July. 1492, Priebus , 11398 Smates v. B.CNS..6.\n",
      "527 Hacked 1031:17:12. Maguation Trump Nec\n",
      "\n",
      "### : Temperature == .6 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " investigation of Russian election interference. The Office charged some of those lies as violations of the federal false-statements statute. Former National Security Advisor Michael Flynn pleaded guilty to lying about his interactions with Russian Ambassador Kislyak during the transition period. George Papadopoulos, a foreign policy advisor during the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || The President Accessal of Iffyers, information that the President Trump said Party about the Compution , 158 Manaforts Deralion Times New York Times January 2014, Russian Alcomare ban and Russian Emile\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      " investigation of Russian election interference. The Office charged some of those lies as violations of the federal false-statements statute. Former National Security Advisor Michael Flynn pleaded guilty to lying about his interactions with Russian Ambassador Kislyak during the transition period. George Papadopoulos, a foreign policy advisor during the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || 906\n",
      "Office., Trump-frepresent questions. 8:41 3 Smichmsi37, 13:08 Flynn 17/8/17 302, at 11-whealedulfitmedby of yimuryay obtaintwiti-gandout November, Night, I p-Where Weelle toing turks/whurked, wast\n",
      "\n",
      "################################################\n",
      "\n",
      "7\n",
      "\n",
      "### : Original Text\n",
      "es arising from his receipt of loans\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates, ■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "es arising from his receipt of loans\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || Exgrach SOSCESAKIR\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "es arising from his receipt of loans\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || and June 2016, whe\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "es arising from his receipt of loans\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || 27/26/13 To No. He\n",
      "\n",
      "################################################\n",
      "\n",
      "8\n",
      "\n",
      "### : Original Text\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates, ■■■■■■■■■■■■■■■■■■ , Roger Stone, and ■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || CQ Congless Setarch. Contormer If Seetien\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || Novoxtem Mollier, In offection to Preside\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      "from a bank whose CEO was then seeking a position in the Trump Administration allegations that Papadopoulos committed a crime or crimes by acting as an unregistered agent of the Israeli government and four sets of allegations involving Michael Flynn, the former National Security Advisor to President Trump.\n",
      "On October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || 7:29523 S..kessambs, vaytrovally, husing.\n",
      "\n",
      "################################################\n",
      "\n",
      "9\n",
      "\n",
      "### : Original Text\n",
      " and with other political activists to seek to coordinate political activities, including the staging of political rallies. 5 The investigation did not identify evidence that any U.S. persons knowingly or intentionally coordinated with the IRAs interference operation.\n",
      "By the end of the 2016 U.S. election, the IRA had the ability to reach millions of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3 ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "\n",
      "### : Temperature == 1 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " and with other political activists to seek to coordinate political activities, including the staging of political rallies. 5 The investigation did not identify evidence that any U.S. persons knowingly or intentionally coordinated with the IRAs interference operation.\n",
      "By the end of the 2016 U.S. election, the IRA had the ability to reach millions of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || She bus deection of\n",
      "983 Gorcov 3/13/17 302, at 3.\n",
      "909 Cohen 12/12/18 3\n",
      "\n",
      "### : Temperature == .6 prediction\n",
      " and with other political activists to seek to coordinate political activities, including the staging of political rallies. 5 The investigation did not identify evidence that any U.S. persons knowingly or intentionally coordinated with the IRAs interference operation.\n",
      "By the end of the 2016 U.S. election, the IRA had the ability to reach millions of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || August 18, 2017, the deedings and samothord did not recall and senior ha\n",
      "\n",
      "### : Temperature == 1.3 prediction\n",
      " and with other political activists to seek to coordinate political activities, including the staging of political rallies. 5 The investigation did not identify evidence that any U.S. persons knowingly or intentionally coordinated with the IRAs interference operation.\n",
      "By the end of the 2016 U.S. election, the IRA had the ability to reach millions of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || 4290511 SCr GWaY'23/14/16 TrampTrad/TOR, US HOThe NI averizozzon soon, t\n"
     ]
    }
   ],
   "source": [
    "print(most_recent_750_checkpoint)\n",
    "\n",
    "model_750_1batch = build_model_750(vocab_size, embedding_dim_750, rnn_units_750, batch_size=1)\n",
    "model_750_1batch.load_weights(most_recent_750_checkpoint)\n",
    "model_750_1batch.build(tf.TensorShape([1, None]))\n",
    "model_750_1batch.summary()\n",
    "\n",
    "print(\"750 Sequence Length predictors\")\n",
    "for index, redaction in enumerate(test_text_750[:10]):\n",
    "    print(\"\\n################################################\\n\")\n",
    "    print(index)\n",
    "    print(\"\\n### : Original Text\")\n",
    "    print(redaction)\n",
    "    print(\"\\n### : Temperature == 1 prediction\")\n",
    "    print(generate_text(model_750_1batch, start_string=redaction, temperature=1))\n",
    "    print(\"\\n### : Temperature == .6 prediction\")\n",
    "    print(generate_text(model_750_1batch, start_string=redaction, temperature=.6))\n",
    "    print(\"\\n### : Temperature == 1.3 prediction\")\n",
    "    print(generate_text(model_750_1batch, start_string=redaction, temperature=1.3))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 400 Sequence Length Model\n",
    "\n",
    "\n",
    "#### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:24:07.309378Z",
     "start_time": "2019-08-13T21:24:07.304563Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<DatasetV1Adapter shapes: ((64, 400), (64, 400)), types: (tf.int64, tf.int64)>\n"
     ]
    }
   ],
   "source": [
    "learning_rate_400 = 5e-4\n",
    "embedding_dim_400 = 256\n",
    "nb_epoches_400 = 1\n",
    "rnn_units_400 = 600\n",
    "load_weights_400 = True\n",
    "keep_training_400 = False\n",
    "examples_per_epoch_400 = len(sub_text)//seq_length_400\n",
    "steps_per_epoch_400 = int(examples_per_epoch_400 / BATCH_SIZE)\n",
    "\n",
    "\n",
    "dataset_sb_400 = dataset_400.shuffle(BUFFER_SIZE).batch(BATCH_SIZE, drop_remainder=True)\n",
    "print(dataset_sb_400)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set up Checkpoint Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:24:14.972999Z",
     "start_time": "2019-08-13T21:24:14.968302Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoints_400_model\n"
     ]
    }
   ],
   "source": [
    "if drive_load == True:\n",
    "    checkpoint_400_path = \"/content/gdrive/My Drive/code/redaction model/checkpoints_400_model/epochs:{epoch:03d}-loss:{loss:.3f}.hdf5\"\n",
    "else:\n",
    "    checkpoint_400_path = \"checkpoints_400_model/epochs:{epoch:03d}-loss:{loss:.3f}.hdf5\"\n",
    "    \n",
    "checkpoint_400_dir = os.path.dirname(checkpoint_400_path)\n",
    "\n",
    "if not os.path.isdir(checkpoint_400_dir):\n",
    "    os.mkdir(checkpoint_400_dir)\n",
    "    \n",
    "print(checkpoint_400_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Build Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:24:23.263770Z",
     "start_time": "2019-08-13T21:24:23.256259Z"
    }
   },
   "outputs": [],
   "source": [
    "def build_model_400(vocab_size, embedding_dim, rnn_units, batch_size):\n",
    "    model = tf.keras.Sequential([\n",
    "        tf.keras.layers.Embedding(vocab_size, embedding_dim,\n",
    "                              batch_input_shape=[batch_size, None]),\n",
    "        tf.keras.layers.GRU(rnn_units,\n",
    "                        return_sequences=True,\n",
    "                        stateful=True,\n",
    "                        recurrent_initializer='glorot_uniform'),\n",
    "        tf.keras.layers.Dropout(0.5),         \n",
    "        tf.keras.layers.GRU(rnn_units,\n",
    "                        return_sequences=True,\n",
    "                        stateful=True,\n",
    "                        recurrent_initializer='glorot_uniform'),\n",
    "        tf.keras.layers.Dropout(0.5), \n",
    "        tf.keras.layers.Dense(2*vocab_size),\n",
    "        tf.keras.layers.Dense(vocab_size)])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:24:31.574489Z",
     "start_time": "2019-08-13T21:24:31.076560Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Weights loaded from :  checkpoints_400_model/epochs_029-loss_0.953.hdf5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_5 (Embedding)      (64, None, 256)           19456     \n",
      "_________________________________________________________________\n",
      "gru_5 (GRU)                  (64, None, 600)           1542600   \n",
      "_________________________________________________________________\n",
      "dropout_5 (Dropout)          (64, None, 600)           0         \n",
      "_________________________________________________________________\n",
      "gru_6 (GRU)                  (64, None, 600)           2161800   \n",
      "_________________________________________________________________\n",
      "dropout_6 (Dropout)          (64, None, 600)           0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (64, None, 152)           91352     \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (64, None, 76)            11628     \n",
      "=================================================================\n",
      "Total params: 3,826,836\n",
      "Trainable params: 3,826,836\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model_400 = build_model_400(\n",
    "    vocab_size = len(vocab),\n",
    "    embedding_dim=embedding_dim_400,\n",
    "    rnn_units=rnn_units_400,\n",
    "    batch_size=BATCH_SIZE)\n",
    "\n",
    "adam_opt = Adam(lr=learning_rate_400)\n",
    "\n",
    "model_400.compile(optimizer=adam_opt, loss=loss)\n",
    "\n",
    "most_recent_400_checkpoint = max([checkpoint_400_dir + '/' + i for i in os.listdir(checkpoint_400_dir)], key = os.path.getctime)\n",
    "\n",
    "if load_weights_400 == True:\n",
    "    model_400.load_weights(most_recent_400_checkpoint)\n",
    "    print(\"Weights loaded from : \", most_recent_400_checkpoint)\n",
    "\n",
    "model_400.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Fit Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:25:03.750815Z",
     "start_time": "2019-08-13T21:24:39.518056Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "1/1 [==============================] - 24s 24s/step - loss: 0.9504\n"
     ]
    }
   ],
   "source": [
    "checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_400_path, period=1,\n",
    "                                                         monitor='loss', save_best_only=True, \n",
    "                                                         mode='min')\n",
    "\n",
    "earlystop_callback = tf.keras.callbacks.EarlyStopping(monitor='loss', patience=3)\n",
    "\n",
    "callbacks_list = [checkpoint_callback, earlystop_callback]\n",
    "\n",
    "if load_weights_400 == True:\n",
    "    if keep_training_400 == True:\n",
    "        history = model_400.fit(\n",
    "                            dataset_sb_400, \n",
    "                            epochs=nb_epoches_400, \n",
    "                            steps_per_epoch=steps_per_epoch_400, \n",
    "                            callbacks=callbacks_list)\n",
    "    elif keep_training_400 == False:\n",
    "        history = model_400.fit(\n",
    "                            dataset_sb_400, \n",
    "                            epochs=1, \n",
    "                            steps_per_epoch=1, \n",
    "                            callbacks=callbacks_list)\n",
    "else:\n",
    "    history = model_400.fit(\n",
    "                        dataset_sb_400, \n",
    "                        epochs=nb_epoches_400, \n",
    "                        steps_per_epoch=steps_per_epoch_400, \n",
    "                        callbacks=callbacks_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 1 Batch and Redactions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-13T21:46:09.824051Z",
     "start_time": "2019-08-13T21:43:43.062561Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "checkpoints_400_model/epochs_029-loss_0.953.hdf5\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_10 (Embedding)     (1, None, 256)            19456     \n",
      "_________________________________________________________________\n",
      "gru_14 (GRU)                 (1, None, 600)            1542600   \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (1, None, 600)            0         \n",
      "_________________________________________________________________\n",
      "gru_15 (GRU)                 (1, None, 600)            2161800   \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (1, None, 600)            0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (1, None, 152)            91352     \n",
      "_________________________________________________________________\n",
      "dense_21 (Dense)             (1, None, 76)             11628     \n",
      "=================================================================\n",
      "Total params: 3,826,836\n",
      "Trainable params: 3,826,836\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "400 Sequence Length predictors\n",
      "\n",
      "################################################\n",
      "\n",
      "0\n",
      "###\n",
      "the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || recognized that it would be or in formal indicating that would make the issues and warranced on Russia or what, after the plang agains\n",
      "### : Temperature == 0.6 prediction\n",
      "the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || and he controlled to the Trump Campaign. Donaldson 11/6/17 302, at 3.\n",
      "1015 Deputy Attorney General Jeff Sessions Beniodment of Flynn.\n",
      "### : Temperature == 1.3 prediction\n",
      "the earliest Russian interference operations identified by the investigation  a social media campaign designed to provoke and amplify political and social discord in the United States. The IRA was based in St. Petersburg, Russia, and received funding from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin,  || had said that Prince would be keed Kushner 1:12 p.m. befw Lot Office propuller explaining derivately quotated of with carey, whrug all\n",
      "\n",
      "################################################\n",
      "\n",
      "1\n",
      "###\n",
      "ng from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "ng from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || with constitutional statutes computersy for U.S.-pullic. In an interview with Russia in Masch, internalized and the executive of Russia in his Papadopoulos about the Russia investigation, saye more while the investigatiys campaign stoffile donuths thas on approximately 200. Christie also recalled y oc aters that he had asked at the June 13, 2017 quoting Brions, coordination on a United States v. Ranyabid , 7:18 a.m..\n",
      "542 FBI National Slajri, Juryd,oe 11/16/17 302, at 14 McGahn 3/8/18 \n",
      "### : Temperature == 0.6 prediction\n",
      "ng from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || and that the President was senting the investigation. 520 Page told the President had decided to negative press conference, and how story to the Presidents personal counsel, and that he did not investigate all of the conversation between the Trump Campaign and the Presidents actions and the conduct that the President was not included the project. Although the President had not defended the President that he had not established that the President recalled that he had decided to fire the\n",
      "### : Temperature == 1.3 prediction\n",
      "ng from Russian oligarch Yevgeniy Prigozhin and companies he controlled. Prigozhin is widely reported to have ties to Russian President Vladimir Putin, \n",
      "In mid-2014, the IRA sent employees to the United States on an intelligence-gathering mission with instructions  || concire of these matters. Records of Josh Prifes in sturnate or Congress. Yuort, but the me is  calling?\n",
      "Id evidence because he explained Gates a\n",
      "263 Ses in 2019, the President dinueset or the accaurtanitchs, Gacefor Clecs adved applying everitedims. Constitution leader, Those ecutifle\n",
      "8. Hocume the October 8. Tile 819 Facebook gained in Congress, including lating? Sessions 8/17 302 and Jarke0.X8\"t1 Attocneyem 3/13/18 302, at 10-14.\n",
      "911 ... Sem\n",
      "590 ...\n",
      "858 Uur Borsited July 8, 201\n",
      "\n",
      "################################################\n",
      "\n",
      "2\n",
      "###\n",
      ", the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016, ■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      ", the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || the President was inves\n",
      "### : Temperature == 0.6 prediction\n",
      ", the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || and White House Counsel\n",
      "### : Temperature == 1.3 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ", the GRU began disseminating stolen materials through the fictitious online personas DCLeaks and Guccifer 2.0. The GRU later released additional materials through the organization WikiLeaks.\n",
      "The presidential campaign of Donald J. Trump Trump Campaign or Campaign\" showed interest in WikiLeakss releases of documents and welcomed their potential to damage candidate Clinton. Beginning in June 2016,  || Campaigns foreign palsp\n",
      "\n",
      "################################################\n",
      "\n",
      "3\n",
      "###\n",
      "■■■■■■■■■■■■■■ forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      " forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || The change signed caresing emails exclusive of the investigation. 1129\n",
      "B. Wollfteon I talk, the legal behe forwer candidate Clinton. On July 22, 2017, Flynn told this the weeks on other rece that talking briefed the abreement. JOAs.\n",
      "4 \n",
      "### : Temperature == 0.6 prediction\n",
      " forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || Advisor to the President that the President considered meetings with the President, he did not communicate with the Russian government. 873\n",
      "Beginning in the trip to Potential Conduct Priebus to talk obtained about whether he was looked \n",
      "### : Temperature == 1.3 prediction\n",
      " forecast to senior Campaign officials that WikiLeaks would release information damaging to candidate Clinton. WikiLeakss first release came in July 2016. Around the same time, candidate Trump announced that he hoped Russia would recover emails described as missing from a private server used by Clinton when she was Secretary of State he later said that he was speaking sarcastically.  || Geared Hielchs of Onind.\" 11200 Consistent with conduct. 592 Beskain or asked fhotes a messages and were dedijenc\\text, then because Whytik , 095 F.2d at 15-1d, the note by the ouirsing insuir conversation, iblusof. 748 Readandeofoccaled\n",
      "\n",
      "################################################\n",
      "\n",
      "4\n",
      "###\n",
      "y, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C.. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "y, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || ....\n",
      "108 ...\n",
      "808 ...\n",
      "925 ...\n",
      "172 ...\n",
      "Advertisement\n",
      "Complainance for all, yature omeiled then took Russia was crassibly about the\n",
      "### : Temperature == 0.6 prediction\n",
      "y, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || Orae Internet Research Agency Indictment , U.S. CONST. ART. IV, see Shate Sean Sessions Sent Fore grand jury in the Office of the Coun\n",
      "### : Temperature == 1.3 prediction\n",
      "y, et al. , No. 18-cr-32 D.D.C.. Separately, Russian intelligence officers who carried out the hacking into Democratic Party computers and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..  || ... At ONSUC. 342. SNC Obama Administration 3/27/18 302, at 3.\n",
      "839 Kislykk Aatures to Russia SITbaf: 9280 Comey defined his beier and\n",
      "\n",
      "################################################\n",
      "\n",
      "5\n",
      "###\n",
      "and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C.. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■ ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 1211, and, WikiLeaks on Extendat of State for Ranif Unit 26355 an explanations under investigation and told the replating Crimea as hat story that he mode and cohen case would be the money airuved one of his contacts between Trump Tower. Morch of this information, inc\n",
      "### : Temperature == 0.6 prediction\n",
      "and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 501. The President personally or the President asked McGahn to refus in let this information documents the evidence of the Trump Tower Moscow project and the President directed him to discuss sanctions with the Manafort stated that it could component that the Presiden\n",
      "### : Temperature == 1.3 prediction\n",
      "and the personal email accounts of individuals affiliated with the Clinton Campaign conspired to violate, among other federal laws, the federal computer-intrusion statute, and they have been so charged. See United States v. Netyksho, et al. , No. 18-cr-215 D.D.C..   || 613iv. Thar, it. 254 Kh sent Killman, Iven his relevant general, a statute, reSpecial Counsel asked assuring those concerns about the Trump Organization\n",
      "storderswand\n",
      "817 Autenn Developers from FolTTimALPE  Hy DirshiLIG nQ Multiqu Sky Lexassion,, see XI, sought Lewan\n",
      "\n",
      "################################################\n",
      "\n",
      "6\n",
      "###\n",
      " the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project. ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || 517 The next day, the President was interedeing him director.\n",
      "95 Robert Goldstone :DEselaated enemyn. By Flynn pleaded guilty, puilieg it to candidate Trump than lubbind the Trump Campaign. 526 Dana B\n",
      "### : Temperature == 0.6 prediction\n",
      " the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || 1253 The President called McGahn very trip and directed at a press conference. McGahn did not think Russia as a threat to contact the President to be controlled by Youse and the President to subsequent\n",
      "### : Temperature == 1.3 prediction\n",
      " the campaign period, pleaded guilty to lying to investigators about, inter alia , the nature and timing of his interactions with Joseph Mifsud, the professor who told Papadopoulos that the Russians had dirt on candidate Clinton in the form of thousands of emails. Former Trump Organization attorney Michael Cohen pleaded guilty to making false statements to Congress about the Trump Moscow project.  || Again Completed them. 689 ... tapatly and consults. Of Gusci\n",
      "951 Lewandowski 4/3/19 302, at 8 Graphan Aktaskouwoffur our interest of anytun.\n",
      "Iusion that Page was comments w under Constitution, LCIe, \n",
      "\n",
      "################################################\n",
      "\n",
      "7\n",
      "###\n",
      "October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates, ■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || on July 12, 2018, \n",
      "### : Temperature == 0.6 prediction\n",
      "October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || Donald Trump Jr. a\n",
      "### : Temperature == 1.3 prediction\n",
      "October 20, 2017, the Acting Attorney General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  || CSo had been infra\n",
      "\n",
      "################################################\n",
      "\n",
      "8\n",
      "###\n",
      "General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates, ■■■■■■■■■■■■■■■■■■ , Roger Stone, and ■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || McGahn continued the project. 880 McGahn,\n",
      "### : Temperature == 0.6 prediction\n",
      "General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || had planned to make a proposal who inform\n",
      "### : Temperature == 1.3 prediction\n",
      "General confirmed in a memorandum the Special Counsels investigative authority as to several individuals and entities. First, as part of a full and thorough investigation of the Russian governments efforts to interfere in the 2016 presidential election, the Special Counsel was authorized to investigate the pertinent activities of Michael Cohen, Richard Gates,  , Roger Stone, and  || . 470 Phone reelated no one oplied he gaB\n",
      "\n",
      "################################################\n",
      "\n",
      "9\n",
      "###\n",
      "of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3 ■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■■\n",
      "### : Temperature == 1 prediction\n",
      "of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || Nader 8/23/18 302, at 1.\n",
      "410 G-KMA Cohen 18/14/17 302, at 4 stating tha\n",
      "### : Temperature == 0.6 prediction\n",
      "of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || even though on the activities in the days after the election, including \n",
      "### : Temperature == 1.3 prediction\n",
      "of U.S. persons through their social media accounts. Multiple IRA-controlled Facebook groups and\n",
      "2 The Office is aware of reports that other Russian entities engaged in similar active measures operations targeting the United States. Some evidence collected by the Office corroborates those reports, and the Office has shared that evidence with other offices in the Department of Justice and FBI.\n",
      "3  || 2/rig was no persona becouse. We day he was ongoing\n",
      "962 Parties Papadop\n"
     ]
    }
   ],
   "source": [
    "print(most_recent_400_checkpoint)\n",
    "\n",
    "model_400_1batch = build_model_400(vocab_size, embedding_dim_400, rnn_units_400, batch_size=1)\n",
    "model_400_1batch.load_weights(most_recent_400_checkpoint)\n",
    "model_400_1batch.build(tf.TensorShape([1, None]))\n",
    "model_400_1batch.summary()\n",
    "\n",
    "print(\"400 Sequence Length predictors\")\n",
    "for index, redaction in enumerate(test_text_400[:10]):\n",
    "    print(\"\\n################################################\\n\")\n",
    "    print(index)\n",
    "    print(\"###\")\n",
    "    print(redaction)\n",
    "    print(\"### : Temperature == 1 prediction\")\n",
    "    print(generate_text(model_400_1batch, start_string=redaction, temperature=1))\n",
    "    print(\"### : Temperature == 0.6 prediction\")\n",
    "    print(generate_text(model_400_1batch, start_string=redaction, temperature=0.6))\n",
    "    print(\"### : Temperature == 1.3 prediction\")\n",
    "    print(generate_text(model_400_1batch, start_string=redaction, temperature=1.3))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}