20210925_wmt19_cs_en_preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Version:  2.5.0\n",
      "Eager mode:  True\n",
      "Hub version:  0.12.0\n",
      "GPU is available\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import nltk\n",
    "from nltk.tokenize import word_tokenize\n",
    "from gensim.models import Word2Vec\n",
    "\n",
    "import tensorflow as tf\n",
    "import tensorflow_hub as hub\n",
    "import tensorflow_datasets as tfds\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES']='1'\n",
    "tf.config.set_soft_device_placement(True)\n",
    "\n",
    "print(\"Version: \", tf.__version__)\n",
    "print(\"Eager mode: \", tf.executing_eagerly())\n",
    "print(\"Hub version: \", hub.__version__)\n",
    "print(\"GPU is\", \"available\" if tf.config.list_physical_devices(\"GPU\") else \"NOT AVAILABLE\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_name = '20210925_wmt19_cs_en'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data\n",
    "Source: [data.statmt.org](http://data.statmt.org/news-commentary/v14/)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ai-lab/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:6: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support skipfooter; you can avoid this warning by specifying engine='python'.\n",
      "  \n",
      "Skipping line 11501: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 31651: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 36410: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 38172: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 48214: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 52929: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 59456: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 65598: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 66298: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 71766: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 73797: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 90155: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 90195: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 90211: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 97398: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 99538: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 101816: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 102348: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 106454: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 106568: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 106569: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 113733: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 117158: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 120666: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 125831: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 129109: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 130852: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 131592: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 131594: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 131598: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 131624: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 134115: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 135097: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 135458: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 136201: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 136939: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 137176: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 139532: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 139623: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 139755: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Skipping line 149100: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 149235: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 149237: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 149251: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 152487: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 152489: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 154462: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 157509: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 161201: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 161219: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 183979: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 190876: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 192633: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 196568: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 197664: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 197667: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 197669: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 199064: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 199074: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 202575: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 203341: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 203369: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 204020: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 206363: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 208504: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 208513: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 208964: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 210552: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 210553: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 211324: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 211687: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 213464: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 214466: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 215568: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 215569: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 221492: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 229543: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 239171: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 246218: '\t' expected after '\"'. Error could possibly be due to parsing errors in the skipped footer rows (the skipfooter keyword is only applied after Python's csv library has parsed all rows).\n",
      "Skipping line 17210: Expected 2 fields in line 17210, saw 3\n",
      "Skipping line 17214: Expected 2 fields in line 17214, saw 3\n",
      "Skipping line 17231: Expected 2 fields in line 17231, saw 3\n",
      "Skipping line 33653: Expected 2 fields in line 33653, saw 3\n",
      "Skipping line 33658: Expected 2 fields in line 33658, saw 3\n",
      "Skipping line 33659: Expected 2 fields in line 33659, saw 3\n",
      "Skipping line 33675: Expected 2 fields in line 33675, saw 3\n",
      "Skipping line 33680: Expected 2 fields in line 33680, saw 3\n",
      "Skipping line 33683: Expected 2 fields in line 33683, saw 3\n",
      "Skipping line 38538: Expected 2 fields in line 38538, saw 3\n",
      "Skipping line 38541: Expected 2 fields in line 38541, saw 3\n",
      "Skipping line 38542: Expected 2 fields in line 38542, saw 3\n",
      "Skipping line 38547: Expected 2 fields in line 38547, saw 3\n",
      "Skipping line 38553: Expected 2 fields in line 38553, saw 3\n",
      "Skipping line 38557: Expected 2 fields in line 38557, saw 3\n",
      "Skipping line 38559: Expected 2 fields in line 38559, saw 3\n",
      "Skipping line 38562: Expected 2 fields in line 38562, saw 3\n",
      "Skipping line 38572: Expected 2 fields in line 38572, saw 3\n",
      "Skipping line 39930: Expected 2 fields in line 39930, saw 3\n",
      "Skipping line 39933: Expected 2 fields in line 39933, saw 3\n",
      "Skipping line 39936: Expected 2 fields in line 39936, saw 3\n",
      "Skipping line 39946: Expected 2 fields in line 39946, saw 3\n",
      "Skipping line 39948: Expected 2 fields in line 39948, saw 3\n",
      "Skipping line 41118: Expected 2 fields in line 41118, saw 3\n",
      "Skipping line 41119: Expected 2 fields in line 41119, saw 3\n",
      "Skipping line 41123: Expected 2 fields in line 41123, saw 3\n",
      "Skipping line 41129: Expected 2 fields in line 41129, saw 3\n",
      "Skipping line 41133: Expected 2 fields in line 41133, saw 3\n",
      "Skipping line 41135: Expected 2 fields in line 41135, saw 3\n",
      "Skipping line 52927: Expected 2 fields in line 52927, saw 3\n",
      "Skipping line 59198: Expected 2 fields in line 59198, saw 3\n",
      "Skipping line 59206: Expected 2 fields in line 59206, saw 3\n",
      "Skipping line 59209: Expected 2 fields in line 59209, saw 3\n",
      "Skipping line 59213: Expected 2 fields in line 59213, saw 3\n",
      "Skipping line 59221: Expected 2 fields in line 59221, saw 3\n",
      "Skipping line 59227: Expected 2 fields in line 59227, saw 3\n",
      "Skipping line 59409: Expected 2 fields in line 59409, saw 3\n",
      "Skipping line 59436: Expected 2 fields in line 59436, saw 3\n",
      "Skipping line 61286: Expected 2 fields in line 61286, saw 3\n",
      "Skipping line 61850: Expected 2 fields in line 61850, saw 3\n",
      "Skipping line 61852: Expected 2 fields in line 61852, saw 3\n",
      "Skipping line 61856: Expected 2 fields in line 61856, saw 3\n",
      "Skipping line 61857: Expected 2 fields in line 61857, saw 3\n",
      "Skipping line 77704: Expected 2 fields in line 77704, saw 3\n",
      "Skipping line 77706: Expected 2 fields in line 77706, saw 3\n",
      "Skipping line 77725: Expected 2 fields in line 77725, saw 3\n",
      "Skipping line 78185: Expected 2 fields in line 78185, saw 3\n",
      "Skipping line 78191: Expected 2 fields in line 78191, saw 3\n",
      "Skipping line 78201: Expected 2 fields in line 78201, saw 3\n",
      "Skipping line 78256: Expected 2 fields in line 78256, saw 3\n",
      "Skipping line 78258: Expected 2 fields in line 78258, saw 3\n",
      "Skipping line 78283: Expected 2 fields in line 78283, saw 3\n",
      "Skipping line 78285: Expected 2 fields in line 78285, saw 3\n",
      "Skipping line 85344: Expected 2 fields in line 85344, saw 3\n",
      "Skipping line 85350: Expected 2 fields in line 85350, saw 3\n",
      "Skipping line 85359: Expected 2 fields in line 85359, saw 3\n",
      "Skipping line 92600: Expected 2 fields in line 92600, saw 3\n",
      "Skipping line 104109: Expected 2 fields in line 104109, saw 3\n",
      "Skipping line 104127: Expected 2 fields in line 104127, saw 3\n",
      "Skipping line 104129: Expected 2 fields in line 104129, saw 3\n",
      "Skipping line 104408: Expected 2 fields in line 104408, saw 3\n",
      "Skipping line 104415: Expected 2 fields in line 104415, saw 3\n",
      "Skipping line 104423: Expected 2 fields in line 104423, saw 3\n",
      "Skipping line 104430: Expected 2 fields in line 104430, saw 3\n",
      "Skipping line 104437: Expected 2 fields in line 104437, saw 3\n",
      "Skipping line 104442: Expected 2 fields in line 104442, saw 3\n",
      "Skipping line 110522: Expected 2 fields in line 110522, saw 3\n",
      "Skipping line 110527: Expected 2 fields in line 110527, saw 3\n",
      "Skipping line 110533: Expected 2 fields in line 110533, saw 3\n",
      "Skipping line 122071: Expected 2 fields in line 122071, saw 3\n",
      "Skipping line 122072: Expected 2 fields in line 122072, saw 3\n",
      "Skipping line 129065: Expected 2 fields in line 129065, saw 3\n",
      "Skipping line 129069: Expected 2 fields in line 129069, saw 3\n",
      "Skipping line 129092: Expected 2 fields in line 129092, saw 3\n",
      "Skipping line 132238: Expected 2 fields in line 132238, saw 3\n",
      "Skipping line 132610: Expected 2 fields in line 132610, saw 3\n",
      "Skipping line 132629: Expected 2 fields in line 132629, saw 3\n",
      "Skipping line 132635: Expected 2 fields in line 132635, saw 3\n",
      "Skipping line 145379: Expected 2 fields in line 145379, saw 3\n",
      "Skipping line 145386: Expected 2 fields in line 145386, saw 3\n",
      "Skipping line 145392: Expected 2 fields in line 145392, saw 3\n",
      "Skipping line 145393: Expected 2 fields in line 145393, saw 3\n",
      "Skipping line 145401: Expected 2 fields in line 145401, saw 3\n",
      "Skipping line 145403: Expected 2 fields in line 145403, saw 3\n",
      "Skipping line 145414: Expected 2 fields in line 145414, saw 3\n",
      "Skipping line 145816: Expected 2 fields in line 145816, saw 3\n",
      "Skipping line 145821: Expected 2 fields in line 145821, saw 3\n",
      "Skipping line 145837: Expected 2 fields in line 145837, saw 3\n",
      "Skipping line 145840: Expected 2 fields in line 145840, saw 3\n",
      "Skipping line 145843: Expected 2 fields in line 145843, saw 3\n",
      "Skipping line 148862: Expected 2 fields in line 148862, saw 3\n",
      "Skipping line 148868: Expected 2 fields in line 148868, saw 3\n",
      "Skipping line 148876: Expected 2 fields in line 148876, saw 3\n",
      "Skipping line 148885: Expected 2 fields in line 148885, saw 3\n",
      "Skipping line 161982: Expected 2 fields in line 161982, saw 3\n",
      "Skipping line 161990: Expected 2 fields in line 161990, saw 3\n",
      "Skipping line 161992: Expected 2 fields in line 161992, saw 3\n",
      "Skipping line 161994: Expected 2 fields in line 161994, saw 3\n",
      "Skipping line 161998: Expected 2 fields in line 161998, saw 3\n",
      "Skipping line 161999: Expected 2 fields in line 161999, saw 3\n",
      "Skipping line 173020: Expected 2 fields in line 173020, saw 3\n",
      "Skipping line 173021: Expected 2 fields in line 173021, saw 3\n",
      "Skipping line 173025: Expected 2 fields in line 173025, saw 3\n",
      "Skipping line 174373: Expected 2 fields in line 174373, saw 3\n",
      "Skipping line 174389: Expected 2 fields in line 174389, saw 3\n"
     ]
    }
   ],
   "source": [
    "corpus = pd.read_csv(\n",
    "    'news-commentary-v14.cs-en.tsv', \n",
    "    sep='\\t', \n",
    "    error_bad_lines=False,\n",
    "    skipfooter= 1,\n",
    "    header = None,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs = corpus[0].to_numpy(dtype = str)\n",
    "en = corpus[1].to_numpy(dtype = str)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## English"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert the text into sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_tkr = tf.keras.preprocessing.text.Tokenizer(\n",
    "    filters='!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n',\n",
    ")\n",
    "en_tkr.fit_on_texts(['<bos>', '<eos>'])\n",
    "en_tkr.fit_on_texts(en)\n",
    "en_seq = en_tkr.texts_to_sequences(en)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "41251\n",
      "41252\n"
     ]
    }
   ],
   "source": [
    "en_word2idx = eval(en_tkr.get_config()['word_index'])\n",
    "bosIdx = en_word2idx['<bos>']\n",
    "eosIdx = en_word2idx['<eos>']\n",
    "print(bosIdx)\n",
    "print(eosIdx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Count the average sequence length."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21.316457186469854\n",
      "11.501927989194174\n"
     ]
    }
   ],
   "source": [
    "en_seq_len = [len(s) for s in en_seq]\n",
    "print(np.mean(en_seq_len))\n",
    "print(np.std(en_seq_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_seq_length = 32"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Czech"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_tkr = tf.keras.preprocessing.text.Tokenizer()\n",
    "cs_tkr.fit_on_texts(cs)\n",
    "cs_seq = cs_tkr.texts_to_sequences(cs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18.807237621993796\n",
      "10.329988244009906\n"
     ]
    }
   ],
   "source": [
    "cs_seq_len = [len(s) for s in cs_seq]\n",
    "print(np.mean(cs_seq_len))\n",
    "print(np.std(cs_seq_len))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_seq_length = 28"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# remove long sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193131\n"
     ]
    }
   ],
   "source": [
    "keep = [[cs, en] for cs, en in zip(cs_seq, en_seq) if len(cs)<=cs_seq_length and len(en)<=en_seq_length-2]\n",
    "print(len(keep))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add BOS and EOS into decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_reduce_seq = [pair[0] for pair in keep]\n",
    "en_reduce_seq = [pair[1] for pair in keep]\n",
    "for i, seq in enumerate(en_reduce_seq):\n",
    "    en_reduce_seq[i] = [bosIdx]\n",
    "    en_reduce_seq[i].extend(seq)\n",
    "    en_reduce_seq[i].append(eosIdx)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build the dictionary and re-tokenize sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# English\n",
    "en_reduce = en_tkr.sequences_to_texts(en_reduce_seq)\n",
    "en_tkr = tf.keras.preprocessing.text.Tokenizer(\n",
    "    filters='!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n',\n",
    ")\n",
    "en_tkr.fit_on_texts(en_reduce)\n",
    "en_reduce_seq = en_tkr.texts_to_sequences(en_reduce)\n",
    "for i, seq in enumerate(en_reduce_seq):\n",
    "    en_reduce_seq[i].append(0)\n",
    "en_idx2word = eval(en_tkr.get_config()['index_word'])\n",
    "en_idx2word['0'] = '<pad>'\n",
    "en_word2idx = eval(en_tkr.get_config()['word_index'])\n",
    "en_word2idx['<pad>'] = '0'\n",
    "# Czech\n",
    "cs_reduce = cs_tkr.sequences_to_texts(cs_reduce_seq)\n",
    "cs_tkr = tf.keras.preprocessing.text.Tokenizer(\n",
    "    filters='!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n',\n",
    ")\n",
    "cs_tkr.fit_on_texts(cs_reduce)\n",
    "cs_reduce_seq = cs_tkr.texts_to_sequences(cs_reduce)\n",
    "cs_idx2word = eval(cs_tkr.get_config()['index_word'])\n",
    "cs_idx2word['0'] = '<pad>'\n",
    "cs_word2idx = eval(cs_tkr.get_config()['word_index'])\n",
    "cs_word2idx['<pad>'] = '0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<bos>', 'meanwhile', 'in', 'about', '2005', 'i', 'was', 'in', 'south', 'africa', 'with', 'a', 'small', 'group', 'advising', 'former', 'president', 'thabo', 'mbeki', 'and', 'his', 'government', 'about', 'its', 'it', 'policy', '<eos>', '<pad>']\n"
     ]
    }
   ],
   "source": [
    "print([ en_idx2word[str(i)] for i in en_reduce_seq[121]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num. of en. word: 52575\n",
      "num. of cs. word: 155078\n"
     ]
    }
   ],
   "source": [
    "en_num_words = len(en_word2idx)\n",
    "cs_num_words = len(cs_word2idx)\n",
    "print(f'num. of en. word: {en_num_words}')\n",
    "print(f'num. of cs. word: {cs_num_words}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Padding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_pad_seq = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "    en_reduce_seq,\n",
    "    dtype='int32',\n",
    "    padding='post',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(193131, 33)\n"
     ]
    }
   ],
   "source": [
    "print(en_pad_seq.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_pad_seq = tf.keras.preprocessing.sequence.pad_sequences(\n",
    "    cs_reduce_seq,\n",
    "    dtype='int32',\n",
    "    padding='post',\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(193131, 28)\n"
     ]
    }
   ],
   "source": [
    "print(cs_pad_seq.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Distribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_pair = len(en_pad_seq)\n",
    "encoder_train = cs_pad_seq[int(num_pair*0.1):]\n",
    "decoder_train = en_pad_seq[int(num_pair*0.1):, :-1]\n",
    "teacher_train = en_pad_seq[int(num_pair*0.1):, 1:]\n",
    "encoder_vali  = cs_pad_seq[:int(num_pair*0.1)]\n",
    "decoder_vali  = en_pad_seq[:int(num_pair*0.1), :-1]\n",
    "teacher_vali  = en_pad_seq[:int(num_pair*0.1), 1:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(173818, 28)\n",
      "(173818, 32)\n",
      "(173818, 32)\n"
     ]
    }
   ],
   "source": [
    "print(encoder_train.shape)\n",
    "print(decoder_train.shape)\n",
    "print(teacher_train.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Word2Vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 32 dim. English embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29727808, 63733230)"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = [list(e) for e in en_pad_seq]\n",
    "model = Word2Vec(\n",
    "    sentences=tmp, \n",
    "    vector_size=32, \n",
    "    window=5, \n",
    "    min_count=1, \n",
    "    workers=16,\n",
    "    sg = 1,\n",
    "    negative = 10,\n",
    ")\n",
    "model.build_vocab(tmp)\n",
    "model.train(tmp, total_examples=model.corpus_count, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.5323224 , -0.33435014, -0.00609   ,  0.7025845 ,  0.2990808 ,\n",
       "       -0.7005396 ,  0.56037045,  0.04551583,  0.2818839 ,  0.31856662,\n",
       "        0.48854238,  0.2362166 , -0.41161078, -0.15584254,  0.39335993,\n",
       "       -0.16227524, -0.34686658, -0.18837813, -0.742977  ,  0.18588375,\n",
       "       -0.28991964,  0.7114349 ,  0.38617498,  0.41311225, -0.10103195,\n",
       "       -0.8036458 , -1.4665223 ,  0.34539932, -0.52019036, -0.26782683,\n",
       "        0.18907385,  0.02807016], dtype=float32)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.wv[5257]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "en_emb32 = np.array([ model.wv[i] for i in range(en_num_words)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 32 dim. Czech embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(28097970, 54076680)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp = [list(e) for e in cs_pad_seq]\n",
    "model = Word2Vec(\n",
    "    sentences=tmp, \n",
    "    vector_size=32, \n",
    "    window=5, \n",
    "    min_count=1, \n",
    "    workers=16,\n",
    "    sg = 1,\n",
    "    negative = 10,\n",
    ")\n",
    "model.build_vocab(tmp)\n",
    "model.train(tmp, total_examples=model.corpus_count, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs_emb32 = np.array([ model.wv[i] for i in range(cs_num_words)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save preprocessed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "pickle.dump(encoder_train, open(f'{folder_name}/encoder_train.pkl','wb'))\n",
    "pickle.dump(decoder_train, open(f'{folder_name}/decoder_train.pkl','wb'))\n",
    "pickle.dump(teacher_train, open(f'{folder_name}/teacher_train.pkl','wb'))\n",
    "pickle.dump(encoder_vali,  open(f'{folder_name}/encoder_vali.pkl','wb'))\n",
    "pickle.dump(decoder_vali,  open(f'{folder_name}/decoder_vali.pkl','wb'))\n",
    "pickle.dump(teacher_vali,  open(f'{folder_name}/teacher_vali.pkl','wb'))\n",
    "\n",
    "pickle.dump(en_idx2word, open(f'{folder_name}/en_idx2word.pkl','wb'))\n",
    "pickle.dump(en_word2idx, open(f'{folder_name}/en_word2idx.pkl','wb'))\n",
    "pickle.dump(cs_idx2word, open(f'{folder_name}/cs_idx2word.pkl','wb'))\n",
    "pickle.dump(cs_word2idx, open(f'{folder_name}/cs_word2idx.pkl','wb'))\n",
    "\n",
    "pickle.dump(en_emb32, open(f'{folder_name}/en_emb32.pkl','wb'))\n",
    "pickle.dump(cs_emb32, open(f'{folder_name}/cs_emb32.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder_train = pickle.load(open(f'{folder_name}/encoder_train.pkl', 'rb'))\n",
    "decoder_train = pickle.load(open(f'{folder_name}/decoder_train.pkl', 'rb'))\n",
    "teacher_train = pickle.load(open(f'{folder_name}/teacher_train.pkl', 'rb'))\n",
    "encoder_vali  = pickle.load(open(f'{folder_name}/encoder_vali.pkl', 'rb'))\n",
    "decoder_vali  = pickle.load(open(f'{folder_name}/decoder_vali.pkl', 'rb'))\n",
    "teacher_vali  = pickle.load(open(f'{folder_name}/teacher_vali.pkl', 'rb'))\n",
    "\n",
    "en_idx2word   = pickle.load(open(f'{folder_name}/en_idx2word.pkl','rb'))\n",
    "en_word2idx   = pickle.load(open(f'{folder_name}/en_word2idx.pkl','rb'))\n",
    "cs_idx2word   = pickle.load(open(f'{folder_name}/cs_idx2word.pkl','rb'))\n",
    "cs_word2idx   = pickle.load(open(f'{folder_name}/cs_word2idx.pkl','rb'))\n",
    "\n",
    "en_emb32    = pickle.load(open(f'{folder_name}/en_emb32.pkl', 'rb'))\n",
    "cs_emb32    = pickle.load(open(f'{folder_name}/cs_emb32.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}