quora_competition.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "710ed17d0c57bd287be0ee3b2782a53a54510561"
   },
   "source": [
    "## IMPORTS "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "abb7e3c30b8a412a50c6b451c49939e3cf4bc11b",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import random\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import re\n",
    "import torch\n",
    "from torchtext import data\n",
    "import spacy\n",
    "from tqdm import tqdm_notebook, tnrange\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "tqdm.pandas(desc='Progress')\n",
    "from collections import Counter\n",
    "from textblob import TextBlob\n",
    "from nltk import word_tokenize\n",
    "\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence\n",
    "from torch.autograd import Variable\n",
    "from torchtext.data import Example\n",
    "from sklearn.metrics import f1_score\n",
    "import torchtext\n",
    "import os \n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# cross validation and metrics\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.metrics import f1_score\n",
    "from torch.optim.optimizer import Optimizer\n",
    "from unidecode import unidecode\n",
    "import regex as re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "9a4ff5590a6f152dc1bec5aeca79aef10218f7de"
   },
   "source": [
    "### Basic Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "deee49df5ca1c4413f71677939e26aa1ff784e44",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "embed_size = 300\n",
    "max_features = 120000\n",
    "maxlen = 70\n",
    "batch_size = 512\n",
    "n_epochs = 5 \n",
    "n_splits = 5 # kfold splits\n",
    "\n",
    "SEED = 1029"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "654cbe3c8a1f2a618a2441afe00df3b4a89e0a58"
   },
   "source": [
    "### Ensure deterministic results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "58bbf87335799247586aaed16531f4d28d10ed4a",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def seed_everything(seed=1029):\n",
    "    random.seed(seed)\n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    torch.cuda.manual_seed(seed)\n",
    "    torch.backends.cudnn.deterministic = True\n",
    "seed_everything()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "c890692644acce2dc4f6e2f929d6d294faca4ad2"
   },
   "source": [
    "### Load embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "7026ee1d913f54dd4b560f654efdb9f833581cd3",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def load_glove(word_index):\n",
    "    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = -0.005838499,0.48782197\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: \n",
    "            embedding_matrix[i] = embedding_vector\n",
    "        else: \n",
    "            embedding_vector = embeddings_index.get(word.upper())\n",
    "            if embedding_vector is not None: \n",
    "                embedding_matrix[i] = embedding_vector\n",
    "            else:\n",
    "                embedding_vector = embeddings_index.get(word.capitalize())\n",
    "                if embedding_vector is not None: \n",
    "                    embedding_matrix[i] = embedding_vector\n",
    "    return embedding_matrix\n",
    "    \n",
    "def load_fasttext(word_index):    \n",
    "    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: \n",
    "            embedding_matrix[i] = embedding_vector\n",
    "        else: \n",
    "            embedding_vector = embeddings_index.get(word.upper())\n",
    "            if embedding_vector is not None: \n",
    "                embedding_matrix[i] = embedding_vector\n",
    "            else:\n",
    "                embedding_vector = embeddings_index.get(word.capitalize())\n",
    "                if embedding_vector is not None: \n",
    "                    embedding_matrix[i] = embedding_vector\n",
    "    return embedding_matrix\n",
    "\n",
    "def load_para(word_index):\n",
    "    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n",
    "    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n",
    "    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n",
    "\n",
    "    all_embs = np.stack(embeddings_index.values())\n",
    "    emb_mean,emb_std = -0.0053247833,0.49346462\n",
    "    embed_size = all_embs.shape[1]\n",
    "\n",
    "    # word_index = tokenizer.word_index\n",
    "    nb_words = min(max_features, len(word_index))\n",
    "    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n",
    "    for word, i in word_index.items():\n",
    "        if i >= max_features: continue\n",
    "        embedding_vector = embeddings_index.get(word)\n",
    "        if embedding_vector is not None: \n",
    "            embedding_matrix[i] = embedding_vector\n",
    "        else: \n",
    "            embedding_vector = embeddings_index.get(word.upper())\n",
    "            if embedding_vector is not None: \n",
    "                embedding_matrix[i] = embedding_vector\n",
    "            else:\n",
    "                embedding_vector = embeddings_index.get(word.capitalize())\n",
    "                if embedding_vector is not None: \n",
    "                    embedding_matrix[i] = embedding_vector\n",
    "    return embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "173753f0178464d2ba26baf22899884d76d1c83d",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"../input/train.csv\")\n",
    "df_test = pd.read_csv(\"../input/test.csv\")\n",
    "df = pd.concat([df_train ,df_test],sort=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "0f75559b6fa28c27ecbf145121309378afc884ee",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def build_vocab(texts):\n",
    "    sentences = texts.apply(lambda x: x.split()).values\n",
    "    vocab = {}\n",
    "    for sentence in sentences:\n",
    "        for word in sentence:\n",
    "            try:\n",
    "                vocab[word] += 1\n",
    "            except KeyError:\n",
    "                vocab[word] = 1\n",
    "    return vocab\n",
    "vocab = build_vocab(df['question_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "28ebb28ba78972bb8d4fee9b53437045542d20fb",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def build_vocab(texts):\n",
    "    sentences = texts.apply(lambda x: x.split()).values\n",
    "    vocab = {}\n",
    "    for sentence in sentences:\n",
    "        for word in sentence:\n",
    "            try:\n",
    "                vocab[word] += 1\n",
    "            except KeyError:\n",
    "                vocab[word] = 1\n",
    "    return vocab\n",
    "\n",
    "def known_contractions(embed):\n",
    "    known = []\n",
    "    for contract in contraction_mapping:\n",
    "        if contract in embed:\n",
    "            known.append(contract)\n",
    "    return known\n",
    "def clean_contractions(text, mapping):\n",
    "    specials = [\"’\", \"‘\", \"´\", \"`\"]\n",
    "    for s in specials:\n",
    "        text = text.replace(s, \"'\")\n",
    "    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n",
    "    return text\n",
    "def correct_spelling(x, dic):\n",
    "    for word in dic.keys():\n",
    "        x = x.replace(word, dic[word])\n",
    "    return x\n",
    "def unknown_punct(embed, punct):\n",
    "    unknown = ''\n",
    "    for p in punct:\n",
    "        if p not in embed:\n",
    "            unknown += p\n",
    "            unknown += ' '\n",
    "    return unknown\n",
    "\n",
    "def clean_numbers(x):\n",
    "    x = re.sub('[0-9]{5,}', '#####', x)\n",
    "    x = re.sub('[0-9]{4}', '####', x)\n",
    "    x = re.sub('[0-9]{3}', '###', x)\n",
    "    x = re.sub('[0-9]{2}', '##', x)\n",
    "    return x\n",
    "\n",
    "def clean_special_chars(text, punct, mapping):\n",
    "    for p in mapping:\n",
    "        text = text.replace(p, mapping[p])\n",
    "    \n",
    "    for p in punct:\n",
    "        text = text.replace(p, f' {p} ')\n",
    "    \n",
    "    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n",
    "    for s in specials:\n",
    "        text = text.replace(s, specials[s])\n",
    "    \n",
    "    return text\n",
    "def add_lower(embedding, vocab):\n",
    "    count = 0\n",
    "    for word in vocab:\n",
    "        if word in embedding and word.lower() not in embedding:  \n",
    "            embedding[word.lower()] = embedding[word]\n",
    "            count += 1\n",
    "    print(f\"Added {count} words to embedding\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "abeab4c80d6829cf2eae706bfa7929e2871af81f",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n",
    " '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n",
    " '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n",
    " '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n",
    " '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n",
    "\n",
    "def clean_text(x):\n",
    "    x = str(x)\n",
    "    for punct in puncts:\n",
    "        x = x.replace(punct, f' {punct} ')\n",
    "    re.sub(r'[^\\x00-\\x7f]',r'', x) \n",
    "    return x\n",
    "\n",
    "def clean_numbers(x):\n",
    "    x = re.sub('[0-9]{5,}', '10000', x)\n",
    "    x = re.sub('[0-9]{4}', '1000', x)\n",
    "    x = re.sub('[0-9]{3}', '100', x)\n",
    "    x = re.sub('[0-9]{2}', '10', x)\n",
    "    return x\n",
    "\n",
    "mispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n",
    "\n",
    "def _get_mispell(mispell_dict):\n",
    "    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n",
    "    return mispell_dict, mispell_re\n",
    "\n",
    "mispellings, mispellings_re = _get_mispell(mispell_dict)\n",
    "def replace_typical_misspell(text):\n",
    "    def replace(match):\n",
    "        return mispellings[match.group(0)]\n",
    "    return mispellings_re.sub(replace, text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "63cb21525251b060aeb309e7be4b48772f8720f5",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "\n",
    "def add_features(df):\n",
    "    \n",
    "    df['question_text'] = df['question_text'].progress_apply(lambda x:str(x))\n",
    "    df['total_length'] = df['question_text'].progress_apply(len)\n",
    "    df['capitals'] = df['question_text'].progress_apply(lambda comment: sum(1 for c in comment if c.isupper()))\n",
    "    df['caps_vs_length'] = df.progress_apply(lambda row: float(row['capitals'])/float(row['total_length']),\n",
    "                                axis=1)\n",
    "    df['num_words'] = df.question_text.str.count('\\S+')\n",
    "    df['num_unique_words'] = df['question_text'].progress_apply(lambda comment: len(set(w for w in comment.split())))\n",
    "    df['words_vs_unique'] = df['num_unique_words'] / df['num_words']  \n",
    "\n",
    "    return df\n",
    "\n",
    "def load_and_prec():\n",
    "    train_df = pd.read_csv(\"../input/train.csv\")\n",
    "    test_df = pd.read_csv(\"../input/test.csv\")\n",
    "    \n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: x.lower())\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: x.lower())\n",
    "\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))\n",
    "\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_numbers(x))\n",
    "\n",
    "    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n",
    "    test_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n",
    "\n",
    "    train = add_features(train_df)\n",
    "    test = add_features(test_df)\n",
    "\n",
    "    features = train[['caps_vs_length', 'words_vs_unique']].fillna(0)\n",
    "    test_features = test[['caps_vs_length', 'words_vs_unique']].fillna(0)\n",
    "\n",
    "    ss = StandardScaler()\n",
    "    ss.fit(np.vstack((features, test_features)))\n",
    "    features = ss.transform(features)\n",
    "    test_features = ss.transform(test_features)\n",
    "\n",
    "    tokenizer = Tokenizer(num_words=max_features,filters='\\t\\n',lower=True)\n",
    "    tokenizer.fit_on_texts(list(np.concatenate([train_X,test_X])))\n",
    "    train_X = tokenizer.texts_to_sequences(train_X)\n",
    "    test_X = tokenizer.texts_to_sequences(test_X)\n",
    "\n",
    "    train_X = pad_sequences(train_X, maxlen=maxlen)\n",
    "    test_X = pad_sequences(test_X, maxlen=maxlen)\n",
    "\n",
    "    train_y = train_df['target'].values\n",
    "\n",
    "    np.random.seed(SEED)\n",
    "    trn_idx = np.random.permutation(len(train_X))\n",
    "\n",
    "    train_X = train_X[trn_idx]\n",
    "    train_y = train_y[trn_idx]\n",
    "    \n",
    "    return train_X, test_X, train_y, features, test_features, tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "3c72fcddb4f680879e231c3dbfc0c71e27fc424c",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "x_train, x_test, y_train, features, test_features, word_index = load_and_prec()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "6a5f4502324d369ff6faa3692accee4f8a233005",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# missing entries in the embedding are set using np.random.normal so we have to seed here too\n",
    "seed_everything()\n",
    "\n",
    "glove_embeddings = load_glove(word_index)\n",
    "paragram_embeddings = load_para(word_index)\n",
    "\n",
    "embedding_matrix = np.mean([glove_embeddings, paragram_embeddings], axis=0)\n",
    "\n",
    "del glove_embeddings, paragram_embeddings\n",
    "gc.collect()\n",
    "\n",
    "np.shape(embedding_matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "d3b707858357eec18403f97368a40c43987b40e8"
   },
   "source": [
    "### Use Stratified K Fold to ensure same target distribution over splits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "fa55c64890964220ded1762236e091e59e502e53",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=SEED).split(x_train, y_train))\n",
    "splits[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "8098bea0cee9117ff9dc4e11feba53e49b80cb55"
   },
   "source": [
    "### Cyclic CLR\n",
    "Code taken from https://www.kaggle.com/dannykliu/lstm-with-attention-clr-in-pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "9d531a7454923f90d0e7443b1ed1373d008c2e88",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# code inspired from: https://github.com/anandsaha/pytorch.cyclic.learning.rate/blob/master/cls.py\n",
    "class CyclicLR(object):\n",
    "    def __init__(self, optimizer, base_lr=1e-3, max_lr=6e-3,\n",
    "                 step_size=2000, mode='triangular', gamma=1.,\n",
    "                 scale_fn=None, scale_mode='cycle', last_batch_iteration=-1):\n",
    "\n",
    "        if not isinstance(optimizer, Optimizer):\n",
    "            raise TypeError('{} is not an Optimizer'.format(\n",
    "                type(optimizer).__name__))\n",
    "        self.optimizer = optimizer\n",
    "\n",
    "        if isinstance(base_lr, list) or isinstance(base_lr, tuple):\n",
    "            if len(base_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} base_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(base_lr)))\n",
    "            self.base_lrs = list(base_lr)\n",
    "        else:\n",
    "            self.base_lrs = [base_lr] * len(optimizer.param_groups)\n",
    "\n",
    "        if isinstance(max_lr, list) or isinstance(max_lr, tuple):\n",
    "            if len(max_lr) != len(optimizer.param_groups):\n",
    "                raise ValueError(\"expected {} max_lr, got {}\".format(\n",
    "                    len(optimizer.param_groups), len(max_lr)))\n",
    "            self.max_lrs = list(max_lr)\n",
    "        else:\n",
    "            self.max_lrs = [max_lr] * len(optimizer.param_groups)\n",
    "\n",
    "        self.step_size = step_size\n",
    "\n",
    "        if mode not in ['triangular', 'triangular2', 'exp_range'] \\\n",
    "                and scale_fn is None:\n",
    "            raise ValueError('mode is invalid and scale_fn is None')\n",
    "\n",
    "        self.mode = mode\n",
    "        self.gamma = gamma\n",
    "\n",
    "        if scale_fn is None:\n",
    "            if self.mode == 'triangular':\n",
    "                self.scale_fn = self._triangular_scale_fn\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'triangular2':\n",
    "                self.scale_fn = self._triangular2_scale_fn\n",
    "                self.scale_mode = 'cycle'\n",
    "            elif self.mode == 'exp_range':\n",
    "                self.scale_fn = self._exp_range_scale_fn\n",
    "                self.scale_mode = 'iterations'\n",
    "        else:\n",
    "            self.scale_fn = scale_fn\n",
    "            self.scale_mode = scale_mode\n",
    "\n",
    "        self.batch_step(last_batch_iteration + 1)\n",
    "        self.last_batch_iteration = last_batch_iteration\n",
    "\n",
    "    def batch_step(self, batch_iteration=None):\n",
    "        if batch_iteration is None:\n",
    "            batch_iteration = self.last_batch_iteration + 1\n",
    "        self.last_batch_iteration = batch_iteration\n",
    "        for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):\n",
    "            param_group['lr'] = lr\n",
    "\n",
    "    def _triangular_scale_fn(self, x):\n",
    "        return 1.\n",
    "\n",
    "    def _triangular2_scale_fn(self, x):\n",
    "        return 1 / (2. ** (x - 1))\n",
    "\n",
    "    def _exp_range_scale_fn(self, x):\n",
    "        return self.gamma**(x)\n",
    "\n",
    "    def get_lr(self):\n",
    "        step_size = float(self.step_size)\n",
    "        cycle = np.floor(1 + self.last_batch_iteration / (2 * step_size))\n",
    "        x = np.abs(self.last_batch_iteration / step_size - 2 * cycle + 1)\n",
    "\n",
    "        lrs = []\n",
    "        param_lrs = zip(self.optimizer.param_groups, self.base_lrs, self.max_lrs)\n",
    "        for param_group, base_lr, max_lr in param_lrs:\n",
    "            base_height = (max_lr - base_lr) * np.maximum(0, (1 - x))\n",
    "            if self.scale_mode == 'cycle':\n",
    "                lr = base_lr + base_height * self.scale_fn(cycle)\n",
    "            else:\n",
    "                lr = base_lr + base_height * self.scale_fn(self.last_batch_iteration)\n",
    "            lrs.append(lr)\n",
    "        return lrs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "8f7e1d15451201efbac2741b03d4b0dfd3b9bfe0"
   },
   "outputs": [],
   "source": [
    "import torch as t\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "embedding_dim = 300\n",
    "embedding_path = '../save/embedding_matrix.npy'\n",
    "use_pretrained_embedding = True\n",
    "\n",
    "hidden_size = 60\n",
    "gru_len = hidden_size\n",
    "\n",
    "Routings = 4 #5\n",
    "Num_capsule = 5\n",
    "Dim_capsule = 5#16\n",
    "dropout_p = 0.25\n",
    "rate_drop_dense = 0.28\n",
    "LR = 0.001\n",
    "T_epsilon = 1e-7\n",
    "num_classes = 30\n",
    "\n",
    "\n",
    "class Embed_Layer(nn.Module):\n",
    "    def __init__(self, embedding_matrix=None, vocab_size=None, embedding_dim=300):\n",
    "        super(Embed_Layer, self).__init__()\n",
    "        self.encoder = nn.Embedding(vocab_size + 1, embedding_dim)\n",
    "        if use_pretrained_embedding:\n",
    "            self.encoder.weight.data.copy_(t.from_numpy(embedding_matrix))  # 方法二\n",
    "\n",
    "    def forward(self, x, dropout_p=0.25):\n",
    "        return nn.Dropout(p=dropout_p)(self.encoder(x))\n",
    "\n",
    "\n",
    "class GRU_Layer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(GRU_Layer, self).__init__()\n",
    "        self.gru = nn.GRU(input_size=300,\n",
    "                          hidden_size=gru_len,\n",
    "                          bidirectional=True)\n",
    "\n",
    "    def init_weights(self):\n",
    "        ih = (param.data for name, param in self.named_parameters() if 'weight_ih' in name)\n",
    "        hh = (param.data for name, param in self.named_parameters() if 'weight_hh' in name)\n",
    "        b = (param.data for name, param in self.named_parameters() if 'bias' in name)\n",
    "        for k in ih:\n",
    "            nn.init.xavier_uniform_(k)\n",
    "        for k in hh:\n",
    "            nn.init.orthogonal_(k)\n",
    "        for k in b:\n",
    "            nn.init.constant_(k, 0)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.gru(x)\n",
    "\n",
    "class Caps_Layer(nn.Module):\n",
    "    def __init__(self, input_dim_capsule=gru_len * 2, num_capsule=Num_capsule, dim_capsule=Dim_capsule, \\\n",
    "                 routings=Routings, kernel_size=(9, 1), share_weights=True,\n",
    "                 activation='default', **kwargs):\n",
    "        super(Caps_Layer, self).__init__(**kwargs)\n",
    "\n",
    "        self.num_capsule = num_capsule\n",
    "        self.dim_capsule = dim_capsule\n",
    "        self.routings = routings\n",
    "        self.kernel_size = kernel_size\n",
    "        self.share_weights = share_weights\n",
    "        if activation == 'default':\n",
    "            self.activation = self.squash\n",
    "        else:\n",
    "            self.activation = nn.ReLU(inplace=True)\n",
    "\n",
    "        if self.share_weights:\n",
    "            self.W = nn.Parameter(\n",
    "                nn.init.xavier_normal_(t.empty(1, input_dim_capsule, self.num_capsule * self.dim_capsule)))\n",
    "        else:\n",
    "            self.W = nn.Parameter(\n",
    "                t.randn(BATCH_SIZE, input_dim_capsule, self.num_capsule * self.dim_capsule))\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        if self.share_weights:\n",
    "            u_hat_vecs = t.matmul(x, self.W)\n",
    "        else:\n",
    "            print('add later')\n",
    "\n",
    "        batch_size = x.size(0)\n",
    "        input_num_capsule = x.size(1)\n",
    "        u_hat_vecs = u_hat_vecs.view((batch_size, input_num_capsule,\n",
    "                                      self.num_capsule, self.dim_capsule))\n",
    "        u_hat_vecs = u_hat_vecs.permute(0, 2, 1, 3) \n",
    "        b = t.zeros_like(u_hat_vecs[:, :, :, 0]) \n",
    "\n",
    "        for i in range(self.routings):\n",
    "            b = b.permute(0, 2, 1)\n",
    "            c = F.softmax(b, dim=2)\n",
    "            c = c.permute(0, 2, 1)\n",
    "            b = b.permute(0, 2, 1)\n",
    "            outputs = self.activation(t.einsum('bij,bijk->bik', (c, u_hat_vecs)))\n",
    "            if i < self.routings - 1:\n",
    "                b = t.einsum('bik,bijk->bij', (outputs, u_hat_vecs))\n",
    "        return outputs\n",
    "\n",
    "    def squash(self, x, axis=-1):\n",
    "        s_squared_norm = (x ** 2).sum(axis, keepdim=True)\n",
    "        scale = t.sqrt(s_squared_norm + T_epsilon)\n",
    "        return x / scale\n",
    "    \n",
    "class Capsule_Main(nn.Module):\n",
    "    def __init__(self, embedding_matrix=None, vocab_size=None):\n",
    "        super(Capsule_Main, self).__init__()\n",
    "        self.embed_layer = Embed_Layer(embedding_matrix, vocab_size)\n",
    "        self.gru_layer = GRU_Layer()\n",
    "        self.gru_layer.init_weights()\n",
    "        self.caps_layer = Caps_Layer()\n",
    "        self.dense_layer = Dense_Layer()\n",
    "\n",
    "    def forward(self, content):\n",
    "        content1 = self.embed_layer(content)\n",
    "        content2, _ = self.gru_layer(\n",
    "            content1)\n",
    "        content3 = self.caps_layer(content2)\n",
    "        output = self.dense_layer(content3)\n",
    "        return output\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "170b27f66c7e546d8e84c79357d40f86c6b1ec42",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "class Attention(nn.Module):\n",
    "    def __init__(self, feature_dim, step_dim, bias=True, **kwargs):\n",
    "        super(Attention, self).__init__(**kwargs)\n",
    "        \n",
    "        self.supports_masking = True\n",
    "\n",
    "        self.bias = bias\n",
    "        self.feature_dim = feature_dim\n",
    "        self.step_dim = step_dim\n",
    "        self.features_dim = 0\n",
    "        \n",
    "        weight = torch.zeros(feature_dim, 1)\n",
    "        nn.init.xavier_uniform_(weight)\n",
    "        self.weight = nn.Parameter(weight)\n",
    "        \n",
    "        if bias:\n",
    "            self.b = nn.Parameter(torch.zeros(step_dim))\n",
    "        \n",
    "    def forward(self, x, mask=None):\n",
    "        feature_dim = self.feature_dim\n",
    "        step_dim = self.step_dim\n",
    "\n",
    "        eij = torch.mm(\n",
    "            x.contiguous().view(-1, feature_dim), \n",
    "            self.weight\n",
    "        ).view(-1, step_dim)\n",
    "        \n",
    "        if self.bias:\n",
    "            eij = eij + self.b\n",
    "            \n",
    "        eij = torch.tanh(eij)\n",
    "        a = torch.exp(eij)\n",
    "        \n",
    "        if mask is not None:\n",
    "            a = a * mask\n",
    "\n",
    "        a = a / torch.sum(a, 1, keepdim=True) + 1e-10\n",
    "\n",
    "        weighted_input = x * torch.unsqueeze(a, -1)\n",
    "        return torch.sum(weighted_input, 1)\n",
    "    \n",
    "class NeuralNet(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(NeuralNet, self).__init__()\n",
    "        \n",
    "        fc_layer = 16\n",
    "        fc_layer1 = 16\n",
    "\n",
    "        self.embedding = nn.Embedding(max_features, embed_size)\n",
    "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
    "        self.embedding.weight.requires_grad = False\n",
    "        \n",
    "        self.embedding_dropout = nn.Dropout2d(0.1)\n",
    "        self.lstm = nn.LSTM(embed_size, hidden_size, bidirectional=True, batch_first=True)\n",
    "        self.gru = nn.GRU(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n",
    "\n",
    "        self.lstm2 = nn.LSTM(hidden_size * 2, hidden_size, bidirectional=True, batch_first=True)\n",
    "\n",
    "        self.lstm_attention = Attention(hidden_size * 2, maxlen)\n",
    "        self.gru_attention = Attention(hidden_size * 2, maxlen)\n",
    "        \n",
    "        self.linear = nn.Linear(hidden_size*8+3, fc_layer1) #643:80 - 483:60 - 323:40\n",
    "        self.relu = nn.ReLU()\n",
    "        self.dropout = nn.Dropout(0.1)\n",
    "        self.fc = nn.Linear(fc_layer**2,fc_layer)\n",
    "        self.out = nn.Linear(fc_layer, 1)\n",
    "        self.lincaps = nn.Linear(Num_capsule * Dim_capsule, 1)\n",
    "        self.caps_layer = Caps_Layer()\n",
    "    \n",
    "    def forward(self, x):\n",
    "        \n",
    "#         Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n",
    "\n",
    "        h_embedding = self.embedding(x[0])\n",
    "        h_embedding = torch.squeeze(\n",
    "            self.embedding_dropout(torch.unsqueeze(h_embedding, 0)))\n",
    "        \n",
    "        h_lstm, _ = self.lstm(h_embedding)\n",
    "        h_gru, _ = self.gru(h_lstm)\n",
    "\n",
    "        ##Capsule Layer        \n",
    "        content3 = self.caps_layer(h_gru)\n",
    "        content3 = self.dropout(content3)\n",
    "        batch_size = content3.size(0)\n",
    "        content3 = content3.view(batch_size, -1)\n",
    "        content3 = self.relu(self.lincaps(content3))\n",
    "\n",
    "        ##Attention Layer\n",
    "        h_lstm_atten = self.lstm_attention(h_lstm)\n",
    "        h_gru_atten = self.gru_attention(h_gru)\n",
    "        \n",
    "        # global average pooling\n",
    "        avg_pool = torch.mean(h_gru, 1)\n",
    "        # global max pooling\n",
    "        max_pool, _ = torch.max(h_gru, 1)\n",
    "        \n",
    "        f = torch.tensor(x[1], dtype=torch.float).cuda()\n",
    "        conc = torch.cat((h_lstm_atten, h_gru_atten,content3, avg_pool, max_pool,f), 1)\n",
    "        conc = self.relu(self.linear(conc))\n",
    "        conc = self.dropout(conc)\n",
    "\n",
    "        out = self.out(conc)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "0c6339acdc14e6688ed47e9f439e81cd7cfca57f"
   },
   "outputs": [],
   "source": [
    "def sigmoid(x):\n",
    "    return 1 / (1 + np.exp(-x))\n",
    "\n",
    "train_preds = np.zeros((len(x_train)))\n",
    "test_preds = np.zeros((len(df_test)))\n",
    "\n",
    "seed_everything()\n",
    "\n",
    "x_test_cuda = torch.tensor(x_test, dtype=torch.long).cuda()\n",
    "test = torch.utils.data.TensorDataset(x_test_cuda)\n",
    "test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "avg_losses_f = []\n",
    "avg_val_losses_f = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "2cfdd46b117a11af54ed01f1f9511782e50a00fa",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "for i, (train_idx, valid_idx) in enumerate(splits):    \n",
    "    x_train = np.array(x_train)\n",
    "    y_train = np.array(y_train)\n",
    "    features = np.array(features)\n",
    "\n",
    "    x_train_fold = torch.tensor(x_train[train_idx.astype(int)], dtype=torch.long).cuda()\n",
    "    y_train_fold = torch.tensor(y_train[train_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n",
    "    \n",
    "    kfold_X_features = features[train_idx.astype(int)]\n",
    "    kfold_X_valid_features = features[valid_idx.astype(int)]\n",
    "    x_val_fold = torch.tensor(x_train[valid_idx.astype(int)], dtype=torch.long).cuda()\n",
    "    y_val_fold = torch.tensor(y_train[valid_idx.astype(int), np.newaxis], dtype=torch.float32).cuda()\n",
    "\n",
    "    model = NeuralNet()\n",
    "\n",
    "    model.cuda()\n",
    "\n",
    "    # define binary cross entropy loss\n",
    "    loss_fn = torch.nn.BCEWithLogitsLoss(reduction='sum')\n",
    "\n",
    "    step_size = 300\n",
    "    base_lr, max_lr = 0.001, 0.002   \n",
    "    optimizer = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), \n",
    "                             lr=max_lr)\n",
    "\n",
    "    scheduler = CyclicLR(optimizer, base_lr=base_lr, max_lr=max_lr,\n",
    "               step_size=step_size, mode='exp_range',\n",
    "               gamma=0.99996)\n",
    "\n",
    "    train = torch.utils.data.TensorDataset(x_train_fold, y_train_fold)\n",
    "    valid = torch.utils.data.TensorDataset(x_val_fold, y_val_fold)\n",
    "    \n",
    "    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "    valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "    print(f'Fold {i + 1}')\n",
    "    for epoch in range(n_epochs):\n",
    "        start_time = time.time()\n",
    "        model.train()\n",
    "\n",
    "        avg_loss = 0.  \n",
    "        for i, (x_batch, y_batch) in enumerate(train_loader):          \n",
    "            f = kfold_X_features[i * batch_size:(i+1) * batch_size]\n",
    "            y_pred = model([x_batch,f])\n",
    "            if scheduler:\n",
    "                scheduler.batch_step()\n",
    "\n",
    "            loss = loss_fn(y_pred, y_batch)\n",
    "            optimizer.zero_grad()\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            avg_loss += loss.item() / len(train_loader)\n",
    "\n",
    "        model.eval()\n",
    "        \n",
    "        # predict all the samples in y_val_fold batch per batch\n",
    "        valid_preds_fold = np.zeros((x_val_fold.size(0)))\n",
    "        test_preds_fold = np.zeros((len(df_test)))\n",
    "        \n",
    "        avg_val_loss = 0.\n",
    "        for i, (x_batch, y_batch) in enumerate(valid_loader):\n",
    "            f = kfold_X_valid_features[i * batch_size:(i+1) * batch_size]            \n",
    "            y_pred = model([x_batch,f]).detach()\n",
    "            \n",
    "            avg_val_loss += loss_fn(y_pred, y_batch).item() / len(valid_loader)\n",
    "            valid_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n",
    "        \n",
    "        elapsed_time = time.time() - start_time \n",
    "        print('Epoch {}/{} \\t loss={:.4f} \\t val_loss={:.4f} \\t time={:.2f}s'.format(\n",
    "            epoch + 1, n_epochs, avg_loss, avg_val_loss, elapsed_time))\n",
    "    avg_losses_f.append(avg_loss)\n",
    "    avg_val_losses_f.append(avg_val_loss) \n",
    "    # predict all samples in the test set\n",
    "    for i, (x_batch,) in enumerate(test_loader):\n",
    "        f = test_features[i * batch_size:(i+1) * batch_size]\n",
    "        y_pred = model([x_batch,f]).detach()\n",
    "\n",
    "        test_preds_fold[i * batch_size:(i+1) * batch_size] = sigmoid(y_pred.cpu().numpy())[:, 0]\n",
    "        \n",
    "    train_preds[valid_idx] = valid_preds_fold\n",
    "    test_preds += test_preds_fold / len(splits)\n",
    "\n",
    "print('All \\t loss={:.4f} \\t val_loss={:.4f} \\t '.format(np.average(avg_losses_f),np.average(avg_val_losses_f)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "1d2187d4bbf48350eaf365b6dd8b027d5be69e0c"
   },
   "source": [
    "### Find final Thresshold\n",
    "\n",
    "Borrowed from: https://www.kaggle.com/ziliwang/baseline-pytorch-bilstm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "dc4a4c681294ba06526fed4e871cfe8639cf25e4",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def best_threshold(y_train,train_preds):\n",
    "    tmp = [0,0,0] # idx, cur, max\n",
    "    delta = 0\n",
    "    for tmp[0] in tqdm(np.arange(0.1, 0.501, 0.01)):\n",
    "        tmp[1] = f1_score(y_train, np.array(train_preds)>tmp[0])\n",
    "        if tmp[1] > tmp[2]:\n",
    "            delta = tmp[0]\n",
    "            tmp[2] = tmp[1]\n",
    "    print('best threshold is {:.4f} with F1 score: {:.4f}'.format(delta, tmp[2]))\n",
    "    return delta\n",
    "delta = best_threshold(y_train,train_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_uuid": "d17dd7b0a92ec98134bf8996fd210edaadf7bed6",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "submission = df_test[['qid']].copy()\n",
    "submission['prediction'] = (test_preds > delta).astype(int)\n",
    "submission.to_csv('submission.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}