Jigsaw_Unintended_Bias_in_Toxicity_Classification_V01.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "bert-lstm-simple-blender-0-93844-lb.ipynb",
      "version": "0.3.2",
      "provenance": [],
      "collapsed_sections": []
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.6"
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uYRm-x_w3fQ_",
        "colab_type": "text"
      },
      "source": [
        "**The below notebook illustrates an exercice for the \"Jigsaw Unintended Bias in Toxicity Classification competition in kaggle\"**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "__ExpeCE4HXx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## importing the libraries we will use:\n",
        "import os\n",
        "import time\n",
        "import gc\n",
        "import sys\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import fastai\n",
        "from fastai.train import Learner\n",
        "from fastai.train import DataBunch\n",
        "from fastai.callbacks import TrainingPhase, GeneralScheduler\n",
        "from fastai.basic_data import DatasetType\n",
        "import fastprogress\n",
        "from fastprogress import force_console_behavior\n",
        "from pprint import pprint\n",
        "import random\n",
        "from tqdm._tqdm_notebook import tqdm_notebook as tqdm\n",
        "from keras.preprocessing import text, sequence\n",
        "import torch\n",
        "from torch import nn\n",
        "from torch.utils import data\n",
        "from torch.nn import functional as F\n",
        "import torch.utils.data\n",
        "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification, BertAdam\n",
        "from pytorch_pretrained_bert import BertConfig\n",
        "from nltk.tokenize.treebank import TreebankWordTokenizer\n",
        "from tqdm import tqdm\n",
        "import warnings\n",
        "from gensim.models import KeyedVectors"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
        "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
        "id": "AvH4xskC3fRA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Loading the directory for our work\n",
        "package_dir = \"../input/pytorch-pretrained-BERT\"\n",
        "sys.path.append(package_dir)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bz8oEMls3fRE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from __future__ import absolute_import\n",
        "from __future__ import division\n",
        "from __future__ import print_function"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
        "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a",
        "collapsed": true,
        "id": "Sb_UNRwE3fRH",
        "colab_type": "code",
        "colab": {},
        "outputId": "ab747c95-8b3a-41f6-ac11-dc8c5e653f8b"
      },
      "source": [
        "%reload_ext autoreload\n",
        "%autoreload 2\n",
        "%matplotlib inline\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aXPgHQ_N3fRR",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## let's define some functions we will use later :\n",
        "def convert_lines(example, max_seq_length,tokenizer):\n",
        "    max_seq_length -=2\n",
        "    all_tokens = []\n",
        "    longer = 0\n",
        "    for text in tqdm(example):\n",
        "        tokens_a = tokenizer.tokenize(text)\n",
        "        if len(tokens_a)>max_seq_length:\n",
        "            tokens_a = tokens_a[:max_seq_length]\n",
        "            longer += 1\n",
        "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
        "        all_tokens.append(one_token)\n",
        "    return np.array(all_tokens)\n",
        "\n",
        "def is_interactive():\n",
        "    return 'SHLVL' not in os.environ\n",
        "\n",
        "def seed_everything(seed=123):\n",
        "    random.seed(seed)\n",
        "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
        "    np.random.seed(seed)\n",
        "    torch.manual_seed(seed)\n",
        "    torch.cuda.manual_seed(seed)\n",
        "    torch.backends.cudnn.deterministic = True\n",
        "    \n",
        "def get_coefs(word, *arr):\n",
        "    return word, np.asarray(arr, dtype='float32')\n",
        "\n",
        "\n",
        "def load_embeddings(path):\n",
        "    #with open(path,'rb') as f:\n",
        "    emb_arr = KeyedVectors.load(path)\n",
        "    return emb_arr\n",
        "\n",
        "def build_matrix(word_index, path):\n",
        "    embedding_index = load_embeddings(path)\n",
        "    embedding_matrix = np.zeros((max_features + 1, 300))\n",
        "    unknown_words = []\n",
        "    \n",
        "    for word, i in word_index.items():\n",
        "        if i <= max_features:\n",
        "            try:\n",
        "                embedding_matrix[i] = embedding_index[word]\n",
        "            except KeyError:\n",
        "                try:\n",
        "                    embedding_matrix[i] = embedding_index[word.lower()]\n",
        "                except KeyError:\n",
        "                    try:\n",
        "                        embedding_matrix[i] = embedding_index[word.title()]\n",
        "                    except KeyError:\n",
        "                        unknown_words.append(word)\n",
        "    return embedding_matrix, unknown_words\n",
        "\n",
        "def sigmoid(x):\n",
        "    return 1 / (1 + np.exp(-x))\n",
        "\n",
        "class SpatialDropout(nn.Dropout2d):\n",
        "    def forward(self, x):\n",
        "        x = x.unsqueeze(2)    # (N, T, 1, K)\n",
        "        x = x.permute(0, 3, 2, 1)  # (N, K, 1, T)\n",
        "        x = super(SpatialDropout, self).forward(x)  # (N, K, 1, T), some features are masked\n",
        "        x = x.permute(0, 3, 2, 1)  # (N, T, 1, K)\n",
        "        x = x.squeeze(2)  # (N, T, K)\n",
        "        return x\n",
        "\n",
        "def train_model(learn,test,output_dim,lr=0.001,\n",
        "                batch_size=512, n_epochs=4,\n",
        "                enable_checkpoint_ensemble=True):\n",
        "    \n",
        "    all_test_preds = []\n",
        "    checkpoint_weights = [2 ** epoch for epoch in range(n_epochs)]\n",
        "    test_loader = torch.utils.data.DataLoader(test, batch_size=batch_size, shuffle=False)\n",
        "    n = len(learn.data.train_dl)\n",
        "    phases = [(TrainingPhase(n).schedule_hp('lr', lr * (0.6**(i)))) for i in range(n_epochs)]\n",
        "    sched = GeneralScheduler(learn, phases)\n",
        "    learn.callbacks.append(sched)\n",
        "    for epoch in range(n_epochs):\n",
        "        learn.fit(1)\n",
        "        test_preds = np.zeros((len(test), output_dim))    \n",
        "        for i, x_batch in enumerate(test_loader):\n",
        "            X = x_batch[0].cuda()\n",
        "            y_pred = sigmoid(learn.model(X).detach().cpu().numpy())\n",
        "            test_preds[i * batch_size:(i+1) * batch_size, :] = y_pred\n",
        "\n",
        "        all_test_preds.append(test_preds)\n",
        "\n",
        "\n",
        "    if enable_checkpoint_ensemble:\n",
        "        test_preds = np.average(all_test_preds, weights=checkpoint_weights, axis=0)    \n",
        "    else:\n",
        "        test_preds = all_test_preds[-1]\n",
        "        \n",
        "    return test_preds\n",
        "\n",
        "def handle_punctuation(x):\n",
        "    x = x.translate(remove_dict)\n",
        "    x = x.translate(isolate_dict)\n",
        "    return x\n",
        "\n",
        "def handle_contractions(x):\n",
        "    x = tokenizer.tokenize(x)\n",
        "    return x\n",
        "\n",
        "def fix_quote(x):\n",
        "    x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n",
        "    x = ' '.join(x)\n",
        "    return x\n",
        "\n",
        "def preprocess(x):\n",
        "    x = handle_punctuation(x)\n",
        "    x = handle_contractions(x)\n",
        "    x = fix_quote(x)\n",
        "    return x\n",
        "\n",
        "class SequenceBucketCollator():\n",
        "    def __init__(self, choose_length, sequence_index, length_index, label_index=None):\n",
        "        self.choose_length = choose_length\n",
        "        self.sequence_index = sequence_index\n",
        "        self.length_index = length_index\n",
        "        self.label_index = label_index\n",
        "        \n",
        "    def __call__(self, batch):\n",
        "        batch = [torch.stack(x) for x in list(zip(*batch))]\n",
        "        \n",
        "        sequences = batch[self.sequence_index]\n",
        "        lengths = batch[self.length_index]\n",
        "        \n",
        "        length = self.choose_length(lengths)\n",
        "        mask = torch.arange(start=maxlen, end=0, step=-1) < length\n",
        "        padded_sequences = sequences[:, mask]\n",
        "        \n",
        "        batch[self.sequence_index] = padded_sequences\n",
        "        \n",
        "        if self.label_index is not None:\n",
        "            return [x for i, x in enumerate(batch) if i != self.label_index], batch[self.label_index]\n",
        "    \n",
        "        return batch\n",
        "    \n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hNhD1Q-E48oq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## More function and class definitions:\n",
        "class NeuralNet(nn.Module):\n",
        "    def __init__(self, embedding_matrix, num_aux_targets):\n",
        "        super(NeuralNet, self).__init__()\n",
        "        embed_size = embedding_matrix.shape[1]\n",
        "        \n",
        "        self.embedding = nn.Embedding(max_features, embed_size)\n",
        "        self.embedding.weight = nn.Parameter(torch.tensor(embedding_matrix, dtype=torch.float32))\n",
        "        self.embedding.weight.requires_grad = False\n",
        "        self.embedding_dropout = SpatialDropout(0.3)\n",
        "        \n",
        "        self.lstm1 = nn.LSTM(embed_size, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
        "        self.lstm2 = nn.LSTM(LSTM_UNITS * 2, LSTM_UNITS, bidirectional=True, batch_first=True)\n",
        "    \n",
        "        self.linear1 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
        "        self.linear2 = nn.Linear(DENSE_HIDDEN_UNITS, DENSE_HIDDEN_UNITS)\n",
        "        \n",
        "        self.linear_out = nn.Linear(DENSE_HIDDEN_UNITS, 1)\n",
        "        self.linear_aux_out = nn.Linear(DENSE_HIDDEN_UNITS, num_aux_targets)\n",
        "        \n",
        "    def forward(self, x, lengths=None):\n",
        "        h_embedding = self.embedding(x.long())\n",
        "        h_embedding = self.embedding_dropout(h_embedding)\n",
        "        \n",
        "        h_lstm1, _ = self.lstm1(h_embedding)\n",
        "        h_lstm2, _ = self.lstm2(h_lstm1)\n",
        "        \n",
        "        # global average pooling\n",
        "        avg_pool = torch.mean(h_lstm2, 1)\n",
        "        # global max pooling\n",
        "        max_pool, _ = torch.max(h_lstm2, 1)\n",
        "        \n",
        "        h_conc = torch.cat((max_pool, avg_pool), 1)\n",
        "        h_conc_linear1  = F.relu(self.linear1(h_conc))\n",
        "        h_conc_linear2  = F.relu(self.linear2(h_conc))\n",
        "        \n",
        "        hidden = h_conc + h_conc_linear1 + h_conc_linear2\n",
        "        \n",
        "        result = self.linear_out(hidden)\n",
        "        aux_result = self.linear_aux_out(hidden)\n",
        "        out = torch.cat([result, aux_result], 1)\n",
        "        \n",
        "        return out\n",
        "    \n",
        "def custom_loss(data, targets):\n",
        "    bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2])(data[:,:1],targets[:,:1])\n",
        "    bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])\n",
        "    return (bce_loss_1 * loss_weight) + bce_loss_2\n",
        "\n",
        "def reduce_mem_usage(df):\n",
        "    start_mem = df.memory_usage().sum() / 1024**2\n",
        "    print('The emory usage of this dataframe is {:.2f} MB'.format(start_mem))\n",
        "    \n",
        "    for col in df.columns:\n",
        "        col_type = df[col].dtype\n",
        "        \n",
        "        if col_type != object:\n",
        "            c_min = df[col].min()\n",
        "            c_max = df[col].max()\n",
        "            if str(col_type)[:3] == 'int':\n",
        "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
        "                    df[col] = df[col].astype(np.int8)\n",
        "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
        "                    df[col] = df[col].astype(np.int16)\n",
        "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
        "                    df[col] = df[col].astype(np.int32)\n",
        "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
        "                    df[col] = df[col].astype(np.int64)  \n",
        "            else:\n",
        "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
        "                    df[col] = df[col].astype(np.float16)\n",
        "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
        "                    df[col] = df[col].astype(np.float32)\n",
        "                else:\n",
        "                    df[col] = df[col].astype(np.float64)\n",
        "        else:\n",
        "            df[col] = df[col].astype('category')\n",
        "\n",
        "    end_mem = df.memory_usage().sum() / 1024**2\n",
        "    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n",
        "    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n",
        "    \n",
        "    return df"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sQ7mooEq3fRV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "warnings.filterwarnings(action='once')\n",
        "SEED = 1234\n",
        "\n",
        "BATCH_SIZE = 512\n",
        "MAX_SEQUENCE_LENGTH = 300\n",
        "\n",
        "device = torch.device('cuda')\n",
        "\n",
        "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-78_a-12/uncased_L-12_H-768_A-12/'\n",
        "np.random.seed(SEED)\n",
        "torch.manual_seed(SEED)\n",
        "torch.cuda.manual_seed(SEED)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "bert_config = BertConfig('../input/arti-bert-inference/bert/bert_config.json')\n",
        "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)\n",
        "\n",
        "tqdm.pandas()\n",
        "CRAWL_EMBEDDING_PATH = '../input/gensim-embeddings-dataset/crawl-300d-2M.gensim'\n",
        "GLOVE_EMBEDDING_PATH = '../input/gensim-embeddings-dataset/glove.840B.300d.gensim'\n",
        "NUM_MODELS = 2\n",
        "LSTM_UNITS = 128\n",
        "DENSE_HIDDEN_UNITS = 4 * LSTM_UNITS\n",
        "MAX_LEN = 220\n",
        "if not is_interactive():\n",
        "    def nop(it, *a, **k):\n",
        "        return it\n",
        "\n",
        "    tqdm = nop\n",
        "\n",
        "    fastprogress.fastprogress.NO_BAR = True\n",
        "    master_bar, progress_bar = force_console_behavior()\n",
        "    fastai.basic_train.master_bar, fastai.basic_train.progress_bar = master_bar, progress_bar\n",
        "\n",
        "seed_everything()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cZewsnBn3fRY",
        "colab_type": "text"
      },
      "source": [
        "**Let's start with an interesting part : the BERT Part**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xy1ryD4w3fRa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## Importing our data set:\n",
        "\n",
        "test_df = pd.read_csv(\"../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv\")\n",
        "test_df['comment_text'] = test_df['comment_text'].astype(str) \n",
        "X_test = convert_lines(test_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"), MAX_SEQUENCE_LENGTH, tokenizer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wx4uX6-V3fRd",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## let's call our model :\n",
        "model = BertForSequenceClassification(bert_config, num_labels=1)\n",
        "model.load_state_dict(torch.load(\"../input/arti-bert-inference/bert/bert_pytorch.bin\"))\n",
        "model.to(device)\n",
        "for param in model.parameters():\n",
        "    param.requires_grad = False\n",
        "model.eval()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0r0SWkDp3fRk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_preds = np.zeros((len(X_test)))\n",
        "test = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\n",
        "test_loader = torch.utils.data.DataLoader(test, batch_size=512, shuffle=False)\n",
        "tk0 = tqdm(test_loader)\n",
        "for i, (x_batch,) in enumerate(tk0):\n",
        "    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n",
        "    test_preds[i * 512:(i + 1) * 512] = pred[:, 0].detach().cpu().squeeze().numpy()\n",
        "\n",
        "test_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7MwIG1ho3fRt",
        "colab_type": "text"
      },
      "source": [
        "**An other cool part of this project is the LSTM  model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LaeOy7pw3fRu",
        "colab_type": "code",
        "colab": {},
        "outputId": "cf42078b-82e5-4f86-859f-a752adf52790"
      },
      "source": [
        "train_df = reduce_mem_usage(pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv'))"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Memory usage of dataframe is 619.65 MB\n",
            "Memory usage after optimization is: 350.87 MB\n",
            "Decreased by 43.4%\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bdRx5IMf3fRz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## some cleaning to do. I should probably think of a way to do this in a more elegent way:\n",
        "\n",
        "symbols_to_isolate = '.,?!-;*\"…:—()%#$&_/@＼・ω+=”“[]^–>\\\\°<~•≠™ˈʊɒ∞§{}·τα❤☺ɡ|¢→̶`❥━┣┫┗Ｏ►★©―ɪ✔®\\x96\\x92●£♥➤´¹☕≈÷♡◐║▬′ɔː€۩۞†μ✒➥═☆ˌ◄½ʻπδηλσερνʃ✬ＳＵＰＥＲＩＴ☻±♍µº¾✓◾؟．⬅℅»Вав❣⋅¿¬♫ＣＭβ█▓▒░⇒⭐›¡₂₃❧▰▔◞▀▂▃▄▅▆▇↙γ̄″☹➡«φ⅓„✋：¥̲̅́∙‛◇✏▷❓❗¶˚˙）сиʿ✨。ɑ\\x80◕！％¯−ﬂﬁ₁²ʌ¼⁴⁄₄⌠♭✘╪▶☭✭♪☔☠♂☃☎✈✌✰❆☙○‣⚓年∎ℒ▪▙☏⅛ｃａｓǀ℮¸ｗ‚∼‖ℳ❄←☼⋆ʒ⊂、⅔¨͡๏⚾⚽Φ×θ￦？（℃⏩☮⚠月✊❌⭕▸■⇌☐☑⚡☄ǫ╭∩╮，例＞ʕɐ̣Δ₀✞┈╱╲▏▕┃╰▊▋╯┳┊≥☒↑☝ɹ✅☛♩☞ＡＪＢ◔◡↓♀⬆̱ℏ\\x91⠀ˤ╚↺⇤∏✾◦♬³の｜／∵∴√Ω¤☜▲↳▫‿⬇✧ｏｖｍ－２０８＇‰≤∕ˆ⚜☁'\n",
        "symbols_to_delete = '\\n🍕\\r🐵😑\\xa0\\ue014\\t\\uf818\\uf04a\\xad😢🐶️\\uf0e0😜😎👊\\u200b\\u200e😁عدويهصقأناخلىبمغر😍💖💵Е👎😀😂\\u202a\\u202c🔥😄🏻💥ᴍʏʀᴇɴᴅᴏᴀᴋʜᴜʟᴛᴄᴘʙғᴊᴡɢ😋👏שלוםבי😱‼\\x81エンジ故障\\u2009🚌ᴵ͞🌟😊😳😧🙀😐😕\\u200f👍😮😃😘אעכח💩💯⛽🚄🏼ஜ😖ᴠ🚲‐😟😈💪🙏🎯🌹😇💔😡\\x7f👌ἐὶήιὲκἀίῃἴξ🙄Ｈ😠\\ufeff\\u2028😉😤⛺🙂\\u3000تحكسة👮💙فزط😏🍾🎉😞\\u2008🏾😅😭👻😥😔😓🏽🎆🍻🍽🎶🌺🤔😪\\x08‑🐰🐇🐱🙆😨🙃💕𝘊𝘦𝘳𝘢𝘵𝘰𝘤𝘺𝘴𝘪𝘧𝘮𝘣💗💚地獄谷улкнПоАН🐾🐕😆ה🔗🚽歌舞伎🙈😴🏿🤗🇺🇸мυтѕ⤵🏆🎃😩\\u200a🌠🐟💫💰💎эпрд\\x95🖐🙅⛲🍰🤐👆🙌\\u2002💛🙁👀🙊🙉\\u2004ˢᵒʳʸᴼᴷᴺʷᵗʰᵉᵘ\\x13🚬🤓\\ue602😵άοόςέὸתמדףנרךצט😒͝🆕👅👥👄🔄🔤👉👤👶👲🔛🎓\\uf0b7\\uf04c\\x9f\\x10成都😣⏺😌🤑🌏😯ех😲Ἰᾶὁ💞🚓🔔📚🏀👐\\u202d💤🍇\\ue613小土豆🏡❔⁉\\u202f👠》कर्मा🇹🇼🌸蔡英文🌞🎲レクサス😛外国人关系Сб💋💀🎄💜🤢َِьыгя不是\\x9c\\x9d🗑\\u2005💃📣👿༼つ༽😰ḷЗз▱ц￼🤣卖温哥华议会下降你失去所有的钱加拿大坏税骗子🐝ツ🎅\\x85🍺آإشء🎵🌎͟ἔ油别克🤡🤥😬🤧й\\u2003🚀🤴ʲшчИОРФДЯМюж😝🖑ὐύύ特殊作戦群щ💨圆明园קℐ🏈😺🌍⏏ệ🍔🐮🍁🍆🍑🌮🌯🤦\\u200d𝓒𝓲𝓿𝓵안영하세요ЖљКћ🍀😫🤤ῦ我出生在了可以说普通话汉语好极🎼🕺🍸🥂🗽🎇🎊🆘🤠👩🖒🚪天一家⚲\\u2006⚭⚆⬭⬯⏖新✀╌🇫🇷🇩🇪🇮🇬🇧😷🇨🇦ХШ🌐\\x1f杀鸡给猴看ʁ𝗪𝗵𝗲𝗻𝘆𝗼𝘂𝗿𝗮𝗹𝗶𝘇𝗯𝘁𝗰𝘀𝘅𝗽𝘄𝗱📺ϖ\\u2000үսᴦᎥһͺ\\u2007հ\\u2001ɩｙｅ൦ｌƽｈ𝐓𝐡𝐞𝐫𝐮𝐝𝐚𝐃𝐜𝐩𝐭𝐢𝐨𝐧Ƅᴨןᑯ໐ΤᏧ௦Іᴑ܁𝐬𝐰𝐲𝐛𝐦𝐯𝐑𝐙𝐣𝐇𝐂𝐘𝟎ԜТᗞ౦〔Ꭻ𝐳𝐔𝐱𝟔𝟓𝐅🐋ﬃ💘💓ё𝘥𝘯𝘶💐🌋🌄🌅𝙬𝙖𝙨𝙤𝙣𝙡𝙮𝙘𝙠𝙚𝙙𝙜𝙧𝙥𝙩𝙪𝙗𝙞𝙝𝙛👺🐷ℋ𝐀𝐥𝐪🚶𝙢Ἱ🤘ͦ💸ج패티Ｗ𝙇ᵻ👂👃ɜ🎫\\uf0a7БУі🚢🚂ગુજરાતીῆ🏃𝓬𝓻𝓴𝓮𝓽𝓼☘﴾̯﴿₽\\ue807𝑻𝒆𝒍𝒕𝒉𝒓𝒖𝒂𝒏𝒅𝒔𝒎𝒗𝒊👽😙\\u200cЛ‒🎾👹⎌🏒⛸公寓养宠物吗🏄🐀🚑🤷操美𝒑𝒚𝒐𝑴🤙🐒欢迎来到阿拉斯ספ𝙫🐈𝒌𝙊𝙭𝙆𝙋𝙍𝘼𝙅ﷻ🦄巨收赢得白鬼愤怒要买额ẽ🚗🐳𝟏𝐟𝟖𝟑𝟕𝒄𝟗𝐠𝙄𝙃👇锟斤拷𝗢𝟳𝟱𝟬⦁マルハニチロ株式社⛷한국어ㄸㅓ니͜ʖ𝘿𝙔₵𝒩ℯ𝒾𝓁𝒶𝓉𝓇𝓊𝓃𝓈𝓅ℴ𝒻𝒽𝓀𝓌𝒸𝓎𝙏ζ𝙟𝘃𝗺𝟮𝟭𝟯𝟲👋🦊多伦🐽🎻🎹⛓🏹🍷🦆为和中友谊祝贺与其想象对法如直接问用自己猜本传教士没积唯认识基督徒曾经让相信耶稣复活死怪他但当们聊些政治题时候战胜因圣把全堂结婚孩恐惧且栗谓这样还♾🎸🤕🤒⛑🎁批判检讨🏝🦁🙋😶쥐스탱트뤼도석유가격인상이경제황을렵게만들지않록잘관리해야합다캐나에서대마초와화약금의품런성분갈때는반드시허된사용🔫👁凸ὰ💲🗯𝙈Ἄ𝒇𝒈𝒘𝒃𝑬𝑶𝕾𝖙𝖗𝖆𝖎𝖌𝖍𝖕𝖊𝖔𝖑𝖉𝖓𝖐𝖜𝖞𝖚𝖇𝕿𝖘𝖄𝖛𝖒𝖋𝖂𝕴𝖟𝖈𝕸👑🚿💡知彼百\\uf005𝙀𝒛𝑲𝑳𝑾𝒋𝟒😦𝙒𝘾𝘽🏐𝘩𝘨ὼṑ𝑱𝑹𝑫𝑵𝑪🇰🇵👾ᓇᒧᔭᐃᐧᐦᑳᐨᓃᓂᑲᐸᑭᑎᓀᐣ🐄🎈🔨🐎🤞🐸💟🎰🌝🛳点击查版🍭𝑥𝑦𝑧ＮＧ👣\\uf020っ🏉ф💭🎥Ξ🐴👨🤳🦍\\x0b🍩𝑯𝒒😗𝟐🏂👳🍗🕉🐲چی𝑮𝗕𝗴🍒ꜥⲣⲏ🐑⏰鉄リ事件ї💊「」\\uf203\\uf09a\\uf222\\ue608\\uf202\\uf099\\uf469\\ue607\\uf410\\ue600燻製シ虚偽屁理屈Г𝑩𝑰𝒀𝑺🌤𝗳𝗜𝗙𝗦𝗧🍊ὺἈἡχῖΛ⤏🇳𝒙ψՁմեռայինրւդձ冬至ὀ𝒁🔹🤚🍎𝑷🐂💅𝘬𝘱𝘸𝘷𝘐𝘭𝘓𝘖𝘹𝘲𝘫کΒώ💢ΜΟΝΑΕ🇱♲𝝈↴💒⊘Ȼ🚴🖕🖤🥘📍👈➕🚫🎨🌑🐻𝐎𝐍𝐊𝑭🤖🎎😼🕷ｇｒｎｔｉｄｕｆｂｋ𝟰🇴🇭🇻🇲𝗞𝗭𝗘𝗤👼📉🍟🍦🌈🔭《🐊🐍\\uf10aლڡ🐦\\U0001f92f\\U0001f92a🐡💳ἱ🙇𝗸𝗟𝗠𝗷🥜さようなら🔼'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O2Bipnon3fR2",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = TreebankWordTokenizer()\n",
        "\n",
        "isolate_dict = {ord(c):f' {c} ' for c in symbols_to_isolate}\n",
        "remove_dict = {ord(c):f'' for c in symbols_to_delete}"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rXSROJFS3fR6",
        "colab_type": "code",
        "colab": {},
        "outputId": "c4649c22-f83f-479d-c90e-731a8385fea8"
      },
      "source": [
        "x_train = train_df['comment_text'].progress_apply(lambda x:preprocess(x))\n",
        "y_aux_train = train_df[['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']]\n",
        "x_test = test_df['comment_text'].progress_apply(lambda x:preprocess(x))\n",
        "\n",
        "identity_columns = [\n",
        "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
        "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
        "# Overall\n",
        "weights = np.ones((len(x_train),)) / 4\n",
        "# Subgroup\n",
        "weights += (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) / 4\n",
        "# Background Positive, Subgroup Negative\n",
        "weights += (( (train_df['target'].values>=0.5).astype(bool).astype(np.int) +\n",
        "   (train_df[identity_columns].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
        "# Background Negative, Subgroup Positive\n",
        "weights += (( (train_df['target'].values<0.5).astype(bool).astype(np.int) +\n",
        "   (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
        "loss_weight = 1.0 / weights.mean()\n",
        "\n",
        "y_train = np.vstack([(train_df['target'].values>=0.5).astype(np.int),weights]).T\n",
        "\n",
        "max_features = 410047"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " 99%|█████████▊| 1780823/1804874 [08:50<00:07, 3356.38it/s]\n",
            "100%|██████████| 97320/97320 [00:28<00:00, 3402.04it/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XIu7PVIx3fR_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer = text.Tokenizer(num_words = max_features, filters='',lower=False)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HU-CRZ8K3fSC",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "tokenizer.fit_on_texts(list(x_train) + list(x_test))\n",
        "\n",
        "crawl_matrix, unknown_words_crawl = build_matrix(tokenizer.word_index, CRAWL_EMBEDDING_PATH)\n",
        "print('n unknown words (crawl): ', len(unknown_words_crawl))\n",
        "\n",
        "glove_matrix, unknown_words_glove = build_matrix(tokenizer.word_index, GLOVE_EMBEDDING_PATH)\n",
        "print('n unknown words (glove): ', len(unknown_words_glove))\n",
        "\n",
        "max_features = max_features or len(tokenizer.word_index) + 1\n",
        "max_features\n",
        "\n",
        "embedding_matrix = np.concatenate([crawl_matrix, glove_matrix], axis=-1)\n",
        "embedding_matrix.shape\n",
        "\n",
        "del crawl_matrix\n",
        "del glove_matrix\n",
        "gc.collect()\n",
        "\n",
        "y_train_torch = torch.tensor(np.hstack([y_train, y_aux_train]), dtype=torch.float32)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r6xFJbqw3fSG",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "x_train = tokenizer.texts_to_sequences(x_train)\n",
        "x_test = tokenizer.texts_to_sequences(x_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jUb6ALXa3fSJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "lengths = torch.from_numpy(np.array([len(x) for x in x_train]))\n",
        " \n",
        "maxlen = 300\n",
        "x_train_padded = torch.from_numpy(sequence.pad_sequences(x_train, maxlen=maxlen))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RbM0mSld3fSO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "test_lengths = torch.from_numpy(np.array([len(x) for x in x_test]))\n",
        "\n",
        "x_test_padded = torch.from_numpy(sequence.pad_sequences(x_test, maxlen=maxlen))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kIYXZgum3fSX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "batch_size = 512\n",
        "test_dataset = data.TensorDataset(x_test_padded, test_lengths)\n",
        "train_dataset = data.TensorDataset(x_train_padded, lengths, y_train_torch)\n",
        "valid_dataset = data.Subset(train_dataset, indices=[0, 1])\n",
        "\n",
        "train_collator = SequenceBucketCollator(lambda lenghts: lenghts.max(), \n",
        "                                        sequence_index=0, \n",
        "                                        length_index=1, \n",
        "                                        label_index=2)\n",
        "test_collator = SequenceBucketCollator(lambda lenghts: lenghts.max(), sequence_index=0, length_index=1)\n",
        "\n",
        "train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=train_collator)\n",
        "valid_loader = data.DataLoader(valid_dataset, batch_size=batch_size, shuffle=False, collate_fn=train_collator)\n",
        "test_loader = data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, collate_fn=test_collator)\n",
        "\n",
        "databunch = DataBunch(train_dl=train_loader, valid_dl=valid_loader, collate_fn=train_collator)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4F2mcXGq3fSh",
        "colab_type": "code",
        "colab": {},
        "outputId": "a9e3fef0-a46e-4ecd-8655-e1e4a71601d7"
      },
      "source": [
        "## now for the most intsresting part : the actual work of the model\n",
        "all_test_preds = []\n",
        "\n",
        "for model_idx in range(NUM_MODELS):\n",
        "    print('Model ', model_idx)\n",
        "    seed_everything(1 + model_idx)\n",
        "    model = NeuralNet(embedding_matrix, y_aux_train.shape[-1])\n",
        "    learn = Learner(databunch, model, loss_func=custom_loss)\n",
        "    test_preds = train_model(learn,test_dataset,output_dim=7)    \n",
        "    all_test_preds.append(test_preds)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model  0\n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.282716    0.011497    10:30     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.265401    0.006181    10:29     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.263534    0.007372    10:32     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.258076    0.009170    10:32     \n",
            "Model  1\n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.276667    0.007452    10:32     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.270660    0.009762    10:31     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.261831    0.015536    10:31     \n",
            "epoch     train_loss  valid_loss  time    \n",
            "0         0.259937    0.013611    10:30     \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZAjGpqGj3fSz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission_lstm = pd.DataFrame.from_dict({\n",
        "    'id': test_df['id'],\n",
        "    'prediction': np.mean(all_test_preds, axis=0)[:, 0]\n",
        "})"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2XR6tzWF3fS3",
        "colab_type": "text"
      },
      "source": [
        "**let's belend our work and hope it will give us a good score in the private test data set**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q5GKcY503fS3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "submission = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/sample_submission.csv')\n",
        "submission['prediction'] = ((submission_bert.prediction + submission_lstm.prediction)/2 + submission_lstm.prediction)/2\n",
        "submission.to_csv('submission.csv', index=False)"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}