21.NLP BERT_vs_Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "m95an8slZmyq",
    "outputId": "e275aee7-e663-41d2-daa1-8fccaa2fc293"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting flair\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/68/a0/a1b41fa2fcb23ff71ba9148af75211dcccc35b256dea821b36e1ee871848/flair-0.7-py3-none-any.whl (448kB)\n",
      "\u001b[K     |████████████████████████████████| 450kB 8.5MB/s \n",
      "\u001b[?25hCollecting janome\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a8/63/98858cbead27df7536c7e300c169da0999e9704d02220dc6700b804eeff0/Janome-0.4.1-py2.py3-none-any.whl (19.7MB)\n",
      "\u001b[K     |████████████████████████████████| 19.7MB 1.2MB/s \n",
      "\u001b[?25hRequirement already satisfied: gensim<=3.8.3,>=3.4.0 in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.0)\n",
      "Collecting segtok>=1.5.7\n",
      "  Downloading https://files.pythonhosted.org/packages/41/08/582dab5f4b1d5ca23bc6927b4bb977c8ff7f3a87a3b98844ef833e2f5623/segtok-1.5.10.tar.gz\n",
      "Requirement already satisfied: lxml in /usr/local/lib/python3.6/dist-packages (from flair) (4.2.6)\n",
      "Collecting langdetect\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/a3/8407c1e62d5980188b4acc45ef3d94b933d14a2ebc9ef3505f22cf772570/langdetect-1.0.8.tar.gz (981kB)\n",
      "\u001b[K     |████████████████████████████████| 983kB 49.3MB/s \n",
      "\u001b[?25hCollecting sqlitedict>=1.6.0\n",
      "  Downloading https://files.pythonhosted.org/packages/5c/2d/b1d99e9ad157dd7de9cd0d36a8a5876b13b55e4b75f7498bc96035fb4e96/sqlitedict-1.7.0.tar.gz\n",
      "Requirement already satisfied: tabulate in /usr/local/lib/python3.6/dist-packages (from flair) (0.8.7)\n",
      "Collecting ftfy\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ff/e2/3b51c53dffb1e52d9210ebc01f1fb9f2f6eba9b3201fa971fd3946643c71/ftfy-5.8.tar.gz (64kB)\n",
      "\u001b[K     |████████████████████████████████| 71kB 12.0MB/s \n",
      "\u001b[?25hCollecting bpemb>=0.3.2\n",
      "  Downloading https://files.pythonhosted.org/packages/91/77/3f0f53856e86af32b1d3c86652815277f7b5f880002584eb30db115b6df5/bpemb-0.3.2-py3-none-any.whl\n",
      "Requirement already satisfied: torch>=1.1.0 in /usr/local/lib/python3.6/dist-packages (from flair) (1.7.0+cu101)\n",
      "Requirement already satisfied: python-dateutil>=2.6.1 in /usr/local/lib/python3.6/dist-packages (from flair) (2.8.1)\n",
      "Requirement already satisfied: matplotlib>=2.2.3 in /usr/local/lib/python3.6/dist-packages (from flair) (3.2.2)\n",
      "Collecting deprecated>=1.2.4\n",
      "  Downloading https://files.pythonhosted.org/packages/76/a1/05d7f62f956d77b23a640efc650f80ce24483aa2f85a09c03fb64f49e879/Deprecated-1.2.10-py2.py3-none-any.whl\n",
      "Requirement already satisfied: hyperopt>=0.1.1 in /usr/local/lib/python3.6/dist-packages (from flair) (0.1.2)\n",
      "Collecting konoha<5.0.0,>=4.0.0\n",
      "  Downloading https://files.pythonhosted.org/packages/ea/01/47358efec5396fc80f98273c42cbdfe7aab056252b07884ffcc0f118978f/konoha-4.6.2-py3-none-any.whl\n",
      "Requirement already satisfied: tqdm>=4.26.0 in /usr/local/lib/python3.6/dist-packages (from flair) (4.41.1)\n",
      "Collecting sentencepiece<=0.1.91\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1MB 40.0MB/s \n",
      "\u001b[?25hCollecting mpld3==0.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/91/95/a52d3a83d0a29ba0d6898f6727e9858fe7a43f6c2ce81a5fe7e05f0f4912/mpld3-0.3.tar.gz (788kB)\n",
      "\u001b[K     |████████████████████████████████| 798kB 47.0MB/s \n",
      "\u001b[?25hRequirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.6/dist-packages (from flair) (0.22.2.post1)\n",
      "Requirement already satisfied: gdown in /usr/local/lib/python3.6/dist-packages (from flair) (3.6.4)\n",
      "Collecting transformers<=3.5.1,>=3.5.0\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/3a/83/e74092e7f24a08d751aa59b37a9fc572b2e4af3918cb66f7766c3affb1b4/transformers-3.5.1-py3-none-any.whl (1.3MB)\n",
      "\u001b[K     |████████████████████████████████| 1.3MB 48.5MB/s \n",
      "\u001b[?25hRequirement already satisfied: regex in /usr/local/lib/python3.6/dist-packages (from flair) (2019.12.20)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.6/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (1.4.1)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.6/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (1.18.5)\n",
      "Requirement already satisfied: smart-open>=1.2.1 in /usr/local/lib/python3.6/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (4.0.1)\n",
      "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.6/dist-packages (from gensim<=3.8.3,>=3.4.0->flair) (1.15.0)\n",
      "Requirement already satisfied: wcwidth in /usr/local/lib/python3.6/dist-packages (from ftfy->flair) (0.2.5)\n",
      "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from bpemb>=0.3.2->flair) (2.23.0)\n",
      "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch>=1.1.0->flair) (0.8)\n",
      "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch>=1.1.0->flair) (3.7.4.3)\n",
      "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch>=1.1.0->flair) (0.16.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (1.3.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.6/dist-packages (from matplotlib>=2.2.3->flair) (2.4.7)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /usr/local/lib/python3.6/dist-packages (from deprecated>=1.2.4->flair) (1.12.1)\n",
      "Requirement already satisfied: networkx in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (2.5)\n",
      "Requirement already satisfied: pymongo in /usr/local/lib/python3.6/dist-packages (from hyperopt>=0.1.1->flair) (3.11.2)\n",
      "Collecting overrides==3.0.0\n",
      "  Downloading https://files.pythonhosted.org/packages/42/8d/caa729f809ecdf8e76fac3c1ff7d3f0b72c398c9dd8a6919927a30a873b3/overrides-3.0.0.tar.gz\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn>=0.21.3->flair) (0.17.0)\n",
      "Requirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers<=3.5.1,>=3.5.0->flair) (3.12.4)\n",
      "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers<=3.5.1,>=3.5.0->flair) (20.7)\n",
      "Collecting sacremoses\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
      "\u001b[K     |████████████████████████████████| 890kB 44.5MB/s \n",
      "\u001b[?25hCollecting tokenizers==0.9.3\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4c/34/b39eb9994bc3c999270b69c9eea40ecc6f0e97991dba28282b9fd32d44ee/tokenizers-0.9.3-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
      "\u001b[K     |████████████████████████████████| 2.9MB 47.8MB/s \n",
      "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers<=3.5.1,>=3.5.0->flair) (3.0.12)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.3.2->flair) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.3.2->flair) (2020.12.5)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.3.2->flair) (1.24.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->bpemb>=0.3.2->flair) (2.10)\n",
      "Requirement already satisfied: decorator>=4.3.0 in /usr/local/lib/python3.6/dist-packages (from networkx->hyperopt>=0.1.1->flair) (4.4.2)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers<=3.5.1,>=3.5.0->flair) (50.3.2)\n",
      "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers<=3.5.1,>=3.5.0->flair) (7.1.2)\n",
      "Building wheels for collected packages: segtok, langdetect, sqlitedict, ftfy, mpld3, overrides, sacremoses\n",
      "  Building wheel for segtok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for segtok: filename=segtok-1.5.10-cp36-none-any.whl size=25020 sha256=d45aede7a3bf99240da993cb427a3a7e3910b26cd75f4a5ee0328bdc426c4dad\n",
      "  Stored in directory: /root/.cache/pip/wheels/b4/39/f6/9ca1c5cabde964d728023b5751c3a206a5c8cc40252321fb6b\n",
      "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for langdetect: filename=langdetect-1.0.8-cp36-none-any.whl size=993193 sha256=49666bbd304eed0606ea7fc05428d8661572406d5875becbd0161003a41796b0\n",
      "  Stored in directory: /root/.cache/pip/wheels/8d/b3/aa/6d99de9f3841d7d3d40a60ea06e6d669e8e5012e6c8b947a57\n",
      "  Building wheel for sqlitedict (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sqlitedict: filename=sqlitedict-1.7.0-cp36-none-any.whl size=14377 sha256=69c73458eff747eee55f2b2d399593989a81431f940add0f64a47788d00c1e84\n",
      "  Stored in directory: /root/.cache/pip/wheels/cf/c6/4f/2c64a43f041415eb8b8740bd80e15e92f0d46c5e464d8e4b9b\n",
      "  Building wheel for ftfy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for ftfy: filename=ftfy-5.8-cp36-none-any.whl size=45613 sha256=3c0ae44299f0d02071b7a13d14c1a732b3071c9eade5a680654ab5f0b5d38a34\n",
      "  Stored in directory: /root/.cache/pip/wheels/ba/c0/ef/f28c4da5ac84a4e06ac256ca9182fc34fa57fefffdbc68425b\n",
      "  Building wheel for mpld3 (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for mpld3: filename=mpld3-0.3-cp36-none-any.whl size=116679 sha256=a98722baaf6a1dd0bdeb4c6e586f01dd844e861b739e60c25f0a09772f685faf\n",
      "  Stored in directory: /root/.cache/pip/wheels/c0/47/fb/8a64f89aecfe0059830479308ad42d62e898a3e3cefdf6ba28\n",
      "  Building wheel for overrides (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for overrides: filename=overrides-3.0.0-cp36-none-any.whl size=5669 sha256=96588c68dbc93a6fbe30b20728b8ccf962d1fb3af613fe1ad72a078db22085a2\n",
      "  Stored in directory: /root/.cache/pip/wheels/6f/1b/ec/6c71a1eb823df7f850d956b2d8c50a6d49c191e1063d73b9be\n",
      "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=4ceb500238771276b6d153f9697745ad59ac64653073b4810698e1c5baf2afef\n",
      "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
      "Successfully built segtok langdetect sqlitedict ftfy mpld3 overrides sacremoses\n",
      "Installing collected packages: janome, segtok, langdetect, sqlitedict, ftfy, sentencepiece, bpemb, deprecated, overrides, konoha, mpld3, sacremoses, tokenizers, transformers, flair\n",
      "Successfully installed bpemb-0.3.2 deprecated-1.2.10 flair-0.7 ftfy-5.8 janome-0.4.1 konoha-4.6.2 langdetect-1.0.8 mpld3-0.3 overrides-3.0.0 sacremoses-0.0.43 segtok-1.5.10 sentencepiece-0.1.91 sqlitedict-1.7.0 tokenizers-0.9.3 transformers-3.5.1\n"
     ]
    }
   ],
   "source": [
    "!pip install flair"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YCGdPHn5ZoIB"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from flair.embeddings import WordEmbeddings\n",
    "from flair.embeddings import TransformerWordEmbeddings\n",
    "from flair.data import Sentence\n",
    "from scipy.spatial import distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "yVc1ycsxb3vn",
    "outputId": "73cbabd4-12f8-4a27-a972-8ba48723e2b2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-15 22:05:00,800 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim.vectors.npy not found in cache, downloading to /tmp/tmpka44n4ov\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 160000128/160000128 [00:10<00:00, 15771281.86B/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-15 22:05:11,444 copying /tmp/tmpka44n4ov to cache at /root/.flair/embeddings/glove.gensim.vectors.npy\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-15 22:05:11,674 removing temp file /tmp/tmpka44n4ov\n",
      "2020-12-15 22:05:12,515 https://flair.informatik.hu-berlin.de/resources/embeddings/token/glove.gensim not found in cache, downloading to /tmp/tmpy06io5_4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 21494764/21494764 [00:02<00:00, 8906829.13B/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-15 22:05:15,434 copying /tmp/tmpy06io5_4 to cache at /root/.flair/embeddings/glove.gensim\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-12-15 22:05:15,455 removing temp file /tmp/tmpy06io5_4\n"
     ]
    }
   ],
   "source": [
    "# Glove Embeddings example\n",
    "\n",
    "glove_embedding = WordEmbeddings('glove')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gfWSAvGRZoPI"
   },
   "outputs": [],
   "source": [
    "sent1 = Sentence('apple released iphone 12 pro max in 2020 year')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oc4nMdLtcsa2",
    "outputId": "4945f2c4-356a-4709-fe5d-af9ac7320162"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Sentence: \"apple released iphone 12 pro max in 2020 year\"   [− Tokens: 9]]"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# converting into glove_embeddings\n",
    "glove_embedding.embed(sent1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "QAi7TK2tcmfG",
    "outputId": "88f68c6e-c325-4236-c20e-cfd737352905"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: 1 apple\n",
      "tensor([-0.5985, -0.4632,  0.1300, -0.0196,  0.4603, -0.3018,  0.8977, -0.6563,\n",
      "         0.6686, -0.4916,  0.0376, -0.0509,  0.6451, -0.5388, -0.3765, -0.0431,\n",
      "         0.5138,  0.1778,  0.2860,  0.9206, -0.4935, -0.4858,  0.6132,  0.7821,\n",
      "         0.1925,  0.9123, -0.0556, -0.1251, -0.6569,  0.0686,  0.5563,  1.6110,\n",
      "        -0.0074, -0.4888,  0.4549,  0.9610, -0.0634,  0.1743,  0.9814, -1.3125,\n",
      "        -0.1580, -0.5430, -0.1389, -0.2615, -0.3691,  0.2684, -0.2438, -0.1948,\n",
      "         0.6258, -0.7377,  0.3835, -0.7500, -0.3905,  0.0915, -0.3659, -1.4715,\n",
      "        -0.4523,  0.2256,  1.1412, -0.3853, -0.0672,  0.5729, -0.3919,  0.3130,\n",
      "        -0.2923, -0.9616,  0.1515, -0.2166,  0.2510,  0.0970,  0.2843,  1.4296,\n",
      "        -0.5056, -0.5137, -0.4722,  0.3204,  0.0231,  0.2262, -0.0972,  0.8213,\n",
      "         0.9260, -1.0086, -0.3864,  0.8641, -1.2060, -0.2853,  0.2265, -0.3877,\n",
      "         0.4088,  0.5930,  0.3077,  0.8380, -0.6366, -0.4464, -0.4341, -0.7936,\n",
      "        -0.2867, -0.0344,  1.3431,  0.3490])\n",
      "Token: 2 released\n",
      "tensor([-0.0451, -0.7260,  0.6271, -0.8506,  1.2913,  0.1556,  0.5372,  0.2967,\n",
      "         0.0272, -0.2156,  0.7911,  0.1594,  0.5834,  0.1422,  0.2127,  0.5273,\n",
      "         0.8467, -0.2552, -0.2266,  0.0687,  0.8454, -1.0389,  0.0572,  0.7512,\n",
      "        -0.4233,  0.3479,  0.3719, -0.0245,  0.4516,  1.0017,  0.7148,  0.0396,\n",
      "        -0.7914,  1.1237,  0.0218,  0.5682, -0.1066,  0.4893, -0.1981, -0.9888,\n",
      "        -0.2954,  0.2199, -0.1736,  1.0455,  0.3021, -0.5135, -0.4400, -0.8309,\n",
      "         0.4581, -0.4946,  0.3384, -0.4309,  0.0789,  0.8418, -0.3683, -2.4302,\n",
      "        -0.5795, -0.0524,  1.2644,  0.8816, -0.0311,  0.9024, -1.3107, -0.7962,\n",
      "         0.9831,  0.0515, -0.0143,  0.6118,  0.2169, -0.0897,  0.1589,  0.5384,\n",
      "        -0.3691,  0.0449, -0.1022,  0.9028, -0.0959, -0.0254, -0.9284, -0.5310,\n",
      "         0.4139, -0.0872, -0.0959,  0.5326, -1.1456, -0.2831,  0.6678, -0.5193,\n",
      "        -0.3507,  0.6526, -0.1945, -0.3041, -0.6515,  0.2619, -0.0246,  0.2302,\n",
      "        -0.3672,  0.1771,  0.5437, -0.9623])\n",
      "Token: 3 iphone\n",
      "tensor([-0.3609, -0.5975,  0.5235,  0.2344,  0.3229, -0.5134,  0.4237, -0.5790,\n",
      "         0.6805,  0.3536,  0.6597, -0.0775,  0.0842, -0.7555,  0.5229,  0.0685,\n",
      "        -0.3928,  1.1090,  0.7667,  0.6545, -0.0071, -1.1910,  0.8565,  1.1666,\n",
      "         1.1307,  0.9571,  0.1755,  0.2592, -0.6803, -0.1571,  0.3787,  1.7457,\n",
      "         0.1398,  0.3117,  1.1883,  0.9100, -0.9956, -0.2546,  0.6254, -0.6041,\n",
      "         0.5139, -0.1973,  0.3690, -0.0774, -0.6047,  0.0814,  0.5492, -0.2851,\n",
      "         0.8859,  0.2887,  0.1664,  0.3513,  0.2691, -0.2735, -0.3812, -0.5627,\n",
      "        -0.7775, -0.2434,  0.8626, -0.2992, -0.1599,  0.5551, -0.3766,  0.3375,\n",
      "         0.0084,  0.3258, -0.2169, -0.4835,  0.4285, -0.6070,  0.7929,  1.0375,\n",
      "        -0.6710, -0.9116, -0.4302,  0.7993,  0.0236, -0.0459, -0.2122, -0.4890,\n",
      "         0.4448, -1.3734,  0.2017,  0.5503, -1.0103, -0.0994,  0.4866, -0.0043,\n",
      "        -0.4854,  0.1720,  0.2872,  1.1088,  0.4544, -0.8195,  0.4331, -0.3293,\n",
      "        -0.3243,  0.2213,  1.4631,  0.2227])\n",
      "Token: 4 12\n",
      "tensor([ 0.0730,  0.1813,  0.0204, -0.2854,  0.1390,  0.1947,  0.7946,  0.5717,\n",
      "        -0.7715,  0.0392,  1.1718, -0.2051, -0.2833,  0.3860,  0.6874,  0.0520,\n",
      "        -0.0109,  0.1280, -1.0185,  0.7395,  0.7109, -0.1611,  0.5491,  0.6137,\n",
      "         0.3223, -0.4123,  0.3328, -0.0841,  0.2424,  0.3612,  0.0668,  0.1613,\n",
      "         0.1812, -0.7899, -0.4400, -0.0025,  0.0969,  0.3055, -0.4306,  0.6801,\n",
      "        -0.0809, -0.7408,  0.7272, -0.3813,  0.3725, -0.4571, -0.3741, -0.8136,\n",
      "        -0.0643, -0.1498, -0.6084, -0.7011, -0.3335,  0.9525, -0.5965, -2.3970,\n",
      "        -0.1315, -0.6259,  1.7407,  0.9491, -0.5584,  0.7337, -0.6383,  0.1429,\n",
      "         0.0523,  0.5167, -0.4722,  0.4084,  0.1880,  0.3140, -0.2562,  0.0069,\n",
      "        -0.0765,  0.5165, -0.0218,  0.1769,  0.1347,  0.1100, -0.7226, -0.3386,\n",
      "         0.7269, -0.0877, -0.3665,  0.1536, -0.4635, -0.4650,  0.3513, -0.1610,\n",
      "         0.2442,  0.5036, -0.1055,  0.4940, -0.2373, -0.1125, -1.2820,  0.4075,\n",
      "        -0.3709, -0.0305,  0.4792, -0.5814])\n",
      "Token: 5 pro\n",
      "tensor([-0.2332,  0.1988, -0.2101, -0.5201,  0.0146,  1.0865,  0.3593, -0.5337,\n",
      "        -0.2569,  0.4887, -0.0698,  0.0261,  0.5549, -0.4796, -0.4538,  0.3965,\n",
      "         0.0355, -1.0047, -0.1908,  0.4501,  0.6563,  0.0932,  0.0396,  0.1032,\n",
      "         0.2977, -0.2558, -0.3840, -0.7571,  0.9946,  0.9950,  0.2262,  1.1416,\n",
      "         0.0645,  0.2327,  0.7881, -0.2046, -0.0235,  0.6137, -0.7019, -0.2946,\n",
      "        -0.6030, -0.2245,  1.0954, -0.4636, -0.3176, -0.9000,  0.2299, -0.3905,\n",
      "        -0.0554, -0.5276, -0.3356,  0.1659, -0.2207,  0.0882,  0.5056, -1.6934,\n",
      "         0.5285,  0.6024,  1.6912,  0.3494, -0.7740, -0.5321, -0.5961, -0.0379,\n",
      "         0.2280,  0.1600,  0.5961,  0.0467,  0.0882,  0.5184,  1.0067, -0.2359,\n",
      "        -0.8654,  0.0373, -0.8648, -0.3432,  0.3989,  0.5280, -0.3100, -0.5621,\n",
      "         1.2702, -0.5630, -0.1432, -0.3489, -0.3617,  0.0104, -0.1854,  0.2372,\n",
      "         0.5165, -0.3110,  0.1728,  0.3704, -0.8039,  0.9957, -0.3425, -0.3929,\n",
      "        -0.7508,  0.7523,  1.3898,  0.4525])\n",
      "Token: 6 max\n",
      "tensor([ 0.3356,  0.1247,  0.2099, -0.0868,  0.2193, -0.6803,  0.0619,  0.2419,\n",
      "        -0.5426, -0.5727,  0.2655, -0.1465, -1.0335, -0.1684, -0.0746,  0.1714,\n",
      "         0.6827, -0.1196, -0.4984,  0.0190,  0.2460,  0.1463,  0.8977,  0.1802,\n",
      "         0.2900,  0.1599,  0.1820, -0.3310,  0.6753,  0.5950, -0.3450,  0.6775,\n",
      "        -0.3929,  0.1636, -0.4901,  0.2833,  0.1133,  0.0250,  0.1972,  0.2408,\n",
      "         0.2869, -0.1621, -0.1463,  0.0591, -1.0204,  0.1693, -0.3522, -0.0614,\n",
      "         0.3289,  0.8992,  0.4285, -0.0598, -0.3730,  0.4329,  0.3592, -1.8269,\n",
      "        -0.4295,  0.9414, -0.3002,  0.2843,  1.0934,  0.2972, -0.6670,  0.3720,\n",
      "         1.3223, -0.1351,  0.4574,  0.8587,  0.7930,  0.4599,  0.1624, -0.8505,\n",
      "        -0.4849,  0.1712,  0.2690,  0.3976,  0.7205, -0.9071,  0.2739,  0.2051,\n",
      "        -0.0022, -0.1362, -0.3767, -0.5283, -0.5826,  0.2282, -0.2979, -0.2571,\n",
      "        -0.2076, -0.4404,  0.5630, -0.1047,  0.6858,  0.6947,  0.1091, -0.2399,\n",
      "         0.0042, -0.0966,  0.2621, -0.8044])\n",
      "Token: 7 in\n",
      "tensor([ 0.0857, -0.2220,  0.1657,  0.1337,  0.3824,  0.3540,  0.0129,  0.2246,\n",
      "        -0.4382,  0.5016, -0.3587, -0.3498,  0.0552,  0.6965, -0.1796,  0.0679,\n",
      "         0.3910,  0.1604, -0.2664, -0.2114,  0.5370,  0.4938,  0.9366,  0.6690,\n",
      "         0.2179, -0.4664,  0.2238, -0.3620, -0.1766,  0.1748, -0.2037,  0.1393,\n",
      "         0.0198, -0.1041, -0.2024,  0.5500, -0.1546,  0.9865, -0.2686, -0.2909,\n",
      "        -0.3287, -0.3419, -0.1694, -0.4200, -0.0467, -0.1633,  0.7082, -0.7491,\n",
      "        -0.0916, -0.9618, -0.1975,  0.1028,  0.5522,  1.3816, -0.6564, -3.2502,\n",
      "        -0.3156, -1.2055,  1.7709,  0.4026, -0.7983,  1.1597, -0.3304,  0.3138,\n",
      "         0.7739,  0.2260,  0.5247, -0.0341,  0.3205,  0.0799,  0.1775, -0.4943,\n",
      "        -0.7005, -0.4457,  0.1724,  0.2028,  0.0233, -0.2068, -1.0158,  0.1832,\n",
      "         0.5675,  0.3182, -0.6501,  0.6828, -0.8658, -0.0594, -0.2926, -0.5567,\n",
      "        -0.3471, -0.3289,  0.4022, -0.1275, -0.2023,  0.8737, -0.5450,  0.7921,\n",
      "        -0.2069, -0.0743,  0.7581, -0.3424])\n",
      "Token: 8 2020\n",
      "tensor([-0.1241,  0.3447,  0.9852, -0.1853,  0.3933, -1.3096, -0.0987,  0.6483,\n",
      "         0.1348,  1.0135,  0.4145, -0.4029,  0.1575, -1.1746,  0.3356, -1.6206,\n",
      "        -0.4004,  1.0787, -0.4370, -1.0824, -0.4016, -0.3710, -0.4385,  0.8210,\n",
      "        -0.1609,  0.4084,  0.1666, -0.0382,  0.4597, -0.0112, -0.5954,  0.4539,\n",
      "        -0.2334, -0.8493,  0.4425,  0.2576,  0.0797, -0.2632, -0.8782,  0.1784,\n",
      "        -0.5862,  0.0843, -0.3571,  0.9713, -0.0594,  0.0907,  0.1479, -0.6011,\n",
      "        -1.4061, -0.1069, -0.6418, -0.6196, -0.6186,  0.8040, -0.5495, -1.1144,\n",
      "         0.1015,  0.2140,  0.5193, -0.5009, -0.1563,  0.1541, -0.6118, -0.4091,\n",
      "         0.2161,  0.3469, -1.0338, -0.7153,  1.1187, -1.1841,  0.8466, -0.3190,\n",
      "        -0.4022,  0.0755, -0.4686, -0.5858,  0.6357, -0.5537, -0.7745, -0.5577,\n",
      "         0.4393,  0.4853, -0.3689, -0.3911,  0.0748,  0.6911,  0.4363, -0.1206,\n",
      "        -0.1954,  0.8056,  0.5050,  0.2136,  0.1071, -0.5304, -1.1339,  0.2752,\n",
      "         0.1600,  0.0397, -0.0084, -1.0729])\n",
      "Token: 9 year\n",
      "tensor([ 4.4234e-01,  4.8431e-01,  3.7284e-01, -5.2861e-01,  2.1558e-01,\n",
      "        -4.6290e-01,  6.0307e-01,  7.1816e-01, -4.1382e-01,  1.5157e-01,\n",
      "        -2.9360e-02, -1.2905e-02,  2.4569e-01, -2.6380e-02,  2.8137e-04,\n",
      "        -4.1139e-01,  1.6784e-01, -9.4465e-02, -3.9723e-01,  2.5608e-01,\n",
      "         3.9144e-01, -1.4293e-01,  1.5430e-01,  1.1509e+00,  5.1744e-01,\n",
      "        -4.2260e-01, -7.9870e-02, -7.7108e-01,  7.8489e-02, -4.9301e-01,\n",
      "        -1.4824e-01,  1.6366e-01,  2.3525e-01, -3.2219e-01, -5.0613e-01,\n",
      "         6.8354e-01, -3.9673e-01,  4.8081e-01, -5.9104e-01, -1.9323e-01,\n",
      "        -2.8618e-01, -1.6291e-01,  4.8853e-01,  3.7437e-02,  3.9265e-02,\n",
      "         2.5850e-02,  4.7258e-01, -1.3542e+00,  2.6895e-02, -1.2436e+00,\n",
      "        -1.3284e-01, -9.8070e-01,  3.9823e-01,  1.1715e+00, -4.9008e-01,\n",
      "        -2.8411e+00, -1.7090e-01, -2.1840e-01,  2.0011e+00,  5.9575e-01,\n",
      "        -3.6505e-01,  7.8590e-01, -4.5694e-01,  7.0966e-02,  1.8904e-01,\n",
      "         2.2404e-01,  6.0091e-02,  5.0271e-02,  8.2711e-01, -2.4997e-01,\n",
      "         4.6051e-01, -1.1298e-01, -1.0912e+00,  1.4487e-01, -8.3451e-01,\n",
      "         1.2452e-01, -2.6793e-01,  2.7680e-02, -1.4872e+00, -1.8120e-01,\n",
      "         8.8620e-01, -5.3061e-02, -4.9827e-01,  3.1571e-01, -1.1596e+00,\n",
      "        -1.8781e-01,  1.5968e-01, -5.7221e-01, -4.7602e-02, -3.2081e-01,\n",
      "        -7.0405e-02,  3.6944e-01, -4.2656e-01, -8.0083e-02, -8.7588e-01,\n",
      "         2.8568e-01,  9.9353e-02,  3.8253e-01,  4.7456e-01, -6.5050e-01])\n"
     ]
    }
   ],
   "source": [
    "# showing the embedding for each and every word in the sentence\n",
    "for token in sent1:\n",
    "  print(token)\n",
    "  print(token.embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "gmRy-50SZoUw",
    "outputId": "ffcfa44b-643a-454d-d627-9b7ec40c2c74"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: 1 apple\n",
      "tensor([-0.5985, -0.4632,  0.1300, -0.0196,  0.4603, -0.3018,  0.8977, -0.6563,\n",
      "         0.6686, -0.4916,  0.0376, -0.0509,  0.6451, -0.5388, -0.3765, -0.0431,\n",
      "         0.5138,  0.1778,  0.2860,  0.9206, -0.4935, -0.4858,  0.6132,  0.7821,\n",
      "         0.1925,  0.9123, -0.0556, -0.1251, -0.6569,  0.0686,  0.5563,  1.6110,\n",
      "        -0.0074, -0.4888,  0.4549,  0.9610, -0.0634,  0.1743,  0.9814, -1.3125,\n",
      "        -0.1580, -0.5430, -0.1389, -0.2615, -0.3691,  0.2684, -0.2438, -0.1948,\n",
      "         0.6258, -0.7377,  0.3835, -0.7500, -0.3905,  0.0915, -0.3659, -1.4715,\n",
      "        -0.4523,  0.2256,  1.1412, -0.3853, -0.0672,  0.5729, -0.3919,  0.3130,\n",
      "        -0.2923, -0.9616,  0.1515, -0.2166,  0.2510,  0.0970,  0.2843,  1.4296,\n",
      "        -0.5056, -0.5137, -0.4722,  0.3204,  0.0231,  0.2262, -0.0972,  0.8213,\n",
      "         0.9260, -1.0086, -0.3864,  0.8641, -1.2060, -0.2853,  0.2265, -0.3877,\n",
      "         0.4088,  0.5930,  0.3077,  0.8380, -0.6366, -0.4464, -0.4341, -0.7936,\n",
      "        -0.2867, -0.0344,  1.3431,  0.3490])\n"
     ]
    }
   ],
   "source": [
    "print(sent1[0])\n",
    "print(sent1[0].embedding) # first word ..all represent 100 dimensional vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2s8NEyGEZoao"
   },
   "outputs": [],
   "source": [
    "sent2 = Sentence('an apple a day keeps a doctor away')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "p_7IRncMduO8",
    "outputId": "a7204690-bdb3-4323-befc-c27fd0bf048b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Sentence: \"an apple a day keeps a doctor away\"   [− Tokens: 8]]"
      ]
     },
     "execution_count": 22,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# converting into glove_embeddings\n",
    "glove_embedding.embed(sent2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "vkUY3x13d-9e",
    "outputId": "2c746988-2b4d-4057-e0ff-91b19c60afa7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: 1 an\n",
      "tensor([-0.4214, -0.1880,  0.4624, -0.1761,  0.3621,  0.3670,  0.2792,  0.1463,\n",
      "        -0.0542,  0.4583,  0.0654, -0.3372,  0.0675, -0.3632,  0.5030, -0.0104,\n",
      "         0.7283, -0.1756, -0.3400,  0.0729,  0.6448, -0.2391,  0.3838,  0.1386,\n",
      "         1.0994, -0.2488, -0.1508, -0.4874, -0.2304,  0.0648, -0.7018,  0.8265,\n",
      "         0.0613,  0.1853, -0.3016, -0.0222,  0.3430,  0.8033,  0.1714,  0.1546,\n",
      "        -0.5076,  0.3957,  0.0543, -0.5308,  0.4825,  0.0862,  0.5958, -0.2238,\n",
      "        -0.3955, -0.7304, -0.1028, -0.3917,  1.2290,  1.2129, -1.0365, -3.4971,\n",
      "         0.1092, -1.0084,  1.9998,  0.7964,  0.3881,  0.4375,  0.0852,  0.3855,\n",
      "         0.6199, -1.0320,  0.7012, -0.2246,  0.0794,  0.0913, -0.2120, -0.5543,\n",
      "        -0.0534, -0.8020,  0.4680, -0.0501, -0.5742, -0.0848, -1.7227, -0.9429,\n",
      "         0.9867,  0.3121, -0.3774,  0.0687, -0.7784, -0.2849,  0.8105,  0.4660,\n",
      "        -0.1186, -0.9341,  0.3372,  0.0379, -0.1827, -0.0199,  0.2049, -0.4772,\n",
      "        -0.4925, -0.5652,  0.7256, -0.1591])\n",
      "\n",
      "\n",
      "Token: 2 apple\n",
      "tensor([-0.5985, -0.4632,  0.1300, -0.0196,  0.4603, -0.3018,  0.8977, -0.6563,\n",
      "         0.6686, -0.4916,  0.0376, -0.0509,  0.6451, -0.5388, -0.3765, -0.0431,\n",
      "         0.5138,  0.1778,  0.2860,  0.9206, -0.4935, -0.4858,  0.6132,  0.7821,\n",
      "         0.1925,  0.9123, -0.0556, -0.1251, -0.6569,  0.0686,  0.5563,  1.6110,\n",
      "        -0.0074, -0.4888,  0.4549,  0.9610, -0.0634,  0.1743,  0.9814, -1.3125,\n",
      "        -0.1580, -0.5430, -0.1389, -0.2615, -0.3691,  0.2684, -0.2438, -0.1948,\n",
      "         0.6258, -0.7377,  0.3835, -0.7500, -0.3905,  0.0915, -0.3659, -1.4715,\n",
      "        -0.4523,  0.2256,  1.1412, -0.3853, -0.0672,  0.5729, -0.3919,  0.3130,\n",
      "        -0.2923, -0.9616,  0.1515, -0.2166,  0.2510,  0.0970,  0.2843,  1.4296,\n",
      "        -0.5056, -0.5137, -0.4722,  0.3204,  0.0231,  0.2262, -0.0972,  0.8213,\n",
      "         0.9260, -1.0086, -0.3864,  0.8641, -1.2060, -0.2853,  0.2265, -0.3877,\n",
      "         0.4088,  0.5930,  0.3077,  0.8380, -0.6366, -0.4464, -0.4341, -0.7936,\n",
      "        -0.2867, -0.0344,  1.3431,  0.3490])\n",
      "\n",
      "\n",
      "Token: 3 a\n",
      "tensor([-0.2709,  0.0440, -0.0203, -0.1740,  0.6444,  0.7121,  0.3551,  0.4714,\n",
      "        -0.2964,  0.5443, -0.7229, -0.0048,  0.0406,  0.0432,  0.2973,  0.1072,\n",
      "         0.4016, -0.5366,  0.0334,  0.0674,  0.6456, -0.0855,  0.1410,  0.0945,\n",
      "         0.7495, -0.1940, -0.6874, -0.4174, -0.2281,  0.1200, -0.4900,  0.8094,\n",
      "         0.0451, -0.1190,  0.2016,  0.3928, -0.2012,  0.3135,  0.7530,  0.2591,\n",
      "        -0.1157, -0.0293,  0.9350, -0.3607,  0.5242,  0.2371,  0.5271,  0.2287,\n",
      "        -0.5196, -0.7935, -0.2037, -0.5019,  0.1875,  0.9428, -0.4483, -3.6792,\n",
      "         0.0442, -0.2675,  2.1997,  0.2410, -0.0334,  0.6955, -0.6447, -0.0072,\n",
      "         0.8957,  0.2001,  0.4649,  0.6193, -0.1066,  0.0869, -0.4623,  0.1826,\n",
      "        -0.1585,  0.0208,  0.1937,  0.0634, -0.3167, -0.4818, -1.3848,  0.1367,\n",
      "         0.9686,  0.0500, -0.2738, -0.0357, -1.0577, -0.2447,  0.9037, -0.1244,\n",
      "         0.0808, -0.8340,  0.5720,  0.0889, -0.4253, -0.0183, -0.0800, -0.2858,\n",
      "        -0.0109, -0.4923,  0.6369,  0.2364])\n",
      "\n",
      "\n",
      "Token: 4 day\n",
      "tensor([-3.6689e-01,  4.1540e-01,  1.3478e-01, -1.7841e-01, -4.0050e-01,\n",
      "         2.0647e-01,  4.2742e-01,  1.1290e+00, -8.3981e-01, -4.7955e-01,\n",
      "         2.1115e-02, -1.8319e-01,  2.4588e-01, -2.9048e-02,  2.9683e-01,\n",
      "        -4.4613e-01,  1.7493e-01, -1.6619e-01, -4.9573e-01, -2.6547e-02,\n",
      "         7.6369e-01, -2.0810e-01,  2.6494e-01,  3.3085e-01,  5.7039e-01,\n",
      "         1.2832e-01, -2.8324e-01,  7.8652e-02,  3.8330e-01, -7.4012e-02,\n",
      "        -5.6725e-01, -2.5826e-01,  1.4013e-01,  1.1077e-01, -2.1629e-01,\n",
      "         4.0637e-01, -4.5740e-01,  2.7936e-01,  1.6717e-01, -1.6388e-01,\n",
      "        -6.9184e-01, -3.1493e-01,  6.5592e-01, -2.2426e-03, -1.6975e-02,\n",
      "         1.0777e-01, -1.8928e-01, -7.7079e-01,  2.4333e-01, -1.1467e+00,\n",
      "         1.2931e-01, -5.9668e-01,  5.2350e-01,  1.0476e+00, -8.3517e-01,\n",
      "        -2.6268e+00, -3.2837e-01,  2.4767e-01,  2.1535e+00,  6.0561e-01,\n",
      "        -5.4019e-01,  9.9737e-01, -1.8484e-01, -4.7044e-01, -1.9265e-02,\n",
      "         3.4195e-01,  3.1006e-01, -3.1499e-01,  2.9990e-01, -3.7647e-01,\n",
      "        -1.0041e-01, -4.8373e-01, -3.8982e-01, -3.4424e-01, -1.2195e-01,\n",
      "         5.8945e-01, -1.5675e-01, -3.3774e-01, -1.0446e+00, -3.7927e-02,\n",
      "         3.8237e-01, -6.3733e-02, -1.2990e-01,  1.2899e-01, -1.4625e+00,\n",
      "        -2.2875e-01, -3.6410e-01, -1.0337e-01,  2.3737e-01, -2.3498e-01,\n",
      "         1.5801e-01,  3.8339e-02,  1.3424e-02,  3.2535e-01, -1.1755e+00,\n",
      "         4.9695e-02,  5.7227e-01,  2.4380e-02,  2.2106e-01,  4.3169e-01])\n",
      "\n",
      "\n",
      "Token: 5 keeps\n",
      "tensor([-2.8212e-01,  8.1407e-02,  1.0041e+00, -2.1619e-01,  3.8559e-01,\n",
      "         5.8096e-01, -7.4637e-01,  9.6754e-02,  1.3090e-01, -5.7634e-01,\n",
      "         5.7288e-01, -1.5900e-01,  1.3582e-01,  2.7628e-01, -4.4655e-01,\n",
      "         1.5342e-01, -4.1672e-01, -2.2789e-01,  1.1763e+00,  4.3135e-04,\n",
      "        -2.0838e-01,  4.4750e-01, -1.4259e-01, -4.7203e-01,  2.8749e-01,\n",
      "         1.8887e-01, -8.8928e-01, -8.2278e-01, -3.1343e-01, -5.6226e-02,\n",
      "        -3.2358e-01,  4.7065e-01,  1.2294e-01,  1.3080e-01,  6.5288e-02,\n",
      "        -3.1797e-01, -2.3859e-01, -4.7632e-01, -2.3364e-02, -4.4278e-01,\n",
      "        -2.0717e-01, -4.5140e-01,  4.4082e-01, -3.7247e-01,  6.3666e-02,\n",
      "        -2.6637e-01, -3.7114e-01, -1.4971e-01,  2.7836e-01, -8.9750e-01,\n",
      "         7.6634e-01, -6.3621e-01, -7.2737e-02,  1.0848e+00,  2.2928e-01,\n",
      "        -1.6516e+00,  8.1383e-02,  5.1793e-02,  7.1174e-01,  2.2299e-01,\n",
      "         8.2580e-01,  3.6125e-02,  2.4621e-01, -1.3305e-01,  9.9692e-02,\n",
      "         7.9609e-01,  5.6096e-01,  1.4129e-01,  2.1741e-01,  2.1998e-01,\n",
      "         4.8408e-01, -9.3512e-02, -1.1885e-01, -4.0587e-02,  6.3986e-01,\n",
      "         2.3246e-01,  2.8093e-01, -2.3691e-01, -2.3029e-01,  2.4496e-01,\n",
      "         1.1252e-01,  2.6858e-02, -1.0912e+00, -8.9866e-02, -5.8186e-01,\n",
      "        -2.7347e-02, -2.6925e-01,  2.2656e-01, -6.4862e-03,  1.1973e-01,\n",
      "        -1.6618e-01,  1.5777e-01,  7.2022e-01, -7.2013e-01, -2.7190e-01,\n",
      "        -2.3454e-01,  1.6691e-01, -2.4730e-01,  3.4683e-01,  8.1010e-01])\n",
      "\n",
      "\n",
      "Token: 6 a\n",
      "tensor([-0.2709,  0.0440, -0.0203, -0.1740,  0.6444,  0.7121,  0.3551,  0.4714,\n",
      "        -0.2964,  0.5443, -0.7229, -0.0048,  0.0406,  0.0432,  0.2973,  0.1072,\n",
      "         0.4016, -0.5366,  0.0334,  0.0674,  0.6456, -0.0855,  0.1410,  0.0945,\n",
      "         0.7495, -0.1940, -0.6874, -0.4174, -0.2281,  0.1200, -0.4900,  0.8094,\n",
      "         0.0451, -0.1190,  0.2016,  0.3928, -0.2012,  0.3135,  0.7530,  0.2591,\n",
      "        -0.1157, -0.0293,  0.9350, -0.3607,  0.5242,  0.2371,  0.5271,  0.2287,\n",
      "        -0.5196, -0.7935, -0.2037, -0.5019,  0.1875,  0.9428, -0.4483, -3.6792,\n",
      "         0.0442, -0.2675,  2.1997,  0.2410, -0.0334,  0.6955, -0.6447, -0.0072,\n",
      "         0.8957,  0.2001,  0.4649,  0.6193, -0.1066,  0.0869, -0.4623,  0.1826,\n",
      "        -0.1585,  0.0208,  0.1937,  0.0634, -0.3167, -0.4818, -1.3848,  0.1367,\n",
      "         0.9686,  0.0500, -0.2738, -0.0357, -1.0577, -0.2447,  0.9037, -0.1244,\n",
      "         0.0808, -0.8340,  0.5720,  0.0889, -0.4253, -0.0183, -0.0800, -0.2858,\n",
      "        -0.0109, -0.4923,  0.6369,  0.2364])\n",
      "\n",
      "\n",
      "Token: 7 doctor\n",
      "tensor([ 0.0432, -0.4753,  0.1581,  0.2041, -0.1538,  0.7228,  0.2614,  0.2089,\n",
      "        -0.3147, -0.0703, -0.4337,  0.0531,  0.7363,  0.9811,  0.2353, -0.1045,\n",
      "         0.5026, -0.0334, -0.3554,  0.6455, -0.3710, -0.1005, -0.7693, -0.1696,\n",
      "        -0.1565,  0.5355,  0.3515, -1.5126,  0.0510,  0.2445, -0.3569,  0.4397,\n",
      "        -0.6298,  0.3289, -0.5301,  0.4983, -1.2061,  0.2780,  0.4273,  0.0958,\n",
      "        -0.4353,  0.9356,  0.3604, -0.8311,  0.1297, -0.1363, -0.5812,  0.0929,\n",
      "        -0.0147,  0.3256,  0.4120,  0.1451,  0.4980,  0.8693, -0.1803, -1.6227,\n",
      "        -0.6457,  0.1750,  0.7385,  0.3916,  0.8314,  0.5131,  0.1300, -0.2129,\n",
      "         0.6846,  0.0563,  0.0908,  0.2803, -0.1223,  0.6076, -0.5791, -0.0241,\n",
      "        -0.0633,  0.4075,  0.1077,  0.5798,  0.0928, -0.1559, -0.3649, -0.4663,\n",
      "         0.3755,  0.1640, -0.7514, -0.1083, -2.1363, -0.1166,  0.2316,  0.2442,\n",
      "        -0.3635, -0.3493,  0.6000, -0.1053,  0.7861,  0.4571, -0.1712,  0.6362,\n",
      "         0.6862, -0.2840, -0.2977, -0.8115])\n",
      "\n",
      "\n",
      "Token: 8 away\n",
      "tensor([-0.1038, -0.0148,  0.5993, -0.5132, -0.0365,  0.6588, -0.5791,  0.1782,\n",
      "         0.2366, -0.2138,  0.5534,  0.5360,  0.0414,  0.1610,  0.0171, -0.3724,\n",
      "         0.0180,  0.3927, -0.2326,  0.1818,  0.6640,  0.9816,  0.4234,  0.0306,\n",
      "         0.3501,  0.2552, -0.7118, -0.4218,  0.1307, -0.4745, -0.0817,  0.1574,\n",
      "        -0.1326,  0.2268, -0.1689, -0.1112, -0.3227, -0.0210, -0.4335,  0.1720,\n",
      "        -0.6737, -0.7905,  0.1056, -0.4219, -0.1239, -0.0635, -0.1784,  0.5636,\n",
      "         0.1699, -0.1780,  0.1396, -0.2017,  0.0790,  1.4497,  0.2356, -2.6014,\n",
      "        -0.5286, -0.1164,  1.7184,  0.3325,  0.1214,  1.1602, -0.2914,  0.4712,\n",
      "         0.4187,  0.3527,  0.4787, -0.0423, -0.1829,  0.1796, -0.2443, -0.3404,\n",
      "         0.2034, -0.9368,  0.0131,  0.0803, -0.3660, -0.4401, -0.3539,  0.1591,\n",
      "         0.5581,  0.1492, -0.8643,  0.0403, -1.0939, -0.2639, -0.2949,  0.2570,\n",
      "        -0.3372, -0.0865, -0.2425, -0.2111,  0.0996,  0.1282, -0.7871, -0.5178,\n",
      "        -0.1094,  0.9763,  0.5703,  0.1358])\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in sent2:\n",
    "  print(i)\n",
    "  print(i.embedding)\n",
    "  print('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rFMj3Kmjdq7c",
    "outputId": "bc386f47-f596-4e7b-bd86-fcded6aadf18"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([100]), torch.Size([100]))"
      ]
     },
     "execution_count": 27,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1[0].embedding.shape , sent2[1].embedding.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "PRL4WLeGdfXl",
    "outputId": "9f6574d3-ad77-4bd9-9eab-e13a1487eb68"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "# Glove distance between same word\n",
    "\n",
    "glove_dst = distance.euclidean(np.array(sent1[0].embedding),\n",
    "                               np.array(sent2[1].embedding))\n",
    "print(glove_dst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "id": "pqmI45itdfaM"
   },
   "outputs": [],
   "source": [
    "# Bert Embeddings\n",
    "\n",
    "bert_embedding = TransformerWordEmbeddings('bert-base-multilingual-cased')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "BL89B10LfZw5",
    "outputId": "56f80f62-473b-466c-d6cc-d64528843dfd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: 1 apple\n",
      "tensor([-0.5985, -0.4632,  0.1300,  ...,  1.3401, -0.0510, -0.0091])\n",
      "Token: 2 released\n",
      "tensor([-0.0451, -0.7260,  0.6271,  ...,  0.7689,  0.5975,  0.2122])\n",
      "Token: 3 iphone\n",
      "tensor([-0.3609, -0.5975,  0.5235,  ...,  0.9507,  0.2802, -0.9523])\n",
      "Token: 4 12\n",
      "tensor([ 0.0730,  0.1813,  0.0204,  ...,  1.0470,  0.4467, -0.1351])\n",
      "Token: 5 pro\n",
      "tensor([-0.2332,  0.1988, -0.2101,  ...,  2.0522,  0.7840, -0.2634])\n",
      "Token: 6 max\n",
      "tensor([0.3356, 0.1247, 0.2099,  ..., 1.5115, 0.1240, 0.2920])\n",
      "Token: 7 in\n",
      "tensor([ 0.0857, -0.2220,  0.1657,  ...,  1.5904, -0.2034, -0.2649])\n",
      "Token: 8 2020\n",
      "tensor([-0.1241,  0.3447,  0.9852,  ...,  1.2510,  0.8323, -0.1932])\n",
      "Token: 9 year\n",
      "tensor([0.4423, 0.4843, 0.3728,  ..., 1.8454, 0.6008, 0.0384])\n"
     ]
    }
   ],
   "source": [
    "bert_embedding.embed(sent1)\n",
    "for i in sent1:\n",
    "  print(i)\n",
    "  print(i.embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "jjWhNFoMdfdV",
    "outputId": "37c24eb9-a766-4a0e-e466-3e115aa0ab57"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Token: 1 an\n",
      "tensor([-0.4214, -0.1880,  0.4624,  ...,  1.3395,  0.8341,  0.1065])\n",
      "Token: 2 apple\n",
      "tensor([-0.5985, -0.4632,  0.1300,  ...,  1.7328,  1.1016,  0.1832])\n",
      "Token: 3 a\n",
      "tensor([-0.2709,  0.0440, -0.0203,  ...,  1.0097, -0.4214,  0.6264])\n",
      "Token: 4 day\n",
      "tensor([-0.3669,  0.4154,  0.1348,  ...,  0.9116,  0.1781,  0.1328])\n",
      "Token: 5 keeps\n",
      "tensor([-0.2821,  0.0814,  1.0041,  ...,  0.2517,  0.6048, -0.0812])\n",
      "Token: 6 a\n",
      "tensor([-0.2709,  0.0440, -0.0203,  ..., -0.2629,  0.0617,  0.2902])\n",
      "Token: 7 doctor\n",
      "tensor([ 0.0432, -0.4753,  0.1581,  ...,  0.7097,  0.2998,  0.4194])\n",
      "Token: 8 away\n",
      "tensor([-0.1038, -0.0148,  0.5993,  ...,  0.9531,  0.3693,  0.3149])\n"
     ]
    }
   ],
   "source": [
    "bert_embedding.embed(sent2)\n",
    "for i in sent2:\n",
    "  print(i)\n",
    "  print(i.embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fpML9CcEdfgT",
    "outputId": "4a0025cb-6770-4062-defc-79d3e12b250d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([-0.5985, -0.4632,  0.1300,  ...,  1.3401, -0.0510, -0.0091])"
      ]
     },
     "execution_count": 35,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1[0].embedding # in sentence 1 apple word is at the 0th index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "EVN7J_TEdfjE",
    "outputId": "5895ea0c-ae62-4160-b0b5-ca4f65b4c55a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([-0.5985, -0.4632,  0.1300,  ...,  1.7328,  1.1016,  0.1832]),\n",
       " torch.Size([3172]))"
      ]
     },
     "execution_count": 38,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent2[1].embedding , sent2[1].embedding.shape # in sentence 2 apple word is at 1st index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "bzC75Ze-gGwj",
    "outputId": "dabe9ccb-5d61-4a04-c4b9-5fb8410bb1db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "43.12472152709961\n"
     ]
    }
   ],
   "source": [
    "# Calculating the bert distance\n",
    "bert_dst = distance.euclidean(np.array(sent1[0].embedding),\n",
    "                               np.array(sent2[1].embedding))\n",
    "print(bert_dst)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Qa0D1LqDgGza"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vXklZvfSgG27"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "M_Vnr6DFgG5q"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_HPfbe_7Zoev"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "BERT vs Word2Vec.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}