Emoji_Text_Search_Notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Emoji Text Search\n",
    "find emoji closest in meaning to text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "Requirement already satisfied: gensim==3.8.3 in /usr/local/lib/python3.9/site-packages (3.8.3)\n",
      "Requirement already satisfied: six>=1.5.0 in /usr/local/lib/python3.9/site-packages (from gensim==3.8.3) (1.15.0)\n",
      "Requirement already satisfied: scipy>=0.18.1 in /usr/local/lib/python3.9/site-packages (from gensim==3.8.3) (1.7.0)\n",
      "Requirement already satisfied: smart-open>=1.8.1 in /usr/local/lib/python3.9/site-packages (from gensim==3.8.3) (5.1.0)\n",
      "Requirement already satisfied: numpy>=1.11.3 in /usr/local/lib/python3.9/site-packages (from gensim==3.8.3) (1.20.2)\n",
      "\u001b[33mWARNING: You are using pip version 21.0.1; however, version 21.1.3 is available.\n",
      "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: emoji_data in /usr/local/lib/python3.9/site-packages (0.1.6.1)\n",
      "\u001b[33mWARNING: You are using pip version 21.0.1; however, version 21.1.3 is available.\n",
      "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Collecting sklearn\n",
      "  Using cached sklearn-0.0.tar.gz (1.1 kB)\n",
      "Collecting scikit-learn\n",
      "  Using cached scikit_learn-0.24.2-cp39-cp39-macosx_10_13_x86_64.whl (7.3 MB)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.9/site-packages (from scikit-learn->sklearn) (1.20.2)\n",
      "Collecting threadpoolctl>=2.0.0\n",
      "  Using cached threadpoolctl-2.1.0-py3-none-any.whl (12 kB)\n",
      "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.9/site-packages (from scikit-learn->sklearn) (1.7.0)\n",
      "Collecting joblib>=0.11\n",
      "  Using cached joblib-1.0.1-py3-none-any.whl (303 kB)\n",
      "Building wheels for collected packages: sklearn\n",
      "  Building wheel for sklearn (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sklearn: filename=sklearn-0.0-py2.py3-none-any.whl size=1316 sha256=58280e74e455374257116c8269afbba644de18a8a7d69bd2e70a3e7273d0f492\n",
      "  Stored in directory: /Users/sebastiankelly/Library/Caches/pip/wheels/e4/7b/98/b6466d71b8d738a0c547008b9eb39bf8676d1ff6ca4b22af1c\n",
      "Successfully built sklearn\n",
      "Installing collected packages: threadpoolctl, joblib, scikit-learn, sklearn\n",
      "Successfully installed joblib-1.0.1 scikit-learn-0.24.2 sklearn-0.0 threadpoolctl-2.1.0\n",
      "\u001b[33mWARNING: You are using pip version 21.0.1; however, version 21.1.3 is available.\n",
      "You should consider upgrading via the '/usr/local/opt/python@3.9/bin/python3.9 -m pip install --upgrade pip' command.\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install gensim==3.8.3\n",
    "%pip install emoji_data\n",
    "%pip install sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import gensim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "APP_FOLDER = os.path.dirname('.')\n",
    "FIXTURES_FOLDER = os.path.join(APP_FOLDER, \"src/fixtures\")\n",
    "GOOGLE_NEWS = os.path.join(FIXTURES_FOLDER, \"GoogleNews-vectors-negative300-SLIM.bin.gz\")\n",
    "model = gensim.models.KeyedVectors.load_word2vec_format(GOOGLE_NEWS, binary=True)\n",
    "model.init_sims(replace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": [
      "         emoji                description\n0       👨‍❤️‍👨  couple with heart man man\n1     👨‍❤️‍💋‍👨               kiss man man\n2          👨‍👦             family man boy\n3        👨‍👦‍👦         family man boy boy\n4          👨‍👧            family man girl\n...        ...                        ...\n3516        🧝🏻                  elf light\n3517        🧝🏼           elf medium-light\n3518        🧝🏽                 elf medium\n3519        🧝🏾            elf medium-dark\n3520        🧝🏿                   elf dark\n\n[2339 rows x 2 columns]\n"
     ]
    }
   ],
   "source": [
    "from emoji_data import EmojiSequence\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "d = {'emoji': [], 'description': []}\n",
    "for (emoji, emoji_meta) in EmojiSequence:\n",
    "    d['emoji'].append(emoji)\n",
    "    d['description'].append(emoji_meta.description)\n",
    "\n",
    "df = pd.DataFrame(d)\n",
    "df = df[df['description'] != '']\n",
    "\n",
    "df['description'] = df['description'].str.split(' skin tone').str[0].str.replace(':', '').str.replace(',', '')\n",
    "df = df.drop_duplicates(subset=['description'])\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_words(text):\n",
    "    words = text.split(\" \")\n",
    "    word_vectors = [ model.syn0norm[model.vocab[word].index] for word in words if word in model.vocab]\n",
    "    if len(word_vectors) > 0:\n",
    "        mean_vector = np.array(word_vectors).mean(axis=0)\n",
    "        unit_vector = gensim.matutils.unitvec(mean_vector).astype(np.float32).tolist()\n",
    "    else:\n",
    "        unit_vector = np.zeros(model.vector_size, ).tolist()\n",
    "    return unit_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/var/folders/z8/0bm67qd10m54kz5_n90f0gvxx01fhr/T/ipykernel_10478/2124056185.py:3: DeprecationWarning: Call to deprecated `syn0norm` (Attribute will be removed in 4.0.0, use self.vectors_norm instead).\n  word_vectors = [ model.syn0norm[model.vocab[word].index] for word in words if word in model.vocab]\n"
     ]
    }
   ],
   "source": [
    "df['vector'] = df['description'].apply(process_words)\n",
    "# class_df = pd.DataFrame(class_keys)\n",
    "# print(df)\n",
    "# df.to_csv(EMOJI_DATA)\n",
    "# EMOJI_DATA = os.path.join(FIXTURES_FOLDER, \"emoji_vectors.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stderr",
     "text": [
      "/var/folders/z8/0bm67qd10m54kz5_n90f0gvxx01fhr/T/ipykernel_10478/2124056185.py:3: DeprecationWarning: Call to deprecated `syn0norm` (Attribute will be removed in 4.0.0, use self.vectors_norm instead).\n  word_vectors = [ model.syn0norm[model.vocab[word].index] for word in words if word in model.vocab]\n"
     ]
    },
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": [
       "     emoji  similarity  description\n",
       "1644     🐕    1.000000          dog\n",
       "1448     🌭    0.775372      hot dog\n",
       "1631     🐈    0.760946          cat\n",
       "2313     🦮    0.759934    guide dog\n",
       "1322   🐕‍🦺    0.722582  service dog"
      ],
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>emoji</th>\n      <th>similarity</th>\n      <th>description</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1644</th>\n      <td>🐕</td>\n      <td>1.000000</td>\n      <td>dog</td>\n    </tr>\n    <tr>\n      <th>1448</th>\n      <td>🌭</td>\n      <td>0.775372</td>\n      <td>hot dog</td>\n    </tr>\n    <tr>\n      <th>1631</th>\n      <td>🐈</td>\n      <td>0.760946</td>\n      <td>cat</td>\n    </tr>\n    <tr>\n      <th>2313</th>\n      <td>🦮</td>\n      <td>0.759934</td>\n      <td>guide dog</td>\n    </tr>\n    <tr>\n      <th>1322</th>\n      <td>🐕‍🦺</td>\n      <td>0.722582</td>\n      <td>service dog</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
     },
     "metadata": {},
     "execution_count": 65
    }
   ],
   "source": [
    "import numpy as np\n",
    "from numpy import dot\n",
    "from numpy.linalg import norm\n",
    "import pandas as pd\n",
    "import random\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "def find_similarity_to_search(search_vector):\n",
    "    def func(emoji_vector):\n",
    "        b_emoji = np.array(emoji_vector)\n",
    "        cos_sim = dot(search_vector, b_emoji) / (norm(search_vector) * norm(b_emoji))\n",
    "        return cos_sim\n",
    "    return func\n",
    "\n",
    "search_text = \"dog\"\n",
    "search_vector = np.array(process_words(search_text))\n",
    "similarity_search = find_similarity_to_search(search_vector)\n",
    "df['similarity'] = df['vector'].apply(similarity_search)\n",
    "x = df.nlargest(5, 'similarity')\n",
    "x[['emoji', 'similarity', 'description']]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.2 64-bit"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  },
  "interpreter": {
   "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}