Y Combinator keywords.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>url</th>\n",
       "      <th>batch</th>\n",
       "      <th>vertical</th>\n",
       "      <th>description</th>\n",
       "      <th>dead</th>\n",
       "      <th>has_ff</th>\n",
       "      <th>all_ff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Thunderpod</td>\n",
       "      <td>https://www.thunderpod.com/</td>\n",
       "      <td>w2020</td>\n",
       "      <td>None</td>\n",
       "      <td>Social Fitness App where content is consumed i...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Dinesafe</td>\n",
       "      <td>https://dinesafe.org</td>\n",
       "      <td>s2018</td>\n",
       "      <td>B2B</td>\n",
       "      <td>We crowdsource food poisoning reports and help...</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Parakey</td>\n",
       "      <td>http://parakey.com</td>\n",
       "      <td>s2005</td>\n",
       "      <td>None</td>\n",
       "      <td></td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Demeanor.co</td>\n",
       "      <td>https://www.demeanor.co</td>\n",
       "      <td>s2018</td>\n",
       "      <td>Media</td>\n",
       "      <td>Now part of thentwrk.com</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Oolu</td>\n",
       "      <td>http://oolusolar.com</td>\n",
       "      <td>s2015</td>\n",
       "      <td>Hardware</td>\n",
       "      <td>Oolu is a social enterprise that finances high...</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          name                          url  batch  vertical  \\\n",
       "0   Thunderpod  https://www.thunderpod.com/  w2020      None   \n",
       "1     Dinesafe         https://dinesafe.org  s2018       B2B   \n",
       "2      Parakey           http://parakey.com  s2005      None   \n",
       "3  Demeanor.co      https://www.demeanor.co  s2018     Media   \n",
       "4         Oolu         http://oolusolar.com  s2015  Hardware   \n",
       "\n",
       "                                         description   dead has_ff all_ff  \n",
       "0  Social Fitness App where content is consumed i...  False  False  False  \n",
       "1  We crowdsource food poisoning reports and help...  False  False  False  \n",
       "2                                                     False   None   None  \n",
       "3                           Now part of thentwrk.com  False  False  False  \n",
       "4  Oolu is a social enterprise that finances high...  False   True  False  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "r = requests.get(\"https://api.ycombinator.com/companies/export.json\")\n",
    "df = pd.DataFrame(r.json())\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "\n",
    "# !python -m spacy download en_core_web_sm     \n",
    "import en_core_web_sm\n",
    "nlp = en_core_web_sm.load()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['s2019']"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batches = sorted(df.batch.unique(), key=lambda x: int(x[1:])+0.1 if x[0]=='s' else int(x[1:]))\n",
    "\n",
    "batches[-2:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.wordnet import WordNetLemmatizer\n",
    "from nltk.corpus import stopwords\n",
    "#Word cloud\n",
    "from os import path\n",
    "from PIL import Image\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "import matplotlib.pyplot as plt\n",
    "lem = WordNetLemmatizer()\n",
    "stop_words = set(stopwords.words(\"english\"))\n",
    "result = []\n",
    "batches = sorted(df.batch.unique(), key=lambda x: int(x[1:])+0.1 if x[0]=='s' else int(x[1:]))\n",
    "\n",
    "for batch in batches:\n",
    "    dataset = df[df.batch == batch]\n",
    "    dataset = dataset[dataset['description'].astype(bool)].reset_index()   \n",
    "    corpus = []\n",
    "    for i in range(0, len(dataset)-1):\n",
    "        result.append((batch, nlp(dataset['description'][i])))\n",
    "    words = {}\n",
    "    for x in result:\n",
    "        for word in list(x[1].noun_chunks):\n",
    "            word = str(word)\n",
    "            word = \" \".join(token for token in word.split(\" \") if lem.lemmatize(token.lower()) not in stop_words)\n",
    "            if word.lower() == \"latin america\":\n",
    "                word = \"LatAm\"\n",
    "            if word.lower() == \"latam\":\n",
    "                word = \"LatAm\"\n",
    "            if word:\n",
    "                if word in words:\n",
    "                    words[word] += 1\n",
    "                else:\n",
    "                    words[word] = 1\n",
    "    wordcloud = WordCloud(\n",
    "                          background_color='white',\n",
    "                          max_words=100,\n",
    "                          max_font_size=70, \n",
    "                          random_state=42,\n",
    "                          width = 600,\n",
    "                          height = 300\n",
    "                         ).fit_words(words)\n",
    "    wordcloud.to_file(f\"result/{batch}.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": ".venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}