user-cluster-analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cff32eeb-926b-4d0f-b0d0-67636deb0bae",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "pd.options.display.max_colwidth = 300\n",
    "import config\n",
    "data_root= config.DATA_ROOT\n",
    "tqdm.pandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f5dbc8cc-a590-4e65-875d-afac8d275165",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(df):  1698096\n"
     ]
    }
   ],
   "source": [
    "df=pd.read_pickle(data_root+'users-en-unique.pkl')\n",
    "print(\"len(df): \", len(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "af3fc1dc-61f6-4a0c-9eda-ed79587760ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "academic_topics = [1,6,8,21,25, 30,42]\n",
    "tech_topics = [2, 4, 23]\n",
    "brand_topics = [12,17,20,28,32,33,35,36,41]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bff44e8b-8d90-4172-b4d4-9723eafcdf61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# df['user_group'] = df.apply(lambda row: \n",
    "#                             'academic' if row.topic in academic_topics else \\\n",
    "#                             ('tech' if row.topic in tech_topics else \\\n",
    "#                              ('brand' if row.topic in brand_topics else 'other')), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "445bd9f2-11f5-46d4-966b-3e77ed76eaee",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_pickle(data_root+'users-en-unique.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "44a188ce-d79b-491e-9d25-6ca79f8cdcbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_group\n",
       "other       772943\n",
       "academic    414860\n",
       "brand       375612\n",
       "tech        134680\n",
       "Name: username, dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby('user_group').username.count().sort_values(ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f7dc097e-0637-4d86-aca3-b3d1895a273c",
   "metadata": {},
   "outputs": [],
   "source": [
    "aca_df = df[df.user_group=='academic']\n",
    "tech_df=df[df.user_group=='tech']\n",
    "brand_df = df[df.user_group=='brand']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "67b8c5a4-ce62-4108-b18a-8e74ae5e0572",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['username', 'followers', 'tweets', 'profile_desc', 'location',\n",
       "       'verified', 'entities', 'search_term', 'profile_desc_clean', 'lang1',\n",
       "       'lang2', 'lang3', 'profile_lang', 'topic', 'user_group'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "34b3b4bf-2fe6-4526-8b84-a4f1f14d700e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>profile_desc</th>\n",
       "      <th>profile_desc_clean</th>\n",
       "      <th>user_group</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>userid</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2783235942</th>\n",
       "      <td>Senior Lecturer in #History, @Uni_Newcastle, Australia. #Hunter #Wine (NewSouth 2018). #Fulbright Scholar #UCDavis, USA. She/her. #EnvHist</td>\n",
       "      <td>senior lecturer history uni_newcastle australia hunter wine newsouth 2018 fulbright scholar ucdavis usa envhist</td>\n",
       "      <td>brand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20425834</th>\n",
       "      <td>Dad to my beautiful son &amp; daughter, love playing rugby and spending time with my family.</td>\n",
       "      <td>dad beautiful son daughter love playing rugby spending time family</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>636041091</th>\n",
       "      <td>The spirit of independence and freedom of thought.\\r\\n\\r\\nA fan of Warren Kole. Follow Warren Kole here: @Warren_Kole.</td>\n",
       "      <td>spirit independence freedom fan warren kole follow warren kole warren_kole</td>\n",
       "      <td>brand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>772814411006406656</th>\n",
       "      <td>.::Promoting @ElevenPaths own products, innovation and technologies worldwide::.</td>\n",
       "      <td>promoting elevenpaths products innovation technologies worldwide</td>\n",
       "      <td>tech</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>746142067</th>\n",
       "      <td>Conservator; ex-host @thecwordpodcast; Coordinator @Cam_Repro. Museums, detective fic, trains, politics, VI/disability, beer, crosswords &amp; general geekery #AFOL</td>\n",
       "      <td>conservator host thecwordpodcast coordinator cam_repro museums detective fic trains politics disability beer crosswords geekery afol</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1620928082</th>\n",
       "      <td>Infosec noob | He/Him | Engaged to @theminisatan</td>\n",
       "      <td>infosec noob engaged theminisatan</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4126052735</th>\n",
       "      <td>Official Twitter feed for Whipps Cross University Hospital, part of @NHSBartsHealth. Here to chat between Monday to Friday, 9am-5pm.</td>\n",
       "      <td>official twitter feed whipps cross university hospital nhsbartshealth chat monday friday 9am 5pm</td>\n",
       "      <td>brand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>376136115</th>\n",
       "      <td>Local member for Ginninderra. Minister for the Arts, Business, Multicultural Affairs, Human Rights.\\nAuthorised by Tara Cheyne for ACT Labor.</td>\n",
       "      <td>local ginninderra minister arts business multicultural affairs human rights authorised tara cheyne act labor</td>\n",
       "      <td>brand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>819999</th>\n",
       "      <td>Founder of Uncork Capital, one of the OG micro-VC firms. Invested in 240+ cos. @bernadette hubby, dad of 2, wino, skier, happily living in Woodside/SF/Kirkwood</td>\n",
       "      <td>founder uncork capital micro firms invested 240 cos bernadette hubby dad wino skier happily living woodside kirkwood</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1236973318680588288</th>\n",
       "      <td>We are proud that our founder won “Favourite Wellness Influencer 2021” at the Health &amp; Wellbeing Awards 🏆🥇 we are also proud to be #SBS &amp; #KingOf Winners</td>\n",
       "      <td>proud founder favourite wellness influencer 2021 health wellbeing awards proud sbs kingof winners</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                         profile_desc  \\\n",
       "userid                                                                                                                                                                                  \n",
       "2783235942                                 Senior Lecturer in #History, @Uni_Newcastle, Australia. #Hunter #Wine (NewSouth 2018). #Fulbright Scholar #UCDavis, USA. She/her. #EnvHist   \n",
       "20425834                                                                                     Dad to my beautiful son & daughter, love playing rugby and spending time with my family.   \n",
       "636041091                                                      The spirit of independence and freedom of thought.\\r\\n\\r\\nA fan of Warren Kole. Follow Warren Kole here: @Warren_Kole.   \n",
       "772814411006406656                                                                                   .::Promoting @ElevenPaths own products, innovation and technologies worldwide::.   \n",
       "746142067            Conservator; ex-host @thecwordpodcast; Coordinator @Cam_Repro. Museums, detective fic, trains, politics, VI/disability, beer, crosswords & general geekery #AFOL   \n",
       "1620928082                                                                                                                           Infosec noob | He/Him | Engaged to @theminisatan   \n",
       "4126052735                                       Official Twitter feed for Whipps Cross University Hospital, part of @NHSBartsHealth. Here to chat between Monday to Friday, 9am-5pm.   \n",
       "376136115                               Local member for Ginninderra. Minister for the Arts, Business, Multicultural Affairs, Human Rights.\\nAuthorised by Tara Cheyne for ACT Labor.   \n",
       "819999                Founder of Uncork Capital, one of the OG micro-VC firms. Invested in 240+ cos. @bernadette hubby, dad of 2, wino, skier, happily living in Woodside/SF/Kirkwood   \n",
       "1236973318680588288         We are proud that our founder won “Favourite Wellness Influencer 2021” at the Health & Wellbeing Awards 🏆🥇 we are also proud to be #SBS & #KingOf Winners   \n",
       "\n",
       "                                                                                                                                       profile_desc_clean  \\\n",
       "userid                                                                                                                                                      \n",
       "2783235942                                senior lecturer history uni_newcastle australia hunter wine newsouth 2018 fulbright scholar ucdavis usa envhist   \n",
       "20425834                                                                               dad beautiful son daughter love playing rugby spending time family   \n",
       "636041091                                                                      spirit independence freedom fan warren kole follow warren kole warren_kole   \n",
       "772814411006406656                                                                       promoting elevenpaths products innovation technologies worldwide   \n",
       "746142067            conservator host thecwordpodcast coordinator cam_repro museums detective fic trains politics disability beer crosswords geekery afol   \n",
       "1620928082                                                                                                              infosec noob engaged theminisatan   \n",
       "4126052735                                               official twitter feed whipps cross university hospital nhsbartshealth chat monday friday 9am 5pm   \n",
       "376136115                                    local ginninderra minister arts business multicultural affairs human rights authorised tara cheyne act labor   \n",
       "819999                               founder uncork capital micro firms invested 240 cos bernadette hubby dad wino skier happily living woodside kirkwood   \n",
       "1236973318680588288                                     proud founder favourite wellness influencer 2021 health wellbeing awards proud sbs kingof winners   \n",
       "\n",
       "                    user_group  \n",
       "userid                          \n",
       "2783235942               brand  \n",
       "20425834                 other  \n",
       "636041091                brand  \n",
       "772814411006406656        tech  \n",
       "746142067                other  \n",
       "1620928082               other  \n",
       "4126052735               brand  \n",
       "376136115                brand  \n",
       "819999                   other  \n",
       "1236973318680588288      other  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sample(10)[['profile_desc', 'profile_desc_clean', 'user_group']] "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82b3bb53-257d-4bbd-8fe6-48e2b03596ba",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "edenv",
   "language": "python",
   "name": "edenv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}