Dataset_Cleaning_and_Merging_Venice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0bc75d67",
   "metadata": {},
   "source": [
    "# Dataset Cleaning and Merging\n",
    "\n",
    "by Nan BAI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd762055",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from argparse import Namespace\n",
    "from collections import Counter\n",
    "import pickle\n",
    "import os\n",
    "import urllib\n",
    "\n",
    "args = Namespace(\n",
    "    # Data and Path information\n",
    "    save_dir = 'data_storage/',\n",
    "    image_dir = 'data_storage/images/'\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "84a7a9a2",
   "metadata": {},
   "source": [
    "## For Small Dataset of Venice"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7bbb3241",
   "metadata": {},
   "source": [
    "### Data Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 915,
   "id": "bb57093e",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_df = pd.read_csv('Venezia/data_storage/images/photos_sizes.csv',sep='\\t').rename(columns={'Unnamed: 0':'index'})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bef0beb",
   "metadata": {},
   "source": [
    "The small data has a maximum index of 5000 during the data collection period"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 916,
   "id": "fb575e68",
   "metadata": {},
   "outputs": [],
   "source": [
    "size_df['index'] = size_df['index'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 917,
   "id": "7c129cfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5000"
      ]
     },
     "execution_count": 917,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(size_df['index'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c75923fc",
   "metadata": {},
   "source": [
    "Only the data with downloadable marks are kept for further analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 918,
   "id": "ab495484",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = size_df[size_df['candownload']==1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 919,
   "id": "0db2ae69",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>comments</th>\n",
       "      <th>taken</th>\n",
       "      <th>views</th>\n",
       "      <th>people</th>\n",
       "      <th>tags</th>\n",
       "      <th>lat</th>\n",
       "      <th>lon</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '47908950-51870743082-128896', 'author...</td>\n",
       "      <td>45.438413</td>\n",
       "      <td>12.318019</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870743082/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '47908950-51871789478-128896', 'author...</td>\n",
       "      <td>45.438405</td>\n",
       "      <td>12.318063</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51871789478/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '47908950-51870742942-128896', 'author...</td>\n",
       "      <td>45.438802</td>\n",
       "      <td>12.319472</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870742942/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>3</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>378</td>\n",
       "      <td>0</td>\n",
       "      <td>[]</td>\n",
       "      <td>45.433650</td>\n",
       "      <td>12.339427</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/64148082@N02/518...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51863124066</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>186704588@N07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venice - Ponte dei Sospiri</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>3</td>\n",
       "      <td>2022-01-20 18:14:39</td>\n",
       "      <td>242</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '186683258-51863124066-11002', 'author...</td>\n",
       "      <td>45.433686</td>\n",
       "      <td>12.341019</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/glolucach/518631...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>51233430558</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-20 14:58:59</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '6752254-51233430558-36842', 'author':...</td>\n",
       "      <td>45.428733</td>\n",
       "      <td>12.357844</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233430...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>51234286630</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-20 14:57:46</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '6752254-51234286630-36842', 'author':...</td>\n",
       "      <td>45.428886</td>\n",
       "      <td>12.357227</td>\n",
       "      <td>Castello</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234286...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>51234294135</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_164727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-20 16:47:27</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '6752254-51234294135-36842', 'author':...</td>\n",
       "      <td>45.428200</td>\n",
       "      <td>12.357805</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234294...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>51234291635</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-20 15:56:46</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '6752254-51234291635-36842', 'author':...</td>\n",
       "      <td>45.429366</td>\n",
       "      <td>12.358475</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234291...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>51233434888</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2021-05-20 15:55:25</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>[{'id': '6752254-51233434888-36842', 'author':...</td>\n",
       "      <td>45.429341</td>\n",
       "      <td>12.358516</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233434...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2952 rows × 19 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "13    51863124066            1   \n",
       "...           ...          ...   \n",
       "4981  51233430558            1   \n",
       "4982  51234286630            1   \n",
       "4983  51234294135            1   \n",
       "4984  51234291635            1   \n",
       "4985  51233434888            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "13    https://live.staticflickr.com/65535/5186312406...  186704588@N07   \n",
       "...                                                 ...            ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   83293103@N00   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   83293103@N00   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   83293103@N00   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   83293103@N00   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   83293103@N00   \n",
       "\n",
       "             owner_loc                       title  \\\n",
       "3     Tallinn, Estonia                         ...   \n",
       "4     Tallinn, Estonia                         ...   \n",
       "5     Tallinn, Estonia                         ...   \n",
       "6                  NaN     VENEZIA. CARNEVALE 2019   \n",
       "13                 NaN  Venice - Ponte dei Sospiri   \n",
       "...                ...                         ...   \n",
       "4981     Milano, Italy             20210520_145859   \n",
       "4982     Milano, Italy             20210520_145745   \n",
       "4983     Milano, Italy             20210520_164727   \n",
       "4984     Milano, Italy             20210520_155646   \n",
       "4985     Milano, Italy             20210520_155525   \n",
       "\n",
       "                                            description  comments  \\\n",
       "3                                         Venice, Italy         0   \n",
       "4                                         Venice, Italy         0   \n",
       "5                                         Venice, Italy         0   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...         3   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...         3   \n",
       "...                                                 ...       ...   \n",
       "4981                                                NaN         0   \n",
       "4982                                                NaN         0   \n",
       "4983                                                NaN         0   \n",
       "4984                                                NaN         0   \n",
       "4985                                                NaN         0   \n",
       "\n",
       "                    taken  views  people  \\\n",
       "3     2021-10-05 14:45:41      8       0   \n",
       "4     2021-10-05 14:46:05      2       0   \n",
       "5     2021-10-05 15:13:13      6       0   \n",
       "6     2019-02-28 14:11:02    378       0   \n",
       "13    2022-01-20 18:14:39    242       0   \n",
       "...                   ...    ...     ...   \n",
       "4981  2021-05-20 14:58:59      3       0   \n",
       "4982  2021-05-20 14:57:46      5       0   \n",
       "4983  2021-05-20 16:47:27     12       0   \n",
       "4984  2021-05-20 15:56:46     11       0   \n",
       "4985  2021-05-20 15:55:25     10       0   \n",
       "\n",
       "                                                   tags        lat        lon  \\\n",
       "3     [{'id': '47908950-51870743082-128896', 'author...  45.438413  12.318019   \n",
       "4     [{'id': '47908950-51871789478-128896', 'author...  45.438405  12.318063   \n",
       "5     [{'id': '47908950-51870742942-128896', 'author...  45.438802  12.319472   \n",
       "6                                                    []  45.433650  12.339427   \n",
       "13    [{'id': '186683258-51863124066-11002', 'author...  45.433686  12.341019   \n",
       "...                                                 ...        ...        ...   \n",
       "4981  [{'id': '6752254-51233430558-36842', 'author':...  45.428733  12.357844   \n",
       "4982  [{'id': '6752254-51234286630-36842', 'author':...  45.428886  12.357227   \n",
       "4983  [{'id': '6752254-51234294135-36842', 'author':...  45.428200  12.357805   \n",
       "4984  [{'id': '6752254-51234291635-36842', 'author':...  45.429366  12.358475   \n",
       "4985  [{'id': '6752254-51233434888-36842', 'author':...  45.429341  12.358516   \n",
       "\n",
       "           neighbourhood                                                url  \n",
       "3          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870743082/  \n",
       "4          Piazzale Roma   https://www.flickr.com/photos/ruudi/51871789478/  \n",
       "5          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870742942/  \n",
       "6              San Marco  https://www.flickr.com/photos/64148082@N02/518...  \n",
       "13             San Marco  https://www.flickr.com/photos/glolucach/518631...  \n",
       "...                  ...                                                ...  \n",
       "4981  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233430...  \n",
       "4982            Castello  https://www.flickr.com/photos/badswan/51234286...  \n",
       "4983  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234294...  \n",
       "4984  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234291...  \n",
       "4985  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233434...  \n",
       "\n",
       "[2952 rows x 19 columns]"
      ]
     },
     "execution_count": 919,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e383c046",
   "metadata": {},
   "source": [
    "### Data Cleaning and Processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "552c926b",
   "metadata": {},
   "source": [
    "#### Finding Locals and Neighbourhoods"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c23ff3ce",
   "metadata": {},
   "source": [
    "The data are distributed in 33 different neighbourhoods in Venice, posted by 331 different owners, among which 5 are local Venician people, due to their indicated locations in Flickr user profile, who contributed 24 posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 920,
   "id": "7a186974",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Piazzale Roma', 'San Marco', 'Murano', 'Rialto', 'Dorsoduro',\n",
       "       'Città Giardino', 'Sestière di San Polo', 'Cannaregio',\n",
       "       'San Giuliano - Venezia', nan, 'San Clemente', 'Venice',\n",
       "       'San Giorgio Maggiore', 'Venice-Lido', 'Santa Croce', 'Castello',\n",
       "       'Giudecca', 'Quartiere S. Elena', 'Quartiere San Giacomo',\n",
       "       'Quartiere Campo di Marte', \"Sant'Elena\",\n",
       "       'San Lazzaro degli Armeni', 'Malamocco', 'Moranzani', 'Tessera',\n",
       "       'Passo Campalto', \"Sant'Erasmo\", 'Le Vignole',\n",
       "       'San Francesco del Deserto', 'San Nicolò di Lido',\n",
       "       'Quattro Fontane', 'San Croce', 'San Polo'], dtype=object)"
      ]
     },
     "execution_count": 920,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['neighbourhood'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 921,
   "id": "82a84915",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "33"
      ]
     },
     "execution_count": 921,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['neighbourhood'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 922,
   "id": "e6b827c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Tallinn, Estonia', nan, 'Valencia, España', 'Venezia, ITALIA',\n",
       "       'Portugal', 'Niedersachsen, Deutschland',\n",
       "       'Mogliano Veneto, Italia', 'Guelph, Canada', 'Deutschland',\n",
       "       'Errenteria.', 'Texas, USA', 'NICE, France', 'France',\n",
       "       'Newton, MA, USA', 'London, UK', 'Virginia, USA', 'Michigan, USA',\n",
       "       'Brunswick, Maine, USA', 'Schiedam, Netherlands',\n",
       "       'Arlington, Massachusetts, USA', 'Bologna, Italia',\n",
       "       'Venezia, Italy', 'Milano', 'Garmisch, Germany',\n",
       "       'Tilburg, Netherlands', 'Lausanne, Switzerland',\n",
       "       'Freiburg, Germany', 'Trieste, Italia', 'Lecco, Italia',\n",
       "       'Bordeaux, France', 'Darwin, Australia', 'Naaldwijk, Netherlands',\n",
       "       'Italy', 'Seattle', 'Salzburg - Vienna, Austria', 'PAVIA, ITALIA',\n",
       "       'Kentucky, USA', 'Feltre, Italia', 'Italia', 'Udine, Italia',\n",
       "       'Toulouse, France', 'Venezia, Italia', 'Basel, Switzerland',\n",
       "       'Marcon, Italia', 'Prague, Czech Republic', 'Antibes, France',\n",
       "       'Roma', 'Torvaianica  (Roma), Italia',\n",
       "       'Port Townsend, United States of America', '-FRIULI-, Italia',\n",
       "       'Padova, Italy', 'Ankara, Turkey', 'Seattle, USA',\n",
       "       'Muggiò, Italia', 'Paris, France', 'Copenhagen, Denmark',\n",
       "       'Entre-Deux, Réunion', 'Méru, France', 'Moscow, Russia',\n",
       "       'Verona, Italia', 'Vienna, Austria', 'La Plata, Argentina',\n",
       "       'Torino', 'LA SPEZIA, Italy', 'Milano, Italy', 'Nîmes, France',\n",
       "       'Balcarce, Argentina', 'Germany', 'USA', 'Pau, France',\n",
       "       'Treviso, Italia', 'Worms, Deutschland', 'Helsinki, Finland',\n",
       "       'Milan, Italy', 'MADRID, España', 'Córdoba',\n",
       "       'St. Georgen/Stfld., Austria', 'Buenos Aires, Argentina',\n",
       "       'Madrid, España', 'Dieppe c.a., Frankrijk',\n",
       "       'Les Villes de Benicàssim, Spain', 'Australia',\n",
       "       'Wolkersdorf, Austria', 'Forlì, Italy', 'Ravenna, Italia',\n",
       "       'Gerlingen', 'Enghien-les-Bains', 'Toowoomba, Australia',\n",
       "       'Brisbane, Australia', 'Niederrhein, Germany', 'Venezia, Mestre',\n",
       "       'Poissy, France', 'Milano, Italia', 'España', 'San Francisco',\n",
       "       'Firenze, Italia', 'Nottingham, UK', 'Henderson, United States',\n",
       "       'Derby, UK', 'Alexandria, Virginia, USA', 'Sydney',\n",
       "       'Czech Republic', 'Reggio Emilia, Italy',\n",
       "       'Tarazona (Zaragoza), España', 'Bristol, UK',\n",
       "       'Cape Town , South Africa', 'Den Haag, Netherlands',\n",
       "       'Guildford, United Kingdom', 'Neuhaus am Rennweg, Germany',\n",
       "       'NYC, USA', 'Russia', 'Vauréal (Cergy-Pontoise), France',\n",
       "       'Portland, Oregon, USA', 'Kiev, Ukraine', 'Riehen, Switzerland',\n",
       "       'Birmingham, UK', 'Gif-sur-Yvette, France',\n",
       "       'Jamaica Plain, Massachusetts, USA', 'Zurich, Switzerland',\n",
       "       'Padova, Italia', 'Atlanta, USA', 'New York, USA', 'Alicante',\n",
       "       'Lille, France', 'San Donà di Piave', 'Gloucester, United Kingdom',\n",
       "       'Oderzo, Italia', 'The Hague, The Netherlands', 'Egna, Italia',\n",
       "       'UK', 'La Roche-sur-Yon, Vendée,  France', 'Seefeld, Merano',\n",
       "       'Liverpool, United Kingdom', 'Minneapolis, MN, USA',\n",
       "       'San Donà di Piave  (Venezia), Italia', 'Vimodrone, Italia',\n",
       "       'Manchester, United Kingdom', 'Monfalcone, Italy',\n",
       "       'Breskens (NL); Serris (F)', 'Chiari (Bs), Italia', 'italy',\n",
       "       'City of Oz', 'Spain', 'Juan les Pins, France', 'Dresden',\n",
       "       'Modena, ITALY', 'Montrouge (Paris), France.', 'Monselice',\n",
       "       'Trieste, Italy', 'san francisco', 'Bucharest, Romania',\n",
       "       'Vung Tau, Vietnam', 'San Antonio, TX, USA', 'Tolentino, Italia',\n",
       "       'Wesel, Deutschland', 'Barcelona'], dtype=object)"
      ]
     },
     "execution_count": 922,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['owner_loc'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7abccc96",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Venezia, ITALIA',\n",
       " 'Venezia, Italy',\n",
       " 'Venezia, Italia',\n",
       " 'Venezia, Mestre',\n",
       " 'San Donà di Piave  (Venezia), Italia']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "locations = [now for now in list(df['owner_loc'].unique()) if 'venice' in str(now).lower() and 'USA' not in str(now).lower() or 'venezia' in str(now).lower()]\n",
    "locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "3bac5d51",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\1482302024.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['local'] = df['owner_loc'].apply(lambda x: x in locations)\n"
     ]
    }
   ],
   "source": [
    "df['local'] = df['owner_loc'].apply(lambda x: x in locations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 857,
   "id": "eadec365",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5"
      ]
     },
     "execution_count": 857,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['local']]['owner'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 858,
   "id": "4a678700",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "24"
      ]
     },
     "execution_count": 858,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df[df['local']])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 856,
   "id": "110f3168",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "330"
      ]
     },
     "execution_count": 856,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['owner'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "566fbc35",
   "metadata": {},
   "source": [
    "#### Processing Tags and Converting to lists"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "28d04ba2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['adriaticsea', 'italy', 'venice', 'bridge', 'water']"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "re.findall(r'_content\\': \\'([a-zA-Z]+)',df['tags'].iloc[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dbdf230d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\1574953216.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['tag_list'] = df['tags'].apply(lambda x: re.findall(r'_content\\': \\'([a-zA-Z]+)',x))\n"
     ]
    }
   ],
   "source": [
    "df['tag_list'] = df['tags'].apply(lambda x: re.findall(r'_content\\': \\'([a-zA-Z]+)',x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "84e5b3de",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\1455675285.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['tag_len'] = df['tag_list'].apply(lambda x: len(x))\n"
     ]
    }
   ],
   "source": [
    "df['tag_len'] = df['tag_list'].apply(lambda x: len(x))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "16706e51",
   "metadata": {},
   "source": [
    "#### Filtering out Erotic images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "29d75da1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\4184844319.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['NOT'] = df['tag_list'].apply(lambda x: 'erotic' in x or 'nude' in x)\n"
     ]
    }
   ],
   "source": [
    "df['NOT'] = df['tag_list'].apply(lambda x: 'erotic' in x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 597,
   "id": "7c11db77",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[~df['NOT']]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0219af66",
   "metadata": {},
   "source": [
    "#### Generating Timestamps for the posts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "e2043e24",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\40314514.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['year'] = df['taken'].apply(lambda x: int(x[:4]))\n"
     ]
    }
   ],
   "source": [
    "df['year'] = df['taken'].apply(lambda x: int(x[:4]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "5bcf2fb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\3965765013.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['time'] = df['taken'].apply(pd.Timestamp)\n"
     ]
    }
   ],
   "source": [
    "df['time'] = df['taken'].apply(pd.Timestamp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "53e8c032",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\1498983366.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['week'] = df['time'].apply(lambda x: x.week)\n"
     ]
    }
   ],
   "source": [
    "df['week'] = df['time'].apply(lambda x: x.week)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "08cfac60",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\3367497227.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df['unique_week'] = df['year'] * 100 + df['week']\n"
     ]
    }
   ],
   "source": [
    "df['unique_week'] = df['year'] * 100 + df['week']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "1aca438c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "273"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df['unique_week'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "67fc5492",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2021, 2019, 2022, 2020, 2017, 2011, 2018, 1983, 2015, 1981, 2016,\n",
       "       2008, 1995, 2007, 2013, 1971, 2009, 1963, 1954, 2012, 2014, 2010,\n",
       "       1973, 2004, 1958, 2005, 2006, 1974, 2003, 1991], dtype=int64)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['year'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 849,
   "id": "3ecedf63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870743082/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, bridge, water]</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51871789478/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice]</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870742942/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, water]</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>...</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/64148082@N02/518...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>9</td>\n",
       "      <td>201909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51863124066</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>186704588@N07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venice - Ponte dei Sospiri</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>...</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/glolucach/518631...</td>\n",
       "      <td>False</td>\n",
       "      <td>[ponte, pont, canal, river, city, history, cul...</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-01-20 18:14:39</td>\n",
       "      <td>3</td>\n",
       "      <td>202203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>51233430558</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233430...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 14:58:59</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>51234286630</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Castello</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234286...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 14:57:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>51234294135</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_164727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234294...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 16:47:27</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>51234291635</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234291...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 15:56:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>51233434888</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233434...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 15:55:25</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "13    51863124066            1   \n",
       "...           ...          ...   \n",
       "4981  51233430558            1   \n",
       "4982  51234286630            1   \n",
       "4983  51234294135            1   \n",
       "4984  51234291635            1   \n",
       "4985  51233434888            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "13    https://live.staticflickr.com/65535/5186312406...  186704588@N07   \n",
       "...                                                 ...            ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   83293103@N00   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   83293103@N00   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   83293103@N00   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   83293103@N00   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   83293103@N00   \n",
       "\n",
       "             owner_loc                       title  \\\n",
       "3     Tallinn, Estonia                         ...   \n",
       "4     Tallinn, Estonia                         ...   \n",
       "5     Tallinn, Estonia                         ...   \n",
       "6                  NaN     VENEZIA. CARNEVALE 2019   \n",
       "13                 NaN  Venice - Ponte dei Sospiri   \n",
       "...                ...                         ...   \n",
       "4981     Milano, Italy             20210520_145859   \n",
       "4982     Milano, Italy             20210520_145745   \n",
       "4983     Milano, Italy             20210520_164727   \n",
       "4984     Milano, Italy             20210520_155646   \n",
       "4985     Milano, Italy             20210520_155525   \n",
       "\n",
       "                                            description  ...  \\\n",
       "3                                         Venice, Italy  ...   \n",
       "4                                         Venice, Italy  ...   \n",
       "5                                         Venice, Italy  ...   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  ...   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...  ...   \n",
       "...                                                 ...  ...   \n",
       "4981                                                NaN  ...   \n",
       "4982                                                NaN  ...   \n",
       "4983                                                NaN  ...   \n",
       "4984                                                NaN  ...   \n",
       "4985                                                NaN  ...   \n",
       "\n",
       "           neighbourhood                                                url  \\\n",
       "3          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870743082/   \n",
       "4          Piazzale Roma   https://www.flickr.com/photos/ruudi/51871789478/   \n",
       "5          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870742942/   \n",
       "6              San Marco  https://www.flickr.com/photos/64148082@N02/518...   \n",
       "13             San Marco  https://www.flickr.com/photos/glolucach/518631...   \n",
       "...                  ...                                                ...   \n",
       "4981  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233430...   \n",
       "4982            Castello  https://www.flickr.com/photos/badswan/51234286...   \n",
       "4983  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234294...   \n",
       "4984  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234291...   \n",
       "4985  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233434...   \n",
       "\n",
       "      local                                           tag_list tag_len    NOT  \\\n",
       "3     False        [adriaticsea, italy, venice, bridge, water]       5  False   \n",
       "4     False                       [adriaticsea, italy, venice]       3  False   \n",
       "5     False                [adriaticsea, italy, venice, water]       4  False   \n",
       "6     False                                                 []       0  False   \n",
       "13    False  [ponte, pont, canal, river, city, history, cul...      16  False   \n",
       "...     ...                                                ...     ...    ...   \n",
       "4981  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4982  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4983  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4984  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4985  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "\n",
       "      year                time week  unique_week  \n",
       "3     2021 2021-10-05 14:45:41   40       202140  \n",
       "4     2021 2021-10-05 14:46:05   40       202140  \n",
       "5     2021 2021-10-05 15:13:13   40       202140  \n",
       "6     2019 2019-02-28 14:11:02    9       201909  \n",
       "13    2022 2022-01-20 18:14:39    3       202203  \n",
       "...    ...                 ...  ...          ...  \n",
       "4981  2021 2021-05-20 14:58:59   20       202120  \n",
       "4982  2021 2021-05-20 14:57:46   20       202120  \n",
       "4983  2021 2021-05-20 16:47:27   20       202120  \n",
       "4984  2021 2021-05-20 15:56:46   20       202120  \n",
       "4985  2021 2021-05-20 15:55:25   20       202120  \n",
       "\n",
       "[2951 rows x 27 columns]"
      ]
     },
     "execution_count": 849,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e6b1625",
   "metadata": {},
   "source": [
    "### Creating a New Dataset of data with text descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "553bcd3c",
   "metadata": {},
   "source": [
    "#### Filtering out an initial subset with text descriptions"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1e82c85",
   "metadata": {},
   "source": [
    "In total, 2801 out of 2951 complete posts have at least one text field (description, title, or tags) that is not empty."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "d2de58fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>neighbourhood</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870743082/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, bridge, water]</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51871789478/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice]</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>Piazzale Roma</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870742942/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, water]</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>...</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/64148082@N02/518...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>9</td>\n",
       "      <td>201909</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51863124066</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>186704588@N07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venice - Ponte dei Sospiri</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>...</td>\n",
       "      <td>San Marco</td>\n",
       "      <td>https://www.flickr.com/photos/glolucach/518631...</td>\n",
       "      <td>False</td>\n",
       "      <td>[ponte, pont, canal, river, city, history, cul...</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-01-20 18:14:39</td>\n",
       "      <td>3</td>\n",
       "      <td>202203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>51233430558</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233430...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 14:58:59</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>51234286630</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Castello</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234286...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 14:57:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>51234294135</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_164727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234294...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 16:47:27</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>51234291635</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51234291...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 15:56:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>51233434888</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>Quartiere S. Elena</td>\n",
       "      <td>https://www.flickr.com/photos/badswan/51233434...</td>\n",
       "      <td>False</td>\n",
       "      <td>[biennale, architettura, architecture, venezia...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-05-20 15:55:25</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2801 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "13    51863124066            1   \n",
       "...           ...          ...   \n",
       "4981  51233430558            1   \n",
       "4982  51234286630            1   \n",
       "4983  51234294135            1   \n",
       "4984  51234291635            1   \n",
       "4985  51233434888            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "13    https://live.staticflickr.com/65535/5186312406...  186704588@N07   \n",
       "...                                                 ...            ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   83293103@N00   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   83293103@N00   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   83293103@N00   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   83293103@N00   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   83293103@N00   \n",
       "\n",
       "             owner_loc                       title  \\\n",
       "3     Tallinn, Estonia                         ...   \n",
       "4     Tallinn, Estonia                         ...   \n",
       "5     Tallinn, Estonia                         ...   \n",
       "6                  NaN     VENEZIA. CARNEVALE 2019   \n",
       "13                 NaN  Venice - Ponte dei Sospiri   \n",
       "...                ...                         ...   \n",
       "4981     Milano, Italy             20210520_145859   \n",
       "4982     Milano, Italy             20210520_145745   \n",
       "4983     Milano, Italy             20210520_164727   \n",
       "4984     Milano, Italy             20210520_155646   \n",
       "4985     Milano, Italy             20210520_155525   \n",
       "\n",
       "                                            description  ...  \\\n",
       "3                                         Venice, Italy  ...   \n",
       "4                                         Venice, Italy  ...   \n",
       "5                                         Venice, Italy  ...   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  ...   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...  ...   \n",
       "...                                                 ...  ...   \n",
       "4981                                                NaN  ...   \n",
       "4982                                                NaN  ...   \n",
       "4983                                                NaN  ...   \n",
       "4984                                                NaN  ...   \n",
       "4985                                                NaN  ...   \n",
       "\n",
       "           neighbourhood                                                url  \\\n",
       "3          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870743082/   \n",
       "4          Piazzale Roma   https://www.flickr.com/photos/ruudi/51871789478/   \n",
       "5          Piazzale Roma   https://www.flickr.com/photos/ruudi/51870742942/   \n",
       "6              San Marco  https://www.flickr.com/photos/64148082@N02/518...   \n",
       "13             San Marco  https://www.flickr.com/photos/glolucach/518631...   \n",
       "...                  ...                                                ...   \n",
       "4981  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233430...   \n",
       "4982            Castello  https://www.flickr.com/photos/badswan/51234286...   \n",
       "4983  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234294...   \n",
       "4984  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51234291...   \n",
       "4985  Quartiere S. Elena  https://www.flickr.com/photos/badswan/51233434...   \n",
       "\n",
       "      local                                           tag_list tag_len    NOT  \\\n",
       "3     False        [adriaticsea, italy, venice, bridge, water]       5  False   \n",
       "4     False                       [adriaticsea, italy, venice]       3  False   \n",
       "5     False                [adriaticsea, italy, venice, water]       4  False   \n",
       "6     False                                                 []       0  False   \n",
       "13    False  [ponte, pont, canal, river, city, history, cul...      16  False   \n",
       "...     ...                                                ...     ...    ...   \n",
       "4981  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4982  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4983  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4984  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "4985  False  [biennale, architettura, architecture, venezia...       5  False   \n",
       "\n",
       "      year                time week  unique_week  \n",
       "3     2021 2021-10-05 14:45:41   40       202140  \n",
       "4     2021 2021-10-05 14:46:05   40       202140  \n",
       "5     2021 2021-10-05 15:13:13   40       202140  \n",
       "6     2019 2019-02-28 14:11:02    9       201909  \n",
       "13    2022 2022-01-20 18:14:39    3       202203  \n",
       "...    ...                 ...  ...          ...  \n",
       "4981  2021 2021-05-20 14:58:59   20       202120  \n",
       "4982  2021 2021-05-20 14:57:46   20       202120  \n",
       "4983  2021 2021-05-20 16:47:27   20       202120  \n",
       "4984  2021 2021-05-20 15:56:46   20       202120  \n",
       "4985  2021 2021-05-20 15:55:25   20       202120  \n",
       "\n",
       "[2801 rows x 27 columns]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new = df[~((df['description'].isnull())&(df['title'].isnull())&(df['tag_len']==0))]\n",
    "df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "0d954109",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new['description'] = df_new['description'].fillna('')\n",
    "df_new['title'] = df_new['title'].fillna('')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "10a120e4",
   "metadata": {},
   "source": [
    "Merging the three types of text data into one complete paragraph with the order of description -> title -> tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 302,
   "id": "53fea2f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_new['text_data'] = df_new.apply(lambda x: str(x['description']) + '. It has title of ' + str(x['title']) + '. It has tags of ' + ', '.join(x['tag_list']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 303,
   "id": "5cf19896",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3       Venice, Italy. It has title of .... It has tag...\n",
       "4       Venice, Italy. It has title of .... It has tag...\n",
       "5       Venice, Italy. It has title of .... It has tag...\n",
       "6       IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...\n",
       "13      Ponte dei Sospiri. Il ponte dei Sospiri fu rea...\n",
       "                              ...                        \n",
       "4981    . It has title of 20210520_145859. It has tags...\n",
       "4982    . It has title of 20210520_145745. It has tags...\n",
       "4983    . It has title of 20210520_164727. It has tags...\n",
       "4984    . It has title of 20210520_155646. It has tags...\n",
       "4985    . It has title of 20210520_155525. It has tags...\n",
       "Name: text_data, Length: 2801, dtype: object"
      ]
     },
     "execution_count": 303,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_new['text_data']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ccaab49e",
   "metadata": {},
   "source": [
    "Only 1952 out of the 2801 text data are unique to each other, i.e., not repetitive."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "ee5b160f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1952"
      ]
     },
     "execution_count": 304,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_new['text_data'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf10b4b4",
   "metadata": {},
   "source": [
    "#### Filtering out a sub-dataset with unique text data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef0abe8f",
   "metadata": {},
   "source": [
    "Only the *FIRST* data point is kept while there are duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 305,
   "id": "4af81920",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1952"
      ]
     },
     "execution_count": 305,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates = df_new.drop_duplicates(subset='text_data',keep='first')\n",
    "len(candidates)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a97fe51",
   "metadata": {},
   "source": [
    "### Detecting Different Languages in the Texts and Translating all to English"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e2e3eda",
   "metadata": {},
   "source": [
    "#### Library Imports with Google Translator API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 418,
   "id": "be9e07d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import detect\n",
    "from deep_translator import GoogleTranslator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 420,
   "id": "7cafab73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'I drink a cappucino'"
      ]
     },
     "execution_count": 420,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translated = GoogleTranslator(source='auto', target='en').translate(\"io bevo uno cappucino\")\n",
    "translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 426,
   "id": "a378859b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Towards the lagoon'"
      ]
     },
     "execution_count": 426,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translated = GoogleTranslator(source='auto', target='en').translate('Verso la laguna')\n",
    "translated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "66a604cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'it'"
      ]
     },
     "execution_count": 243,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "detect(candidates['text_data'].iloc[4])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c1aa2da",
   "metadata": {},
   "source": [
    "#### Text normalization and processing, Detecting Languages, and Performing Translations"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6594327c",
   "metadata": {},
   "source": [
    "A new sentence occur when there is a period(.), exclamation mark(!), question mark(?) or semi-colon(;). The html signs, http addresses, formating elements of the texts, special quotation marks, and copyright symbol is removed. Only a sentence piece with no less than 3 words are considered for translation task.\n",
    "\n",
    "Italian is distinguished from the other non-English texts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 622,
   "id": "268a6a2e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n",
      "No features in text.\n"
     ]
    }
   ],
   "source": [
    "candi_dict_en = {}\n",
    "candi_dict_it = {}\n",
    "candi_dict_it_trans = {}\n",
    "candi_dict_other = {}\n",
    "candi_dict_other_trans = {}\n",
    "origin_en = []\n",
    "origin_it = []\n",
    "origin_other = []\n",
    "origin_other_lan = []\n",
    "for i in candidates['text_data'].index:\n",
    "    for sent in candidates['text_data'][i].replace('St.', 'St').replace('S.', 'S').replace('!', '.').replace('?', '.').replace(';', '.').split('. '):\n",
    "        try:\n",
    "            sent = re.sub(r'\\<.*\\>', '', sent)\n",
    "            sent = re.sub(r'http[s]?://\\S+', '', sent)\n",
    "            sent = re.sub(r'--+', '', sent)\n",
    "            sent = re.sub(r'\\*+', '', sent)\n",
    "            sent = sent.replace('\\n', ' ').replace('&quot', '').replace('&amp', '').replace('#', '').replace('\\\\','')\n",
    "            sent = sent.replace('“', ' ').replace('„', ' ').replace('”', ' ')\n",
    "            sent = sent.replace('©', '')\n",
    "            for senty in sent.split('. '):\n",
    "                tt = 0\n",
    "                if len(senty.split())<3:\n",
    "                    continue\n",
    "                if senty.startswith('It has title of') or senty.startswith('It has tags of'):\n",
    "                    senti = ' '.join(senty.split()[4:])\n",
    "                    tt = 1\n",
    "                else:\n",
    "                    senti = senty\n",
    "                if detect(senti)=='en' and senty.strip() not in candi_dict_en:\n",
    "                    candi_dict_en[len(candi_dict_en)] = senty.strip()\n",
    "                    origin_en.append(i)\n",
    "                elif detect(senti)=='it' and senty.strip() not in candi_dict_it:\n",
    "                    candi_dict_it[len(candi_dict_it)] = senty.strip()\n",
    "                    if tt==1:\n",
    "                        candi_dict_it_trans[len(candi_dict_it_trans)] = ' '.join(senty.split()[:4]) + ' ' + GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    else:\n",
    "                        candi_dict_it_trans[len(candi_dict_it_trans)] = GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    origin_it.append(i)\n",
    "                elif detect(senti)!='it' and detect(senti)!='en' and senty.strip() not in candi_dict_other:\n",
    "                    candi_dict_other[len(candi_dict_other)] = senty.strip()\n",
    "                    if tt==1:\n",
    "                        candi_dict_other_trans[len(candi_dict_other_trans)] = ' '.join(senty.split()[:4]) + ' ' + GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    else:\n",
    "                        candi_dict_other_trans[len(candi_dict_other_trans)] = GoogleTranslator(source='auto', target='en').translate(senti.strip())\n",
    "                    origin_other.append(i)\n",
    "                    origin_other_lan.append(detect(senty))\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "            continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55993d2b",
   "metadata": {},
   "source": [
    "In total, 754, 1246, and 926 sentences are detected as English, Italian, and Other languages, respectively. After filtering out text data with less than 3 words, 1913 out of 1952 sentences remained."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 850,
   "id": "99df49d4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(754, 1246, 926, 1913)"
      ]
     },
     "execution_count": 850,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(origin_en)), len(set(origin_it)), len(set(origin_other)), len(set(origin_en).union(set(origin_it)).union(set(origin_other)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a65381f9",
   "metadata": {},
   "source": [
    "#### Filtering out Non-relevant Text data for Heritage Values and Attributes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7a7b491",
   "metadata": {},
   "source": [
    "The stop word list includes camera brands, file names, advertisements, and place-holder formats for specific types of data (e.g., scanned from analog)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 625,
   "id": "d36bd444",
   "metadata": {},
   "outputs": [],
   "source": [
    "stoplist = ['sony', 'canon ', 'iphone','dilution','chesterton','leica','nikon','fuji','img','buy','steam','rolleiflex','ilford',\n",
    "            'sigma','e6', 'click', 'gear','luminance','fs','mercedes','spp','actv','atvo','samyang','thank','notes','twitter',\n",
    "            'gaz','oebb','follow','scan','voightlander','dsc', 'm1x', 'p10', 'f21', 'dhg', 'd01','l10', '_mg', '[','psx', 'zmr',\n",
    "            'discount', 'c_','shot','imag1','imag2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 626,
   "id": "f7be7282",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_dict_en = {}\n",
    "filter_dict_it = {}\n",
    "filter_dict_other = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 627,
   "id": "e334e44e",
   "metadata": {},
   "outputs": [],
   "source": [
    "origindicts = [candi_dict_en, candi_dict_it_trans, candi_dict_other_trans]\n",
    "targdicts = [filter_dict_en, filter_dict_it, filter_dict_other]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 628,
   "id": "e6b6d7da",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7 CANON EOS 6D Mark II con ob canon \n",
      "8 CANON EF 70-300 f./4-5,6 IS USM canon \n",
      "29 CANON EOS 6D Mark II con ob canon \n",
      "30 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "56 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "57 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0706.jpg sony\n",
      "100 It has title of 1981 European Trafalgar tour 203 Venice, Italy IMAG 0229.jpg sony\n",
      "102 It has title of 1981 European Trafalgar tour 202 Venice, Italy IMAG 0228.jpg sony\n",
      "104 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "105 It has title of 1981 European Trafalgar tour 201 Venice, Italy IMAG 0227.jpg sony\n",
      "108 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "109 It has title of 1981 European Trafalgar tour 200 Venice, Italy IMAG 0226.jpg sony\n",
      "112 It has title of 1981 European Trafalgar tour 199 Venice, Italy IMAG 0225.jpg sony\n",
      "114 It has title of 1981 European Trafalgar tour 196 Venice, Italy IMAG 0222.jpg sony\n",
      "116 It has title of 1981 European Trafalgar tour 195 Venice, Italy IMAG 0221.jpg sony\n",
      "119 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "120 It has title of 1981 European Trafalgar tour 198 Venice, Italy IMAG 0224.jpg sony\n",
      "178 Scanned from a half frame Kodachrome slide (transparency) scan\n",
      "198 IMG_5365-ed.JPG, Seafront at San Marco img\n",
      "203 Scan of an analog photo taken in 1993 scan\n",
      "204 Scan of an analog photo taken in May 1993 scan\n",
      "207 Scan of an analog photo taken in May 1993 scan\n",
      "210 Scan of an analog photo taken in May 1993 scan\n",
      "211 Scan of an analog photo taken in May 1993 scan\n",
      "213 Scan of an analog photo taken in May 1993 scan\n",
      "214 Scan of an analog photo taken in May 1993 scan\n",
      "216 Scan of an analog photo taken in May 1993 scan\n",
      "217 Scan of an analog photo taken in May 1993 scan\n",
      "219 Scan of an analog photo taken in 1993 scan\n",
      "220 Scan of an analog photo taken in May 1993 scan\n",
      "246 Scan of an analog photo taken in May 1993 scan\n",
      "247 CANON EOS 6D Mark II con ob canon \n",
      "248 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "255 Scan of an analog photo taken in May 1993  The Madonna and Child are depicted between the Saints John and Mark scan\n",
      "263 Scan of an analog photo taken in May 1993 scan\n",
      "267 Scan of an analog photo taken in May 1993 scan\n",
      "268 Scan of an analog photo taken in May 1993 scan\n",
      "269 Scan of an analog photo taken in May 1993 scan\n",
      "270 Scan of an analog photo taken in May 1993 scan\n",
      "271 Scan of an analog photo taken in May 2993 scan\n",
      "272 Scan of an analog photo taken in May 1993 scan\n",
      "273 Scan of an analog photo taken in May 1993 scan\n",
      "274 Scan of an analog photo taken in May 1993 scan\n",
      "275 Scan of an analog photo taken in May 1993 scan\n",
      "276 Scan of an analog photo taken in May 1993 scan\n",
      "277 Scan of an analog photo taken in May 1993 scan\n",
      "279 Scan of an analog photo taken in 1993 scan\n",
      "281 Scan of an analog photo taken in 1993 scan\n",
      "282 Scan of an analog photo taken in 1993 scan\n",
      "283 Scan of an analog photo taken in 1993 scan\n",
      "302 FS Class E.464 no fs\n",
      "303 FS Class E.414 no fs\n",
      "312 Scan of an analog photo taken in May 1993 scan\n",
      "318 Scan of an analog photo taken in May 1993 scan\n",
      "319 Scan of an analog photo taken in May 1993 scan\n",
      "320 Scan of an analog photo taken in May 1993 scan\n",
      "321 Scan of an analog photo taken in May 2993 scan\n",
      "323 Scan of an analog photo taken in May 1993 scan\n",
      "450 Thank you for your kind comments thank\n",
      "475 It has title of Thank you Francesco for the beverage. thank\n",
      "658 CANON EOS 6D Mark II con ob canon \n",
      "669 Scan of an analog photo taken in May 1993 scan\n",
      "681 Scan of an analog photo taken in May 1993 scan\n",
      "683 Scan of an analog photo taken in May 1993 scan\n",
      "691 Scan of an analog photo taken in May 1993 scan\n",
      "697 Scan of an analog photo taken in May 1993 scan\n",
      "698 Scan of an analog photo taken in May 1993 scan\n",
      "738 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "746 Scan of an analog photo taken in May 1993 scan\n",
      "747 Scan of an analog photo taken in May 1993 scan\n",
      "877 iPhone 8 Plus iphone\n",
      "932 Scan of an analog photo taken in May 1993 scan\n",
      "933 Scan of an analog photo taken in May 1993 scan\n",
      "954 Scan of an analog photo taken in May 1993 scan\n",
      "975 CANON EOS 6D Mark II con ob canon \n",
      "976 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "1013 CANON EOS 6D Mark II con ob canon \n",
      "1014 CANON EF 70-300 f./4-5,6 IS USM canon \n",
      "1053 CANON EOS 6D Mark II con ob canon \n",
      "1054 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "1102 Scanned from a Kodak Gold 100 35mm thingy and slightly fiddled with scan\n",
      "1103 Shot with a Pentax LX (don't ask for any more, the original is from my first visit in 1987) shot\n",
      "1146 Thanks very much for stopping by, and for sticking with me while I've been away thank\n",
      "1235 It has title of IMG_20200905_092455_BURST001_COVER img\n",
      "1238 It has title of Follow The Lines follow\n",
      "1290 CANON EOS 6D Mark II con ob canon \n",
      "1291 CANON EF 70-300 f./4-5,6 IS USM canon \n",
      "1293 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0719.jpg sony\n",
      "1296 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "1297 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0709.jpg sony\n",
      "1396 IMG_4761-ed.JPG, Boy with Frog, Charles Ray 2009, img\n",
      "1399 Scan of an analog photo taken in May 1993 scan\n",
      "1400 Scan of an analog photo taken in May 1993 scan\n",
      "1401 Scan of an analog photo taken in 1993 scan\n",
      "1403 Scan of an analog photo taken in May 1993 scan\n",
      "1409 Scan of an analog photo taken in May 1993 scan\n",
      "1410 Scan of an analog photo taken in May 1993 scan\n",
      "1425 Scan of an analog photo taken in May 1993 scan\n",
      "1426 Scan of an analog photo taken in May 1993 scan\n",
      "1427 Scan of an analog photo taken in 1993 scan\n",
      "1430 Scan of an analog photo taken in May 1993 scan\n",
      "1432 Scan of an analog photo taken in 1993 scan\n",
      "1433 Scan of an analog photo taken in May 1993 scan\n",
      "1434 Scan of an analog photo taken in 1993 scan\n",
      "1435 Scan of an analog photo taken in May 1993 scan\n",
      "1436 Scan of an analog photo taken in May 1993 scan\n",
      "1437 Scan of an analog photo taken in May 1993 scan\n",
      "1438 Scan of an analog photo taken in May 1993 scan\n",
      "1439 Scan of an analog photo taken in May 1993 scan\n",
      "1440 Scan of an analog photo taken in May 1993 scan\n",
      "1442 Scan of an analog photo taken in May 1993 scan\n",
      "1443 Scan of an analog photo taken in May 1993 scan\n",
      "1444 Scan of an analog photo taken in May 1993 scan\n",
      "1446 FS Class E.464 no fs\n",
      "1447 FS Class E.464 no fs\n",
      "1449 Scan of an analog photo taken in May 1993 scan\n",
      "1450 Scan of an analog photo taken in 1993 scan\n",
      "1451 Scan of an analog photo taken in 1993 scan\n",
      "1452 Scan of an analog photo taken in 1993 scan\n",
      "1454 Scan of an analog photo taken in 1993 scan\n",
      "1474 CANON EOS 6D Mark II con ob canon \n",
      "1475 CANON EF 70-300 f./4-5,6 IS USM canon \n",
      "1476 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "1477 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0714.jpg sony\n",
      "1540 Scan of an analog photo taken in May 1993 scan\n",
      "1541 Scan of an analog photo taken in 1993 scan\n",
      "1542 Scan of an analog photo taken in May 1993 scan\n",
      "1543 Scan of an analog photo taken in May 1993 scan\n",
      "1544 CANON EOS 6D Mark II con ob canon \n",
      "1545 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "1547 Scan of an analog photo taken in 1993 scan\n",
      "1549 Scan of an analog photo taken in May 1993 scan\n",
      "1550 Scan of an analog photo taken in May 1993 scan\n",
      "1582 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "1583 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0710.jpg sony\n",
      "1588 Scanned from a half frame Kodachrome slide (transparency) scan\n",
      "1614 Scan of an analog photo taken in May 1993 scan\n",
      "1615 Scan of an analog photo taken in 1993 scan\n",
      "1616 Scan of an analog photo taken in 1993 scan\n",
      "1631 Scan of an analog photo taken in May 1993 scan\n",
      "1633 Scan of an analog photo taken in May 1993 scan\n",
      "1634 Scan of an analog photo taken in May 1993 scan\n",
      "1644 Scan of an analog photo taken in May 1993 scan\n",
      "1645 Scan of an analog photo taken in May 1993 scan\n",
      "1646 Scan of an analog photo taken in May 1993 scan\n",
      "1648 Scan of an analog photo taken in May 1993 scan\n",
      "1649 Scan of an analog photo taken in May 1993  .San Zanipolo scan\n",
      "1652 Scan of an analog photo taken in May 1993 scan\n",
      "1655 Scan of an analog photo taken in May 1993 scan\n",
      "1656 Scan of an analog photo taken in May 1993 scan\n",
      "1660 Scan of an analog photo taken in May 1993  The Portrait of the Four Tetrarchs is a porphyry sculpture group of four Roman emperors dating from around 300 AD scan\n",
      "1670 Scan of an analog photo taken in May 1993 scan\n",
      "1671 Scan of an analog photo taken in May 1993 scan\n",
      "1684 SIGMA 10-20 f./4-5,6 EX DC HSM sigma\n",
      "1685 Canon EOS5, EF 17-40, Ilford XP2 400 canon \n",
      "1708 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "1709 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0717.jpg sony\n",
      "1711 It has title of 1983-01 European Trafalgar Tour  Italy, Venice IMAG0711.jpg sony\n",
      "1730 Scan of an analog photo taken in May 1993 scan\n",
      "1731 Scan of an analog photo taken in May 1993 scan\n",
      "1742 Canon EOS 5D Mark III Canon EF 17-40 17mm - 6x32s - f/5.6 - ISO 100  __________________________ canon \n",
      "1748 FS Class E.402 no fs\n",
      "1754 CANON EF 24-85 f./3,5-4,5 USM canon \n",
      "1758 Scan of an analog photo taken in 1993 scan\n",
      "1759 Scan of an analog photo taken in May 1993 scan\n",
      "1760 Scan of an analog photo taken in May 1993 scan\n",
      "1761 Scan of an analog photo taken in 1993 scan\n",
      "1762 Scan of an analog photo taken in 1993 scan\n",
      "1764 Scan of an analog photo taken in 1993 scan\n",
      "1765 Scan of an analog photo taken in 1993 scan\n",
      "1767 CANON EOS 6D Mark II con ob canon \n",
      "1772 Sony A7RII, Zeiss Loxia 21mm f2.8 sony\n",
      "1829 Scan of an analog photo taken in May 1997 scan\n",
      "1830 Scan of an analog photo taken in May 1997 scan\n",
      "1832 Scan of an analog photo taken in May 1997 scan\n",
      "1834 Scan of an analog photo taken in May 1997 scan\n",
      "1835 Scan of an analog photo taken in May 1997 scan\n",
      "1837 Scan of an analog photo taken in May 1997 scan\n",
      "1838 Scan of an analog photo taken in May 1997 scan\n",
      "1839 Scan of an analog photo taken in May 1997 scan\n",
      "82 It has title of 2021-11-FL-2111 Venice-4 sony\n",
      "83 It has title of 2021-11-FL-2111 Venice-5 sony\n",
      "84 It has title of 2021-11-FL-2111 Venice-6 sony\n",
      "85 It has title of 2021-11-FL-2111 Venice-47 sony\n",
      "262 It has title of 20210731 VeniceOne 316 sony\n",
      "466 It has title of _MG_7643 - Golden Venice / Piazza San Marco _mg\n",
      "468 It has title of IMG_0621 - San Giorgio Maggiore img\n",
      "470 It has title of _MG_7890 - Piazza San Marco _mg\n",
      "595 iPhone 8 Plus 4911.1 Venetian cat next to sign .Don't Sit Here Thanks. iphone\n",
      "708 Notes taken from Wikipedia, the Free Encyclopedia CANON EOS 6D Mark II with ob notes\n",
      "835 It has title of 1216 011 at Venezia Santa Lucia, 21 April 2012, sony\n",
      "1410 It has title of 2021-11-FL-2111 Venice-11 sony\n",
      "1412 It has title of 2021-11-FL-2111 Venice-3 sony\n",
      "1413 It has title of 2021-11-FL-2111 Venice-14 sony\n",
      "1414 It has title of 2021-11-FL-2111 Venice-16 sony\n",
      "1415 It has title of 2021-11-FL-2111 Venice-28 sony\n",
      "1416 It has title of 2021-11-FL-2111 Venice-39 sony\n",
      "1417 It has title of 2021-11-FL-2111 Venice-33 sony\n",
      "1418 It has title of 2021-11-FL-2111 Venice-44 sony\n",
      "1419 It has title of 2021-11-FL-2111 Venice-45 sony\n",
      "1420 It has title of 2021-11-FL-2111 Venice-46 sony\n",
      "1583 It has title of 2021-11-FL-2111 Venice-29 sony\n",
      "1584 It has title of 2021-11-FL-2111 Venice-30 sony\n",
      "1585 It has title of 2021-11-FL-2111 Venice-43 sony\n",
      "1656 It has title of 2021-11-FL-2111 Venice-9 sony\n",
      "1657 It has title of 2021-11-FL-2111 Venice-12 sony\n",
      "1758 It has title of 2021-11-FL-2111 Venice-10 sony\n",
      "1759 It has title of 2021-11-FL-2111 Venice-18 sony\n",
      "1760 It has title of 2021-11-FL-2111 Venice-13 sony\n",
      "1761 It has title of 2021-11-FL-2111 Venice-17 sony\n",
      "1762 It has title of 2021-11-FL-2111 Venice-21 sony\n",
      "1763 It has title of 2021-11-FL-2111 Venice-23 sony\n",
      "1764 It has title of 2021-11-FL-2111 Venice-26 sony\n",
      "1765 It has title of 2021-11-FL-2111 Venice-40 sony\n",
      "1813 It has title of [Explore 01/11/21 11] Long exposure on San Girgio Maggiore - Venice [\n",
      "5 DISCOUNTS from my collaborators: Filters (-10% with the code CISCAR) Backpacks, tripods and accessories (-20% with the code CISCPRO2022) Software (-10€ with the code CISCAR) Star tracker MSM (-5% with the code CISCAR ) discount\n",
      "23 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "24 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "25 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "26 Scanner (Kodak Slide n scan) from Kodachrome slide scan\n",
      "28 It has title of IMG_20211119_095036 img\n",
      "29 It has title of IMG_20211119_113153 img\n",
      "30 It has title of IMG_20211119_115156 img\n",
      "31 It has title of IMG_20211119_120843 img\n",
      "32 It has title of IMG_20211119_120950 img\n",
      "33 It has title of IMG_20211119_121004 img\n",
      "34 It has title of IMG_20211119_121755 img\n",
      "37 It has title of IMG_20211119_113241 img\n",
      "38 It has title of IMG_20211119_114620 img\n",
      "39 It has title of IMG_20211119_164421 img\n",
      "40 It has title of IMG_20211119_164627 img\n",
      "41 It has title of IMG_20211119_165424 img\n",
      "42 It has title of IMG_20211119_165306 img\n",
      "43 It has title of IMG_20211119_193751 img\n",
      "44 It has title of IMG_20211120_115943 img\n",
      "45 It has title of IMG_20211120_151931 img\n",
      "46 It has title of IMG_20211120_155059 img\n",
      "47 It has title of IMG_20211120_160842 img\n",
      "48 It has title of IMG_20211120_161331 img\n",
      "49 It has title of IMG_20211120_164012 img\n",
      "50 It has title of IMG_20211121_213249 img\n",
      "53 It has title of IMG_20211119_120101 img\n",
      "54 It has title of IMG_20211119_191806 img\n",
      "55 It has title of IMG_20211120_160758 img\n",
      "56 It has title of IMG_20211120_161324 img\n",
      "96 It has title of 2021-11-FL-2111 Biennale Architecture Venice-10 sony\n",
      "99 It has title of f21.1143 f21\n",
      "100 It has title of 2021-11-FL-2111 Biennale Architecture Venice-4 sony\n",
      "102 It has title of 2021-11-FL-2111 Biennale Architecture Venice-9 sony\n",
      "113 It has title of zmr82020 zmr\n",
      "116 IMG_4878-ed, Locks of lovers on the Ponte dell 'Accademia img\n",
      "122 It has title of DSC_1725 dsc\n",
      "123 It has title of DSC_1758 dsc\n",
      "124 It has title of DSC_1759 dsc\n",
      "125 It has title of DSC_1761 dsc\n",
      "128 It has title of 2014-11-01 17.08.47L1001502 sony\n",
      "130 It has title of 2014-11-02 12:40:35L1001699 sony\n",
      "132 It has title of DSC_1721 dsc\n",
      "133 It has title of DSC_1732 dsc\n",
      "134 It has title of DSC_1738 dsc\n",
      "135 It has title of DSC_1751 dsc\n",
      "137 It has title of DSC_1741 dsc\n",
      "147 It has title of IMG_1787 img\n",
      "148 It has title of IMG_20211031_135442 img\n",
      "149 It has title of PSX_20211031_172212 psx\n",
      "151 Scan of an analog photo taken in May 1993  .San Zanipolo scan\n",
      "156 It has title of IMG_0951 img\n",
      "158 It has title of IMG_20211030_165148_337 img\n",
      "169 It has title of DSC09024 dsc\n",
      "171 It has title of DSC08904 dsc\n",
      "178 It has title of 2021 - Italy (363 of 517) sony\n",
      "179 It has title of 2021 - Italy (365 of 517) sony\n",
      "180 It has title of 2021 - Italy (370 of 517) sony\n",
      "181 It has title of 2021 - Italy (375 of 517) sony\n",
      "186 It has title of IMAG1995 imag1\n",
      "188 It has title of IMAG2029 imag2\n",
      "189 It has title of IMG_16 (116) img\n",
      "190 It has title of IMG_16 (117) img\n",
      "191 It has title of IMG_16 (119) img\n",
      "192 It has title of IMG_16 (123) img\n",
      "193 It has title of IMG_16 (124) img\n",
      "194 It has title of IMG_16 (133) img\n",
      "195 It has title of IMG_1015 img\n",
      "197 It has title of IMG_1061 img\n",
      "198 It has title of IMG_1156 img\n",
      "199 It has title of IMG_1202 img\n",
      "200 It has title of IMG_1255 img\n",
      "220 It has title of f21.1027 f21\n",
      "229 It has title of IMG_7923.jpg img\n",
      "230 It has title of IMG_7898.jpg img\n",
      "233 It has title of 20131115_flickr (223) sony\n",
      "235 It has title of 20131115_flickr (227) sony\n",
      "237 It has title of IMAG1726 imag1\n",
      "239 It has title of IMAG1725 imag1\n",
      "240 It has title of IMAG1787 imag1\n",
      "241 It has title of IMAG1737 imag1\n",
      "243 It has title of IMAG1856 imag1\n",
      "245 It has title of IMAG1913 imag1\n",
      "247 It has title of IMAG1932 imag1\n",
      "249 It has title of IMAG1985 imag1\n",
      "250 It has title of IMG_16 (111) img\n",
      "251 It has title of IMG_16 (114) img\n",
      "252 Scan of an analog photo taken in May 1993  .San Zanipolo scan\n",
      "256 It has title of IMG_7907.jpg img\n",
      "258 It has title of 20131115_flickr (228) sony\n",
      "260 It has title of IMAG1823 imag1\n",
      "262 It has title of IMAG1794 imag1\n",
      "264 It has title of IMAG1872 imag1\n",
      "265 It has title of IMG_16 (126) img\n",
      "266 It has title of IMG_16 (127) img\n",
      "267 It has title of IMG_1059 img\n",
      "268 It has title of IMG_1204 img\n",
      "269 It has title of IMG_1229 img\n",
      "270 It has title of IMG_1248 img\n",
      "271 It has title of IMG_1254 img\n",
      "279 It has title of DSC08680 dsc\n",
      "306 [Photo taken with a Topcon Unirex] [\n",
      "321 It has title of _MG_7916 - San Marco domes _mg\n",
      "323 It has title of _MG_7926 - Saint Jeremiah _mg\n",
      "341 Nikon D7000, Nikon 18-105 nikon\n",
      "345 It has title of IMG_20190410_072308 img\n",
      "346 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "349 It has title of f21.0168 f21\n",
      "350 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "351 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "354 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "357 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "370 CANON EF EF24-85mm f/3.5-4.5 USM canon \n",
      "375 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "378 It has title of IMG_20171114_140302 img\n",
      "379 It has title of IMG_20171114_135908 img\n",
      "387 It has title of DHG700.5 ERF dhg\n",
      "392 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "394 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "406 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "407 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "424 It has title of f21.0099 f21\n",
      "427 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "430 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "431 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "432 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "433 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "434 It has title of M1X00553a_DXO m1x\n",
      "435 It has title of M1X00551a_DXO m1x\n",
      "436 It has title of M1X00534a_DXO m1x\n",
      "443 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "445 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "449 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "454 It has title of 20180611-131346-DSC01552-LR sony\n",
      "455 It has title of 20180611-131500-DSC01556-LR sony\n",
      "456 It has title of 20180611-131513-DSC01558-LR sony\n",
      "457 It has title of 20180611-131523-DSC01560-LR sony\n",
      "458 It has title of 20180611-131528-DSC01561-LR sony\n",
      "459 It has title of 20180611-131533-DSC01562-LR sony\n",
      "460 It has title of 20180611-131544-DSC01563-LR sony\n",
      "461 It has title of 20180611-131620-DSC01564-LR sony\n",
      "462 It has title of 20180611-131703-DSC01565-LR sony\n",
      "463 It has title of 20180611-131711-DSC01567-LR sony\n",
      "464 It has title of 20180611-131727-DSC01569-LR sony\n",
      "465 It has title of 20180611-132216-DSC01572-LR sony\n",
      "466 It has title of 20180611-132233-DSC01573-LR sony\n",
      "469 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "470 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "476 It has title of IMG_20210224_142249-01-01-01 img\n",
      "494 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "496 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "501 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "502 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "504 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "505 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "510 It has title of DSC09409-1-1 dsc\n",
      "511 It has title of DSC08949-1 dsc\n",
      "512 It has title of DSC08952-1 dsc\n",
      "513 It has title of DSC08962-3 dsc\n",
      "514 It has title of DSC09053-1 dsc\n",
      "515 It has title of DSC09286-1 dsc\n",
      "516 It has title of DSC09292-1 dsc\n",
      "517 It has title of DSC09307-1 dsc\n",
      "518 It has title of DSC09518-1 dsc\n",
      "519 It has title of DSC09522-1 dsc\n",
      "520 It has title of DSC09525-1 dsc\n",
      "521 It has title of DSC09451-4 dsc\n",
      "522 It has title of DSC08949-1 dsc\n",
      "523 It has title of DSC09010-1 dsc\n",
      "524 It has title of DSC09041-1 dsc\n",
      "525 It has title of DSC09281-1 dsc\n",
      "526 It has title of DSC09300-1 dsc\n",
      "527 It has title of DSC09342 dsc\n",
      "528 It has title of DSC09344-1 dsc\n",
      "529 It has title of DSC09385-1 dsc\n",
      "530 It has title of DSC09374-1 dsc\n",
      "531 It has title of DSC09417-1 dsc\n",
      "532 It has title of DSC09506-1 dsc\n",
      "533 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "540 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "546 It has title of IMG_20200208_183400-01 img\n",
      "551 It has title of M1X00522_DNG_LR m1x\n",
      "552 It has title of M1X00490_DNG_LR m1x\n",
      "553 It has title of M1X00445a_DNG_LR m1x\n",
      "554 It has title of M1X00300_DNG_LR m1x\n",
      "555 It has title of M1X00201_DNG_LR m1x\n",
      "556 It has title of P1050230 p10\n",
      "557 It has title of P1050176 p10\n",
      "558 It has title of P1050223 p10\n",
      "566 It has title of DSC09163-1 dsc\n",
      "567 It has title of DSC09517-1 dsc\n",
      "568 It has title of DSC09519-1 dsc\n",
      "569 It has title of DSC09244-1 dsc\n",
      "570 It has title of DSC08982-1 dsc\n",
      "587 It has title of DSC09523-1 dsc\n",
      "588 It has title of DSC09524-1 dsc\n",
      "589 It has title of DSC09328-1 dsc\n",
      "590 It has title of DSC09326-1 dsc\n",
      "591 It has title of DSC09387-1 dsc\n",
      "624 It has title of 2014 Expires sony\n",
      "631 It has title of 12112015-_92A0563.jpg sony\n",
      "644 It has title of P1050164 p10\n",
      "645 It has title of P1050213 p10\n",
      "646 It has title of P1050233 p10\n",
      "680 It has title of L1020382 l10\n",
      "698 It has title of IMG_7456 img\n",
      "707 It has title of DSC_0035 dsc\n",
      "712 It has title of IMG_3728 img\n",
      "750 It has title of M1X00388_LR m1x\n",
      "751 It has title of M1X00389_LR m1x\n",
      "752 It has title of M1X00342_LR m1x\n",
      "753 It has title of M1X00330_LR m1x\n",
      "754 It has title of M1X00329_LR m1x\n",
      "755 It has title of M1X00295_LR m1x\n",
      "756 It has title of M1X00280_LR m1x\n",
      "757 It has title of M1X00269_LR m1x\n",
      "758 It has title of M1X00264_LR m1x\n",
      "759 It has title of M1X00257_LR m1x\n",
      "760 It has title of M1X00423_LR m1x\n",
      "794 It has title of D01Venice (2) d01\n",
      "795 It has title of D01Venice (3) d01\n",
      "796 It has title of D01Venice (5) d01\n",
      "797 It has title of D01Venice (6) d01\n",
      "811 It has title of IMG_20200905_090838 img\n",
      "812 It has title of IMG_20200905_090907 img\n",
      "813 It has title of IMG_20200905_091034 img\n",
      "814 It has title of IMG_20200905_093112 img\n",
      "815 It has title of IMG_20200905_094311 img\n",
      "816 It has title of IMG_20200905_095011 img\n",
      "817 It has title of IMG_20200905_095620 img\n",
      "818 It has title of IMG_20200905_102321 img\n",
      "819 It has title of IMG_20200905_103112 img\n",
      "820 It has title of IMG_20200905_103737 img\n",
      "821 It has title of IMG_20200904_154256 img\n",
      "822 It has title of IMG_20200904_172613 img\n",
      "823 It has title of IMG_20200904_094310 img\n",
      "824 It has title of IMG_20200904_142435 img\n",
      "825 It has title of IMG_20200904_142533 img\n",
      "826 It has title of IMG_20200904_143303 img\n",
      "827 It has title of IMG_20200904_143710 img\n",
      "828 It has title of IMG_20200904_150151 img\n",
      "847 ACTV nº402  Mercedes Citaro C2 G actv\n",
      "848 It has title of C_1465_03 c_\n",
      "873 Scanner (Kodak Slide n Scan) from Kodacolor negative scan\n",
      "876 It has title of IMG_20211119_095032 img\n",
      "877 It has title of IMG_20211119_102444 img\n",
      "878 It has title of IMG_20211120_160749 img\n",
      "892 It has title of 2021-11-FL-2111 Biennale Architecture Venice-2 sony\n",
      "894 It has title of 2021-11-FL-2111 Biennale Architecture Venice-3 sony\n",
      "896 It has title of 2021-11-FL-2111 Biennale Architecture Venice-8 sony\n",
      "917 It has title of 2014-11-01 18.48.08L100157 sony\n",
      "919 It has title of 2014-11-01 11.24.02L1001433 sony\n",
      "931 It has title of IMG_1017 img\n",
      "940 It has title of DSC_1711 dsc\n",
      "941 It has title of IMG_20211031_135529 img\n",
      "945 It has title of IMAG1920 imag1\n",
      "947 It has title of IMAG1930 imag1\n",
      "949 It has title of IMG_1756 img\n",
      "952 It has title of IMG_1794 img\n",
      "954 It has title of IMG_0963 img\n",
      "958 It has title of IMG_1024 img\n",
      "961 It has title of IMG_1803 img\n",
      "963 It has title of IMG_0989 img\n",
      "965 It has title of IMG_0996 img\n",
      "966 It has title of IMG_1015 img\n",
      "967 It has title of IMG_1751 img\n",
      "969 It has title of IMG_0978 img\n",
      "971 It has title of IMG_0980 img\n",
      "972 It has title of IMG_0983 img\n",
      "975 It has title of f21.1043 f21\n",
      "976 It has title of f21.1026 f21\n",
      "979 It has title of IMG_0958 img\n",
      "984 FS Class E.402.A no fs\n",
      "987 It has title of IMG_20210321_183321 img\n",
      "988 It has title of IMG_1748 img\n",
      "1004 It has title of IMG_20211119_113248 img\n",
      "1005 It has title of IMG_20211120_115936 img\n",
      "1007 It has title of IMG_20211119_101217 img\n",
      "1008 It has title of IMG_20211119_110940 img\n",
      "1009 It has title of IMG_20211119_115048 img\n",
      "1010 It has title of IMG_20211119_120518 img\n",
      "1011 It has title of IMG_20211119_164633 img\n",
      "1012 It has title of IMG_20211119_165447 img\n",
      "1013 It has title of IMG_20211120_161119 img\n",
      "1014 It has title of IMG_20211120_164351 img\n",
      "1015 It has title of IMG_20211119_121716 img\n",
      "1016 It has title of IMG_20211119_164626 img\n",
      "1017 It has title of IMG_20211119_190720 img\n",
      "1031 It has title of DSC_1736 dsc\n",
      "1033 It has title of IMG_20211030_164625 img\n",
      "1035 It has title of IMG_20211031_170555 img\n",
      "1036 Scan of an analogue photo taken in May 1993 The square is named after a church that was destroyed in 1837 and was dedicated to Archangel Michael scan\n",
      "1039 It has title of IMG_0970 img\n",
      "1041 It has title of IMG_1019 img\n",
      "1042 Scan of an analog photo taken in May 1993  .San Zanipolo scan\n",
      "1049 It has title of 2021 - Italy (372 of 517) sony\n",
      "1050 It has title of 2021 - Italy (376 of 517) sony\n",
      "1051 It has title of 2021 - Italy (383 of 517) sony\n",
      "1058 It has title of 2021-11-FL-2111 Biennale Architecture Venice-6 sony\n",
      "1069 It has title of IMG_16 (110) img\n",
      "1070 It has title of IMG_1237 img\n",
      "1075 It has title of IMG_7920.jpg img\n",
      "1076 FS ALe 426/506 .TAF fs\n",
      "1079 It has title of 20131115_flickr (220) sony\n",
      "1082 It has title of IMAG1884 imag1\n",
      "1083 It has title of IMG_16 (120) img\n",
      "1084 It has title of IMG_16 (132) img\n",
      "1085 It has title of IMG_16 (138) img\n",
      "1086 It has title of IMG_1022 img\n",
      "1088 It has title of IMG_1062 img\n",
      "1089 It has title of IMG_1201 img\n",
      "1090 It has title of IMG_1198 img\n",
      "1091 It has title of IMG_1203 img\n",
      "1103 CANON EOS 600D canon \n",
      "1109 It has title of DSC08595 dsc\n",
      "1111 It has title of DSC08494 dsc\n",
      "1113 It has title of DSC08376 dsc\n",
      "1115 It has title of 2021 - Italy (362 of 517) sony\n",
      "1116 It has title of 2021 - Italy (366 of 517) sony\n",
      "1117 It has title of 2021 - Italy (371 of 517) sony\n",
      "1122 It has title of IMG_20211119_101854 img\n",
      "1123 It has title of IMG_20211119_102143 img\n",
      "1124 It has title of IMG_20211119_112334 img\n",
      "1125 It has title of IMG_20211119_114642 img\n",
      "1126 It has title of IMG_20211119_121108 img\n",
      "1127 It has title of IMG_20211119_165412 img\n",
      "1128 It has title of f21.1060 f21\n",
      "1129 It has title of 2021-11-FL-2111 Biennale Architecture Venice-7 sony\n",
      "1132 It has title of 20061122-Venice - 003.jpg sony\n",
      "1146 It has title of DSC_1743 dsc\n",
      "1151 It has title of DSC_1753 dsc\n",
      "1155 Scan of an analog photo taken in May 1993  .San Zanipolo scan\n",
      "1167 It has title of IMG_7913.jpg img\n",
      "1168 It has title of IMG_7896.jpg img\n",
      "1172 It has title of IMAG1724 imag1\n",
      "1174 It has title of IMG_1339 img\n",
      "1177 CANON EOS RT canon \n",
      "1186 It has title of IMAG2041 imag2\n",
      "1187 It has title of IMG_16 (122) img\n",
      "1188 It has title of IMG_16 (128) img\n",
      "1192 It has title of IMAG1938 imag1\n",
      "1193 CANON EF 24-70 f./2.8 L USM canon \n",
      "1232 It has title of FS E646084 fs\n",
      "1234 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n",
      "1235 Fujifilm X-E3 Fujinon XF23mmF1.4 R  See  See fuji\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(origindicts)):\n",
    "    dic1 = origindicts[i]\n",
    "    dic2 = targdicts[i]\n",
    "    for k,v in dic1.items():\n",
    "        record = 1\n",
    "        if v.startswith('It has tags of'):\n",
    "            dic2[k] = v\n",
    "        elif v.startswith('It has title of'):\n",
    "            for w in stoplist:\n",
    "                if ' '.join(v.lower().split()[4:]).startswith(w) or ' '.join(v.lower().split()[4:])[:4].isdigit():\n",
    "                    record = 0\n",
    "                    print(k,v,w)\n",
    "                    break\n",
    "            if record==1:\n",
    "                dic2[k] = v\n",
    "        else:\n",
    "            for w in stoplist:\n",
    "                if v.lower().startswith(w) or v[:5].isdigit():\n",
    "                    record = 0\n",
    "                    print(k,v,w)\n",
    "                    break\n",
    "            if record==1:\n",
    "                dic2[k] = v"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dfe303d1",
   "metadata": {},
   "source": [
    "After the filtering, 1761 out of 1913 data points remained as data with valid text description."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 629,
   "id": "179eeaf2",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_en = [origin_en[k] for k in filter_dict_en]\n",
    "filter_it = [origin_it[k] for k in filter_dict_it]\n",
    "filter_other = [origin_other[k] for k in filter_dict_other]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 851,
   "id": "0fd8e4e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(640, 1215, 657, 1761)"
      ]
     },
     "execution_count": 851,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(filter_en)), len(set(filter_it)), len(set(filter_other)), len(set(filter_en).union(set(filter_it)).union(set(filter_other)))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8160f109",
   "metadata": {},
   "source": [
    "#### A Text-only Dataset with separate sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 840,
   "id": "ebdf18af",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_en = pd.DataFrame()\n",
    "filter_df_en['sentence'] = pd.Series(filter_dict_en)\n",
    "filter_df_en['origin'] = filter_en\n",
    "filter_df_en['lang'] = 'en'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 842,
   "id": "48ad73c0",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_it = pd.DataFrame()\n",
    "filter_df_it['sentence'] = pd.Series(filter_dict_it)\n",
    "filter_df_it['origin'] = filter_it\n",
    "filter_df_it['lang'] = 'it'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 843,
   "id": "ed92ed2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "filter_df_other = pd.DataFrame()\n",
    "filter_df_other['sentence'] = pd.Series(filter_dict_other)\n",
    "filter_df_other['origin'] = filter_other\n",
    "filter_df_other['lang'] = 'other'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2dc799cc",
   "metadata": {},
   "source": [
    "In total, 4557 sentences coming from 1761 original posts are saved for further text analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 846,
   "id": "eb7eb3f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = pd.concat([filter_df_en,filter_df_it,filter_df_other]).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 861,
   "id": "809c669a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1761"
      ]
     },
     "execution_count": 861,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences['origin'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 860,
   "id": "c02330ec",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>sentence</th>\n",
       "      <th>origin</th>\n",
       "      <th>lang</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, bri...</td>\n",
       "      <td>3</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice</td>\n",
       "      <td>4</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, water</td>\n",
       "      <td>5</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>THE KNIGHT WITH A GRIM LOOK   The Carnival of ...</td>\n",
       "      <td>6</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Who has never heard of it</td>\n",
       "      <td>6</td>\n",
       "      <td>en</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4552</th>\n",
       "      <td>1228</td>\n",
       "      <td>It has tags of venice, murano</td>\n",
       "      <td>4837</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4553</th>\n",
       "      <td>1229</td>\n",
       "      <td>It has tags of venice, murano</td>\n",
       "      <td>4838</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4554</th>\n",
       "      <td>1230</td>\n",
       "      <td>It has title of Gondolas on Canale Grande, Venice</td>\n",
       "      <td>4842</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4555</th>\n",
       "      <td>1231</td>\n",
       "      <td>It has title of Frontage</td>\n",
       "      <td>4857</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4556</th>\n",
       "      <td>1233</td>\n",
       "      <td>It has title of Venice, at sunset.</td>\n",
       "      <td>4866</td>\n",
       "      <td>other</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4557 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index                                           sentence  origin   lang\n",
       "0         0  It has tags of adriaticsea, italy, venice, bri...       3     en\n",
       "1         1          It has tags of adriaticsea, italy, venice       4     en\n",
       "2         2   It has tags of adriaticsea, italy, venice, water       5     en\n",
       "3         3  THE KNIGHT WITH A GRIM LOOK   The Carnival of ...       6     en\n",
       "4         4                          Who has never heard of it       6     en\n",
       "...     ...                                                ...     ...    ...\n",
       "4552   1228                      It has tags of venice, murano    4837  other\n",
       "4553   1229                      It has tags of venice, murano    4838  other\n",
       "4554   1230  It has title of Gondolas on Canale Grande, Venice    4842  other\n",
       "4555   1231                           It has title of Frontage    4857  other\n",
       "4556   1233                 It has title of Venice, at sunset.    4866  other\n",
       "\n",
       "[4557 rows x 4 columns]"
      ]
     },
     "execution_count": 860,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentences"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7117664",
   "metadata": {},
   "source": [
    "#### Finalizing the new Subset with valid textual data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 631,
   "id": "e5202e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new = candidates.loc[list(set(filter_en).union(set(filter_it)).union(set(filter_other)))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 632,
   "id": "bef16ecd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>url</th>\n",
       "      <th>local</th>\n",
       "      <th>tag_list</th>\n",
       "      <th>tag_len</th>\n",
       "      <th>NOT</th>\n",
       "      <th>year</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870743082/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, bridge, water]</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51871789478/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice]</td>\n",
       "      <td>3</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/ruudi/51870742942/</td>\n",
       "      <td>False</td>\n",
       "      <td>[adriaticsea, italy, venice, water]</td>\n",
       "      <td>4</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/64148082@N02/518...</td>\n",
       "      <td>False</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "      <td>2019</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>9</td>\n",
       "      <td>201909</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4106</th>\n",
       "      <td>51750158296</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>81438595@N00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Rialto Bridge, Venice, Italy, 1971</td>\n",
       "      <td>Scanned from a half frame Kodachrome slide (tr...</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/east_med_wandere...</td>\n",
       "      <td>False</td>\n",
       "      <td>[italy, venice, rialtobridge, kodachrome, worl...</td>\n",
       "      <td>5</td>\n",
       "      <td>False</td>\n",
       "      <td>1971</td>\n",
       "      <td>1971-01-01 00:00:00</td>\n",
       "      <td>53</td>\n",
       "      <td>197153</td>\n",
       "      <td>Scanned from a half frame Kodachrome slide (tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4057</th>\n",
       "      <td>51813044138</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>61033692@N00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Piove</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/magellano/518130...</td>\n",
       "      <td>False</td>\n",
       "      <td>[venezia, venice, venise, venedig, piazza, san...</td>\n",
       "      <td>16</td>\n",
       "      <td>False</td>\n",
       "      <td>2022</td>\n",
       "      <td>2022-01-05 18:56:59</td>\n",
       "      <td>1</td>\n",
       "      <td>202201</td>\n",
       "      <td>Venice, Italy. It has title of Piove. It has t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4058</th>\n",
       "      <td>51812495828</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>126584084@N05</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>slowly, quietly, Venezia</td>\n",
       "      <td>slowly, quietly ( langsam, leise ) in einer Au...</td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/126584084@N05/51...</td>\n",
       "      <td>False</td>\n",
       "      <td>[venezia]</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>2017</td>\n",
       "      <td>2017-09-03 13:02:45</td>\n",
       "      <td>35</td>\n",
       "      <td>201735</td>\n",
       "      <td>slowly, quietly ( langsam, leise ) in einer Au...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4075</th>\n",
       "      <td>51757171151</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>61579728@N05</td>\n",
       "      <td>London, UK</td>\n",
       "      <td>2021-11-FL-2111 Biennale Architecture Venice-6</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/acme_/51757171151/</td>\n",
       "      <td>False</td>\n",
       "      <td>[architecturebiennale, biennale, exhibition, g...</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-11-19 18:02:18</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of 2021-11-FL-2111 Biennale Arc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4087</th>\n",
       "      <td>51783729807</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>141429878@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Murano -Venezia</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/141429878@N03/51...</td>\n",
       "      <td>False</td>\n",
       "      <td>[murano, venezia, italia, vetro, glass, nikon, d]</td>\n",
       "      <td>7</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-12-28 13:02:42</td>\n",
       "      <td>52</td>\n",
       "      <td>202152</td>\n",
       "      <td>. It has title of Murano -Venezia. It has tags...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4093</th>\n",
       "      <td>51770600998</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>123016824@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venise, le marché</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>https://www.flickr.com/photos/123016824@N05/51...</td>\n",
       "      <td>False</td>\n",
       "      <td>[whl]</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "      <td>2021</td>\n",
       "      <td>2021-09-29 18:55:59</td>\n",
       "      <td>39</td>\n",
       "      <td>202139</td>\n",
       "      <td>. It has title of Venise, le marché. It has ta...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1761 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "4106  51750158296            1   \n",
       "...           ...          ...   \n",
       "4057  51813044138            1   \n",
       "4058  51812495828            1   \n",
       "4075  51757171151            1   \n",
       "4087  51783729807            1   \n",
       "4093  51770600998            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   81438595@N00   \n",
       "...                                                 ...            ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   61033692@N00   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...  126584084@N05   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   61579728@N05   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...  141429878@N03   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...  123016824@N05   \n",
       "\n",
       "             owner_loc                                           title  \\\n",
       "3     Tallinn, Estonia                                             ...   \n",
       "4     Tallinn, Estonia                                             ...   \n",
       "5     Tallinn, Estonia                                             ...   \n",
       "6                  NaN                         VENEZIA. CARNEVALE 2019   \n",
       "4106               NaN              Rialto Bridge, Venice, Italy, 1971   \n",
       "...                ...                                             ...   \n",
       "4057               NaN                                           Piove   \n",
       "4058       Deutschland                        slowly, quietly, Venezia   \n",
       "4075        London, UK  2021-11-FL-2111 Biennale Architecture Venice-6   \n",
       "4087               NaN                                 Murano -Venezia   \n",
       "4093               NaN                               Venise, le marché   \n",
       "\n",
       "                                            description  ...  \\\n",
       "3                                         Venice, Italy  ...   \n",
       "4                                         Venice, Italy  ...   \n",
       "5                                         Venice, Italy  ...   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  ...   \n",
       "4106  Scanned from a half frame Kodachrome slide (tr...  ...   \n",
       "...                                                 ...  ...   \n",
       "4057                                      Venice, Italy  ...   \n",
       "4058  slowly, quietly ( langsam, leise ) in einer Au...  ...   \n",
       "4075                                                     ...   \n",
       "4087                                                     ...   \n",
       "4093                                                     ...   \n",
       "\n",
       "                                                    url  local  \\\n",
       "3      https://www.flickr.com/photos/ruudi/51870743082/  False   \n",
       "4      https://www.flickr.com/photos/ruudi/51871789478/  False   \n",
       "5      https://www.flickr.com/photos/ruudi/51870742942/  False   \n",
       "6     https://www.flickr.com/photos/64148082@N02/518...  False   \n",
       "4106  https://www.flickr.com/photos/east_med_wandere...  False   \n",
       "...                                                 ...    ...   \n",
       "4057  https://www.flickr.com/photos/magellano/518130...  False   \n",
       "4058  https://www.flickr.com/photos/126584084@N05/51...  False   \n",
       "4075   https://www.flickr.com/photos/acme_/51757171151/  False   \n",
       "4087  https://www.flickr.com/photos/141429878@N03/51...  False   \n",
       "4093  https://www.flickr.com/photos/123016824@N05/51...  False   \n",
       "\n",
       "                                               tag_list  tag_len    NOT  year  \\\n",
       "3           [adriaticsea, italy, venice, bridge, water]        5  False  2021   \n",
       "4                          [adriaticsea, italy, venice]        3  False  2021   \n",
       "5                   [adriaticsea, italy, venice, water]        4  False  2021   \n",
       "6                                                    []        0  False  2019   \n",
       "4106  [italy, venice, rialtobridge, kodachrome, worl...        5  False  1971   \n",
       "...                                                 ...      ...    ...   ...   \n",
       "4057  [venezia, venice, venise, venedig, piazza, san...       16  False  2022   \n",
       "4058                                          [venezia]        1  False  2017   \n",
       "4075  [architecturebiennale, biennale, exhibition, g...        7  False  2021   \n",
       "4087  [murano, venezia, italia, vetro, glass, nikon, d]        7  False  2021   \n",
       "4093                                              [whl]        1  False  2021   \n",
       "\n",
       "                    time week unique_week  \\\n",
       "3    2021-10-05 14:45:41   40      202140   \n",
       "4    2021-10-05 14:46:05   40      202140   \n",
       "5    2021-10-05 15:13:13   40      202140   \n",
       "6    2019-02-28 14:11:02    9      201909   \n",
       "4106 1971-01-01 00:00:00   53      197153   \n",
       "...                  ...  ...         ...   \n",
       "4057 2022-01-05 18:56:59    1      202201   \n",
       "4058 2017-09-03 13:02:45   35      201735   \n",
       "4075 2021-11-19 18:02:18   46      202146   \n",
       "4087 2021-12-28 13:02:42   52      202152   \n",
       "4093 2021-09-29 18:55:59   39      202139   \n",
       "\n",
       "                                              text_data  \n",
       "3     Venice, Italy. It has title of .... It has tag...  \n",
       "4     Venice, Italy. It has title of .... It has tag...  \n",
       "5     Venice, Italy. It has title of .... It has tag...  \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  \n",
       "4106  Scanned from a half frame Kodachrome slide (tr...  \n",
       "...                                                 ...  \n",
       "4057  Venice, Italy. It has title of Piove. It has t...  \n",
       "4058  slowly, quietly ( langsam, leise ) in einer Au...  \n",
       "4075  . It has title of 2021-11-FL-2111 Biennale Arc...  \n",
       "4087  . It has title of Murano -Venezia. It has tags...  \n",
       "4093  . It has title of Venise, le marché. It has ta...  \n",
       "\n",
       "[1761 rows x 28 columns]"
      ]
     },
     "execution_count": 632,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 633,
   "id": "91201b67",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\2582107276.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['Italian'].loc[list(set(filter_it))] = 1\n",
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\2582107276.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['English'].loc[list(set(filter_en))] = 1\n",
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\2582107276.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  candidates_new['Other_Lang'].loc[list(set(filter_other))] = 1\n"
     ]
    }
   ],
   "source": [
    "candidates_new['Italian'] = 0\n",
    "candidates_new['English'] = 0\n",
    "candidates_new['Other_Lang'] = 0\n",
    "candidates_new['Italian'].loc[list(set(filter_it))] = 1\n",
    "candidates_new['English'].loc[list(set(filter_en))] = 1\n",
    "candidates_new['Other_Lang'].loc[list(set(filter_other))] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c385de45",
   "metadata": {},
   "source": [
    "#### Some Statistical Facts of the Subset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db824870",
   "metadata": {},
   "source": [
    "Number of unique timestamps (273->265), owners (330->309), and neighbourhoods (33->30) all drop a little bit in the subset compared to the original complete dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 852,
   "id": "516bb474",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 852,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAQjUlEQVR4nO3df4zk9V3H8ee7d0XpnT2hlLUe6EKWohe2WFhL0cbu+gMOyLa1arnraaGQblAx1VzVI/YPEtMUGzGGFNNsWgqNDRvERu+40xPUDYn8EK4p3NEr7dGc5S7IpbQeHhLp1bd/zOfCdNm5m53dmdn53PORbG6+n+93vvPaYXhl5jPf/X4jM5Ek1eV1/Q4gSVp6lrskVchyl6QKWe6SVCHLXZIqtLLfAQDOOOOMHB4e7neME3rppZdYtWpVv2MsiJm7b9Dygpl7pduZd+3a9e3MfPN865ZFuQ8PD/P444/3O8YJzc7OMj4+3u8YC2Lm7hu0vGDmXul25oj4j1brnJaRpAr1tdwjYjIipg8fPtzPGJJUnb6We2Zuy8ypNWvW9DOGJFXHaRlJqpDlLkkVstwlqUKWuyRVyHKXpAotiz9ikqR+Gt6yvSv73Tx6lGtPsO/9t1zVlcf2nbskVchyl6QKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQkte7hHx0xHxmYi4NyJ+e6n3L0k6sbbKPSLuiIhDEbFnzvj6iHg6IvZFxBaAzNybmTcAHwB+fukjS5JOpN137ncC65sHImIFcDtwBbAO2BgR68q69wDbgR1LllSS1LbIzPY2jBgG7svMC8rypcDNmXl5Wb4JIDM/2XSf7Zk574kTImIKmAIYGhq6eGZmZhG/Rm8cOXKE1atX9zvGgpi5+wYtL5h5rt0Hu3Opz6FT4fmXj7/N6NrOr0Q3MTGxKzPH5lu3mBOHrQWebVo+AFwSEePA+4Ef4jjv3DNzGpgGGBsby0G4qrlXX++NQcs8aHnBzHOd6ORendo8epRbdx+/ZvdvGu/KYy/5WSEzcxaYXer9SpLat5ijZQ4CZzctn1XG2hYRkxExffhwdz4SSdLJajHl/hhwXkScExGnABuArQvZQWZuy8ypNWs6n3OSJL1Wu4dC3g08DJwfEQci4vrMPArcCOwE9gL3ZOZT3YsqSWpXW3PumbmxxfgOFnG4Y0RMApMjIyOd7kKSNI++nn7AaRlJ6g7PLSNJFbLcJalCfS13D4WUpO5wzl2SKuS0jCRVyHKXpAo55y5JFXLOXZIq5LSMJFXIcpekClnuklQhv1CVpAr5haokVchpGUmqkOUuSRWy3CWpQpa7JFXIo2UkqUIeLSNJFXJaRpIqZLlLUoUsd0mqkOUuSRWy3CWpQh4KKUkV8lBISaqQ0zKSVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekClnuklQhTz8gSRXy9AOSVCGnZSSpQpa7JFXIcpekClnuklQhy12SKrSy3wEk6ZjhLdtbrts8epRrj7NeP8h37pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalClrskVchyl6QKdeWPmCLifcBVwBuBz2XmP3XjcSRJ82v7nXtE3BERhyJiz5zx9RHxdETsi4gtAJn5d5n5EeAG4OqljSxJOpGFTMvcCaxvHoiIFcDtwBXAOmBjRKxr2uTjZb0kqYciM9vfOGIYuC8zLyjLlwI3Z+blZfmmsukt5ef+zHygxb6mgCmAoaGhi2dmZjr9HXrmyJEjrF69ut8xFsTM3TdoeWH5Zt59sPUlN4dOhedf7mGYJdBO5tG1nV+JbmJiYldmjs23brFz7muBZ5uWDwCXAL8H/DKwJiJGMvMzc++YmdPANMDY2FiOj48vMkr3zc7OMgg5m5m5+wYtLyzfzMc7Mdjm0aPcunuwznXYTub9m8a78thdeaYy8zbgtm7sW5J0Yos9FPIgcHbT8lllrC0RMRkR04cPt/4oJklauMWW+2PAeRFxTkScAmwAtrZ758zclplTa9Z0PuckSXqthRwKeTfwMHB+RByIiOsz8yhwI7AT2Avck5lPdSeqJKldbc+5Z+bGFuM7gB2dPHhETAKTIyMjndxdktRCX08/4LSMJHWH55aRpApZ7pJUob6Wu4dCSlJ3OOcuSRVyWkaSKmS5S1KFnHOXpAo55y5JFXJaRpIqZLlLUoUsd0mqkF+oSlKF/EJVkirktIwkVchyl6QKWe6SVCHLXZIq5NEyklQhj5aRpAq1fYFsSSeH4S3b+x1BS8A5d0mqkOUuSRWy3CWpQpa7JFXIQyElqUIeCilJFXJaRpIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekClnuklQhy12SKuTpBySpQp5+QJIq5LSMJFXIcpekClnuklQhy12SKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVyHKXpApZ7pJUIctdkipkuUtShSx3SarQkpd7RJwbEZ+LiHuXet+SpPa0Ve4RcUdEHIqIPXPG10fE0xGxLyK2AGTmNzPz+m6ElSS1p9137ncC65sHImIFcDtwBbAO2BgR65Y0nSSpI5GZ7W0YMQzcl5kXlOVLgZsz8/KyfBNAZn6yLN+bmb9+nP1NAVMAQ0NDF8/MzCzi1+iNI0eOsHr16n7HWBAzd9+g5YXjZ959cHle9nLoVHj+5X6nWJh2Mo+u7fxKdBMTE7syc2y+dSs73iusBZ5tWj4AXBIRbwI+Abw9Im46VvZzZeY0MA0wNjaW4+Pji4jSG7OzswxCzmZm7r5BywvHz3ztlu29DdOmzaNHuXX3Yiqr99rJvH/TeFcee8mfqcx8AbhhqfcrSWrfYo6WOQic3bR8VhlrW0RMRsT04cPL82OgJA2qxZT7Y8B5EXFORJwCbAC2LmQHmbktM6fWrOl8zkmS9FrtHgp5N/AwcH5EHIiI6zPzKHAjsBPYC9yTmU91L6okqV1tzbln5sYW4zuAHZ0+eERMApMjIyOd7kKSNI++nn7AaRlJ6g7PLSNJFbLcJalCff2LAOfcB89wD/7AZfPo0WXzhzT7b7mq3xGkjjjnLkkVclpGkipkuUtShZxzH0ALmfdeTvPXknrHOXdJqpDTMpJUIctdkipkuUtShfpa7p7PXZK6wy9UJalCTstIUoUsd0mqkOUuSRWy3CWpQh4tI0kV8mgZSaqQ0zKSVCHLXZIqZLlLUoUsd0mqkOUuSRUa+CsxLeSqRIs196pG+2+5qmePLUkL4aGQklQhp2UkqUKWuyRVyHKXpApZ7pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalCXolJkirk6QckqUJOy0hShSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekCq1c6h1GxCrgr4BXgNnM/OJSP4Yk6fjaeuceEXdExKGI2DNnfH1EPB0R+yJiSxl+P3BvZn4EeM8S55UktaHdaZk7gfXNAxGxArgduAJYB2yMiHXAWcCzZbPvL01MSdJCRGa2t2HEMHBfZl5Qli8Fbs7My8vyTWXTA8B3M/O+iJjJzA0t9jcFTAEMDQ1dPDMz09EvsPtg7y7RN3QqPP9yzx5uSZi5+wYtL5i5V9rJPLq28yvRTUxM7MrMsfnWLWbOfS2vvkOHRqlfAtwGfDoirgK2tbpzZk4D0wBjY2M5Pj7eUYhrt2zv6H6d2Dx6lFt3L/nXFF1l5u4btLxg5l5pJ/P+TeNdeewlf6Yy8yXgw0u9X0lS+xZzKORB4Oym5bPKWNsiYjIipg8f7t3UiiSdDBZT7o8B50XEORFxCrAB2LqQHWTmtsycWrOm8zknSdJrtXso5N3Aw8D5EXEgIq7PzKPAjcBOYC9wT2Y+1b2okqR2tTXnnpkbW4zvAHZ0+uARMQlMjoyMdLoLSdI8+nr6AadlJKk7PLeMJFXIcpekCrX9F6pdefAy5w5cDXyjb0Hadwbw7X6HWCAzd9+g5QUz90q3M/9kZr55vhV9LfdBExGPt/pT3+XKzN03aHnBzL3Sz8xOy0hShSx3SaqQ5b4w0/0O0AEzd9+g5QUz90rfMjvnLkkV8p27JFXIcpekCp305T7f9WEj4sKIeDgidkfEtoh4Y9O6t5V1T5X1P1zGLy7L+yLitoiIfueNiE0R8ZWmn/+LiJ/pZd4OMr8+Iu4q43ubrvDV6pq9yyHzKRHx+TL+RESMN92nV6+LsyPiXyPiq+W1+dEyfnpE3B8R3yj/nlbGo+TZFxFPRsRFTfu6pmz/jYi4pht5O8z8U+X5/9+I+NicffXktdFB5k3l+d0dEQ9FxIU9y5yZJ/UP8AvARcCeprHHgHeX29cBf1purwSeBC4sy28CVpTb/w68EwjgH4Ar+p13zv1GgWealnuSt4Pn+IPATLn9BmA/MAysAJ4BzgVOAZ4A1i2TzL8LfL7cPhPYBbyux6+LtwAXlds/AnydxrWNPwVsKeNbgD8rt68seaLke7SMnw58s/x7Wrl92jLJfCbws8AngI817adnr40OMv/cseePxvWmH+1V5pP+nXtmPgh8Z87wW4EHy+37gV8rty8DnszMJ8p9X8jM70fEW4A3ZuYj2fgv9wXgfcsgb7ONwAxAL/PCgjMnsCoiVgKnAq8ALwLvAPZl5jcz85Xyu7x3mWReB/xLud8h4L+AsR6/Lp7LzC+X2/9N4zTca2k8R3eVze5qevz3Al/IhkeAHy15Lwfuz8zvZOZ3y++5fjlkzsxDmfkY8L05u+rZa6ODzA+V5xHgERoXNepJ5pO+3Ft4ilef6N/g1StOvRXIiNgZEV+OiD8q42tpXEP2mANlrFda5W12NXB3ud3vvNA6873AS8BzwLeAP8/M7zD/NXuXS+YngPdExMqIOAe4uKzry/McjYvZvx14FBjKzOfKqv8EhsrtVs9nX57nNjO3MiiZr6fxaQl6kNlyn991wO9ExC4aH71eKeMrgXcBm8q/vxoRv9SfiD+gVV4AIuIS4H8yc898d+6TVpnfAXwf+HHgHGBzRJzbn4iv0SrzHTT+53wc+EvgIRq/Q89FxGrgb4Hfz8wXm9eVTw/L7tjnkyFzREzQKPc/7lXGwbqUeI9k5tdoTMEQEW8FriqrDgAPZua3y7odNOZl/5pXP25BB9eT7VLeYzbw6rt2aGTrW144buYPAv+Ymd8DDkXEvwFjNN7lLOqavYvVKnM2rkr2B8e2i4iHaMzFfpcePs8R8XoahfPFzPxSGX4+It6Smc+VaZdDZbzVNZAPAuNzxmeXSeZWFn0954VYaOaIeBvwWRrft7zQq8y+c59HRJxZ/n0d8HHgM2XVTmA0It5Q5oTfDXy1fBx7MSLeWY6G+BDw98sg77GxD1Dm26Exb9jPvCfI/C3gF8u6VTS+7PsaS3DN3m5lLq+HVeX2rwBHM7Onr4uy/88BezPzL5pWbQWOHfFyTdPjbwU+VI6aeSdwuOTdCVwWEaeVIz4uK2PLIXMrPXttLDRzRPwE8CXgtzLz6z3N3I1vlAfph8Y72udofElzgMZHp4/SeOf1deAWyl/ylu1/k8bc6x7gU03jY2XsGeDTzffpc95x4JF59tOTvAvNDKwG/qY8x18F/rBpP1eW7Z8B/mS5vC5oHM3zNI0v1x6gcRrWXr8u3kVjKuBJ4Cvl50oaR3T9M41Taj8AnF62D+D2kms3MNa0r+uAfeXnw118jhea+cfKf4sXaXxpfYDGF9Y9e210kPmzND7BHdv28V69nj39gCRVyGkZSaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIq9P+69Aqr28MiywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['year'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 853,
   "id": "7bf3ce78",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "265"
      ]
     },
     "execution_count": 853,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['unique_week'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 854,
   "id": "0432961d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "309"
      ]
     },
     "execution_count": 854,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['owner'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 855,
   "id": "d5b40fb2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 855,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(candidates_new['neighbourhood'].unique())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eeb79622",
   "metadata": {},
   "source": [
    "#### Merging the translated texts back to a Revised version of text data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "562d48f6",
   "metadata": {},
   "source": [
    "The data are merged back in the order of English, translated Italian, and translated other languages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 634,
   "id": "75a51a06",
   "metadata": {},
   "outputs": [],
   "source": [
    "rev = {}\n",
    "keys_en = list(filter_dict_en.keys())\n",
    "keys_it = list(filter_dict_it.keys())\n",
    "keys_other = list(filter_dict_other.keys())\n",
    "for i in range(len(filter_en)):\n",
    "    if not filter_en[i] in rev:\n",
    "        rev[filter_en[i]] = filter_dict_en[keys_en[i]]\n",
    "    else:\n",
    "        rev[filter_en[i]] = '. '.join([rev[filter_en[i]], filter_dict_en[keys_en[i]]])\n",
    "        \n",
    "for i in range(len(filter_it)):\n",
    "    if not filter_it[i] in rev:\n",
    "        rev[filter_it[i]] = filter_dict_it[keys_it[i]]\n",
    "    else:\n",
    "        rev[filter_it[i]] = '. '.join([rev[filter_it[i]], filter_dict_it[keys_it[i]]])\n",
    "        \n",
    "for i in range(len(filter_other)):\n",
    "    if not filter_other[i] in rev:\n",
    "        rev[filter_other[i]] = filter_dict_other[keys_other[i]]\n",
    "    else:\n",
    "        rev[filter_other[i]] = '. '.join([rev[filter_other[i]], filter_dict_other[keys_other[i]]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 635,
   "id": "3118edea",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new['revised_text'] = pd.Series(rev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 636,
   "id": "a5a04dd6",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new['text_len'] = candidates_new['revised_text'].apply(lambda x: len(x.split()))\n",
    "candidates_new['num_sent'] = candidates_new['revised_text'].apply(lambda x: len(x.split('. ')))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f731d7f",
   "metadata": {},
   "source": [
    "Distribution of words per sentence and sentences per data point"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 863,
   "id": "84ecb004",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 863,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAOjklEQVR4nO3db4xc51XH8e/BJmmxhZs01SrYEetqrYAVC9KsSKzyYg1FcRqcoqqCWJFokNVVEIGCLCFb8CK8QA0SpjQitKxIiISqLGmoSmJbhDZ0hCJFIbFAjRPX1G3cxlZbp1VZtBYSOBxezHWytXbHM7sznp2z34+0yt7n/nvm5Prn8TPP3BuZiSSplh8ZdgckSf1nuEtSQYa7JBVkuEtSQYa7JBW0ftgdALjuuutyfHy85/3Onz/Phg0b+t+hIqxPZ9anM+vT2Wqoz7Fjx76Xme9ZbN2qCPfx8XFeeumlnvdrtVpMTU31v0NFWJ/OrE9n1qez1VCfiPjmUusclpGkggx3SSrIcJekgoYa7hGxJyJm5ubmhtkNSSpnqOGemU9n5vSmTZuG2Q1JKsdhGUkqyHCXpIIMd0kqaFV8iWklxg8cGdq5Tz9459DOLUmd+M5dkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgrqe7hHxE9HxGci4smI+M1+H1+SdHldhXtEPBoR5yLi+CXtuyPiZESciogDAJl5IjPvA34VeH//uyxJupxu37k/Buxe2BAR64CHgTuA7cDeiNjerLsLOAIc7VtPJUldi8zsbsOIceBwZt7ULO8EHsjM25vlgwCZ+YkF+xzJzEXvrhUR08A0wNjY2C2zs7M9d35+fp7X5t7seb9+2bF5dT9kZH5+no0bNw67G6uW9enM+nS2Guqza9euY5k5udi6ldwVcjPw+oLlM8CtETEFfBi4mg7v3DNzBpgBmJyczKmpqZ470Gq1OPTc+Z7365fT90wN7dzdaLVaLKeua4X16cz6dLba69P3W/5mZgto9fu4kqTurWS2zFnghgXLW5q2rvmAbEkajJWE+4vAtojYGhFXAXcDT/VyAB+QLUmD0e1UyMeB54EbI+JMROzLzAvA/cAzwAngicx8ZXBdlSR1q6sx98zcu0T7UVYw3TEi9gB7JiYmlnsISdIihnr7AYdlJGkwvLeMJBVkuEtSQUMNd6dCStJgOOYuSQU5LCNJBRnuklSQY+6SVJBj7pJUkMMyklSQ4S5JBRnuklSQH6hKUkF+oCpJBTksI0kFGe6SVJDhLkkFGe6SVJCzZSSpIGfLSFJBDstIUkGGuyQVZLhLUkGGuyQVZLhLUkGGuyQV5Dx3SSrIee6SVJDDMpJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQUZ7pJUkOEuSQX5DVVJKshvqEpSQQ7LSFJBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFWS4S1JBhrskFbR+EAeNiF8B7gR+HHgkM/9pEOeRJC2u63fuEfFoRJyLiOOXtO+OiJMRcSoiDgBk5hcy82PAfcCv9bfLkqTL6WVY5jFg98KGiFgHPAzcAWwH9kbE9gWb/GGzXpJ0BUVmdr9xxDhwODNvapZ3Ag9k5u3N8sFm0webny9m5peWONY0MA0wNjZ2y+zsbM+dn5+f57W5N3ver192bF7dT5Can59n48aNw+7GqmV9OrM+na2G+uzatetYZk4utm6lY+6bgdcXLJ8BbgV+G/gAsCkiJjLzM5fumJkzwAzA5ORkTk1N9XzyVqvFoefOL6Pb/XH6nqmhnbsbrVaL5dR1rbA+nVmfzlZ7fQbygWpmPgQ8NIhjS5Iub6VTIc8CNyxY3tK0dSUi9kTEzNzc3Aq7IUlaaKXh/iKwLSK2RsRVwN3AU93unJlPZ+b0pk2re+xakkZNL1MhHweeB26MiDMRsS8zLwD3A88AJ4AnMvOVwXRVktStrsfcM3PvEu1HgaPLOXlE7AH2TExMLGd3SdIShnr7AYdlJGkwvLeMJBU01HB3towkDYbDMpJUkMMyklSQ4S5JBRnuklSQH6hKUkF+oCpJBQ3krpBrxfiBI0M57+kH7xzKeSWNDsfcJakgx9wlqSDH3CWpIIdlJKkgw12SCjLcJakgw12SCnK2jCQV5GwZSSrIYRlJKshwl6SCDHdJKshwl6SCvCvkCOr2bpT7d1zg3j7fudI7UkqjwXfuklSQ89wlqSDnuUtSQQ7LSFJBfqCqnvhoQWk0+M5dkgoy3CWpIMNdkgoy3CWpIMNdkgoy3CWpIMNdkgry9gOSVJC3H5CkghyWkaSCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKshwl6SC+h7uEfHeiHgkIp7s97ElSd3pKtwj4tGIOBcRxy9p3x0RJyPiVEQcAMjMb2TmvkF0VpLUnW7fuT8G7F7YEBHrgIeBO4DtwN6I2N7X3kmSlmV9Nxtl5r9ExPglzT8HnMrMbwBExCzwIeDVbo4ZEdPANMDY2BitVqvLLr9tfn6e/Tve7Hm/tWLsnbB/x4Vhd6MvlnN9XM78/PxAjluF9elstdenq3Bfwmbg9QXLZ4BbI+LdwB8DN0fEwcz8xGI7Z+YMMAMwOTmZU1NTPXeg1Wpx6LnzPe+3VuzfcYFDL6/kf/Hqcfqeqb4fs9VqsZzrbq2wPp2t9vr0/U9+Zn4fuK/fx5UkdW8ls2XOAjcsWN7StHXNB2RL0mCsJNxfBLZFxNaIuAq4G3iqlwP4gGxJGoxup0I+DjwP3BgRZyJiX2ZeAO4HngFOAE9k5iuD66okqVvdzpbZu0T7UeDock8eEXuAPRMTE8s9hNaI8QNH+n7M/TsucO9ljnv6wTv7fl7pShjq7QcclpGkwfDeMpJUkOEuSQUNNdydCilJg+GYuyQV5LCMJBVkuEtSQY65S1JBjrlLUkEOy0hSQYa7JBVkuEtSQX6gKkkF+YGqJBXksIwkFWS4S1JBhrskFWS4S1JBzpaRpIKcLSNJBTksI0kFGe6SVJDhLkkFGe6SVJDhLkkFGe6SVJDz3CWpIOe5S1JBDstIUkGGuyQVZLhLUkGGuyQVZLhLUkGGuyQVZLhLUkGGuyQV5DdUJakgv6EqSQU5LCNJBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklSQ4S5JBRnuklTQ+n4fMCI2AH8J/A/QyszP9vsckqTOunrnHhGPRsS5iDh+SfvuiDgZEaci4kDT/GHgycz8GHBXn/srSepCt8MyjwG7FzZExDrgYeAOYDuwNyK2A1uA15vN3uxPNyVJvYjM7G7DiHHgcGbe1CzvBB7IzNub5YPNpmeAH2Tm4YiYzcy7lzjeNDANMDY2dsvs7GzPnZ+fn+e1Of/+WMrYO+G7/z3sXqxe1qezYdRnx+bhPJXt5bO9P+qzX/VZyWvetWvXscycXGzdSsbcN/P2O3Roh/qtwEPAX0TEncDTS+2cmTPADMDk5GROTU313IFWq8Wh5873vN9asX/HBQ693PePVcqwPp0Noz6n75m6oue76N4DR3rep1/1GdRr7vv/ucw8D/xGv48rSereSqZCngVuWLC8pWnrWkTsiYiZubne/0kkSVraSsL9RWBbRGyNiKuAu4GnejlAZj6dmdObNg1nnE2Squp2KuTjwPPAjRFxJiL2ZeYF4H7gGeAE8ERmvjK4rkqSutXVmHtm7l2i/ShwdLknj4g9wJ6JiYnlHkKStIih3n7AYRlJGgzvLSNJBQ013J0tI0mD0fU3VAfaiYg3gG8uY9frgO/1uTuVWJ/OrE9n1qez1VCfn8zM9yy2YlWE+3JFxEtLffVW1udyrE9n1qez1V4fx9wlqSDDXZIKGvVwnxl2B1Y569OZ9enM+nS2qusz0mPukqTFjfo7d0nSIgx3SSpoZMN9iee3rhkRcUNEfDkiXo2IVyLi4037tRHxxYj4WvPfa5r2iIiHmnp9JSLeN9xXcGVExLqI+LeIONwsb42IF5o6/F1zR1Mi4upm+VSzfnyoHb8CIuJdEfFkRHw1Ik5ExE6vn7dFxO81f7aOR8TjEfGOUbp+RjLcOzy/dS25AOzPzO3AbcBvNTU4ADybmduAZ5tlaNdqW/MzDXz6ynd5KD5O+66lF/0J8MnMnAB+AOxr2vfRfjzkBPDJZrvqPgX8Y2b+FPAztOvk9QNExGbgd4DJ5tGi62jf1nx0rp/MHLkfYCfwzILlg8DBYfdryDX5B+CXgJPA9U3b9cDJ5ve/AvYu2P6t7ar+0H6AzLPALwCHgaD9jcL1l15HtG9dvbP5fX2zXQz7NQywNpuA1y59jV4/b72+i48Rvba5Hg4Dt4/S9TOS79xZ/Pmtm4fUl6Fr/gl4M/ACMJaZ325WfQcYa35fizX7c+D3gf9rlt8N/Ge2n0UAP1yDt+rTrJ9rtq9qK/AG8DfNsNVfR8QGvH4AyMyzwJ8C3wK+Tft6OMYIXT+jGu5qRMRG4O+B383M/1q4LttvI9bkXNeI+GXgXGYeG3ZfVqn1wPuAT2fmzcB53h6CAdb89XMN8CHafwn+BLAB2D3UTvVoVMN9xc9vrSAifpR2sH82Mz/fNH83Iq5v1l8PnGva11rN3g/cFRGngVnaQzOfAt4VERcfUrOwBm/Vp1m/Cfj+lezwFXYGOJOZLzTLT9IOe6+ftg8Ar2XmG5n5v8DnaV9TI3P9jGq4r/j5raMuIgJ4BDiRmX+2YNVTwEeb3z9Keyz+YvuvN7MebgPmFvzzu5zMPJiZWzJznPb18c+ZeQ/wZeAjzWaX1udi3T7SbF/2XWtmfgd4PSJubJp+EXgVr5+LvgXcFhE/1vxZu1if0bl+hv3BxQo+8Pgg8B/A14E/GHZ/hvD6f572P5m/Avx78/NB2uN8zwJfA74EXNtsH7RnGH0deJn2LIChv44rVKsp4HDz+3uBfwVOAZ8Drm7a39Esn2rWv3fY/b4CdflZ4KXmGvoCcI3Xzw/V54+ArwLHgb8Frh6l68fbD0hSQaM6LCNJ6sBwl6SCDHdJKshwl6SCDHdJKshwl6SCDHdJKuj/AfjDKMVS9XhWAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['text_len'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 864,
   "id": "91f6a773",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 864,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAPHklEQVR4nO3dbYxc51nG8f+N80LkLS5polVkR9jBUcCKURovCVGrarcSZBPjpFQR2IpQjEytVEQCEQS2QCh8qNoihZeIQLSQ4IoWLyG8JbZRWqhX/RK1idsktjFp3WAUWyGmlBo2qghubz7MMd1sd9YzOzP7jB//f9LKc86ZPefS453L42fOnhOZiSSpLt9TOoAkqf8sd0mqkOUuSRWy3CWpQpa7JFXoktIBAK666qpcu3Zt2+1vvvkmK1euXL5AHTJXd8zVHXN152LMdejQoa9l5tULbszM4l+bNm3KxRw8eHDR7aWYqzvm6o65unMx5gJeyDa96rSMJFXIcpekChUt94jYEhFTZ86cKRlDkqpTtNwz85nM3Llq1aqSMSSpOk7LSFKFLHdJqpDlLkkVstwlqUJD8RuqvVi7a3+xY++ZHL7fhpMk8J27JFXJcpekClnuklQhy12SKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVqO/lHhE/HBGPRcRTEfHhfu9fknR+HZV7RDwREacj4si89ZMR8UpEHI+IXQCZeSwz7wd+GnhP/yNLks6n03fue4DJuSsiYgXwKHAHsAHYFhEbmm13AfuBA31LKknqWLRuoN3BEyPWAvsy88Zm+Tbgocy8vVneDZCZH53zPfszc3Ob/e0EdgKMjo5ump6ebnvs2dlZRkZGFtx2+FS5W/StW7Wiba6SFhuvkszVHXN152LMNTExcSgzxxba1stVIVcDr81ZPgncGhHjwAeBy1nknXtmTgFTAGNjYzk+Pt72QDMzM7Tbvr3wVSEXy13KYuNVkrm6Y67umOvt+n7J38ycAWb6vV9JUud6OVvmFHDtnOU1zbqORcSWiJg6c6bc1Iok1aiXcn8euD4i1kXEZcBW4OludpCZz2TmzlWrVvUQQ5I0X6enQu4FngNuiIiTEbEjM88CDwDPAseAJzPz6OCiSpI61dGce2Zua7P+AD2c7hgRW4At69evX+ouJEkLKHr5AadlJGkwvLaMJFXIcpekChUtd0+FlKTBcM5dkirktIwkVchyl6QKOecuSRVyzl2SKuS0jCRVyHKXpApZ7pJUIT9QlaQK+YGqJFXIaRlJqpDlLkkVstwlqUKWuyRVyLNlJKlCni0jSRVyWkaSKmS5S1KFLHdJqpDlLkkVstwlqUKeCilJFfJUSEmqkNMyklQhy12SKmS5S1KFLHdJqpDlLkkVstwlqUKWuyRVyHKXpApZ7pJUIS8/IEkV8vIDklQhp2UkqUKWuyRVyHKXpApZ7pJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIqdMkgdhoRHwA2A98HPJ6Znx7EcSRJC+v4nXtEPBERpyPiyLz1kxHxSkQcj4hdAJn5t5n5IeB+4Gf6G1mSdD7dTMvsASbnroiIFcCjwB3ABmBbRGyY85TfaLZLkpZRx+WemZ8Dvj5v9S3A8cx8NTPfAqaBu6Pl48DfZ+YX+xdXktSJyMzOnxyxFtiXmTc2y/cAk5n5883yzwK3Al8G7gOeB17MzMcW2NdOYCfA6Ojopunp6bbHnZ2dZWRkZMFth0+Vu0XfulUr2uYqabHxKslc3TFXdy7GXBMTE4cyc2yhbQP5QDUzHwEeOc9zpoApgLGxsRwfH2/73JmZGdpt375r/1Jj9mzP5Mq2uUpabLxKMld3zNUdc71dr6dCngKunbO8plnXEW+QLUmD0Wu5Pw9cHxHrIuIyYCvwdKff7A2yJWkwujkVci/wHHBDRJyMiB2ZeRZ4AHgWOAY8mZlHBxNVktSpjufcM3Nbm/UHgAN9S3QBOXzqTJE5/xMf27zsx5R0YSl6+QHn3CVpMIqWu3PukjQYXjhMkirktIwkVchpGUmqkNMyklQhy12SKuScuyRVyDl3SaqQ0zKSVCHLXZIqZLlLUoX8QFWSKuQHqpJUIadlJKlClrskVchyl6QKWe6SVCHPlpGkCnm2jCRVyGkZSarQJaUDqHtrd+1fdPuDG8+y/TzPWaoTH9s8kP1K6i/fuUtShSx3SaqQ5S5JFbLcJalCnucuSRXyPHdJqpDTMpJUIctdkipkuUtShSx3SaqQ5S5JFfLaMurK+a5rs5hernnjNW2k7vjOXZIqZLlLUoUsd0mqkJcfkKQKefkBSaqQ0zKSVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIcpekClnuklQhy12SKmS5S1KFLHdJqpDlLkkV6nu5R8R1EfF4RDzV731LkjrTUblHxBMRcToijsxbPxkRr0TE8YjYBZCZr2bmjkGElSR1ptN37nuAybkrImIF8ChwB7AB2BYRG/qaTpK0JJGZnT0xYi2wLzNvbJZvAx7KzNub5d0AmfnRZvmpzLxnkf3tBHYCjI6Obpqenm577NnZWUZGRhbcdvhUubs4jV4Bb3yz2OHbqjHXxtWDu6HLYj9fJZmrOxdjromJiUOZObbQtkt62O9q4LU5yyeBWyPiXcBHgHdHxO5zZT9fZk4BUwBjY2M5Pj7e9kAzMzO027591/6lZO+LBzee5eHDvQzhYNSY68S94/0NM8diP18lmas75nq7vjdAZv4HcH+/9ytJ6lwvZ8ucAq6ds7ymWdcxb5AtSYPRS7k/D1wfEesi4jJgK/B0NzvwBtmSNBidngq5F3gOuCEiTkbEjsw8CzwAPAscA57MzKODiypJ6lRHc+6Zua3N+gPAgaUePCK2AFvWr1+/1F1IkhZQ9PIDTstI0mB4bRlJqpDlLkkVKlrungopSYPhnLskVchpGUmqkOUuSRUqenUpz3PXsFtb8MJ0eyZXFju2LnzOuUtShZyWkaQKWe6SVCHLXZIq5C8xSVKF/EBVkirktIwkVchyl6QKWe6SVCHLXZIq5NkyklQhz5aRpAo5LSNJFbLcJalClrskVchyl6QKWe6SVCHLXZIq5G32dEEY5O3uHtx4lu0Fb6cnDYLnuUtShZyWkaQKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpa7JFXIOzFJUoX8DVVJqpDTMpJUIctdkipkuUtShSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVCHLXZIqZLlLUoUsd0mqkOUuSRWy3CWpQpf0e4cRsRL4Q+AtYCYzP9XvY0iSFtfRO/eIeCIiTkfEkXnrJyPilYg4HhG7mtUfBJ7KzA8Bd/U5rySpA51Oy+wBJueuiIgVwKPAHcAGYFtEbADWAK81T/tWf2JKkroRmdnZEyPWAvsy88Zm+Tbgocy8vVne3Tz1JPCfmbkvIqYzc2ub/e0EdgKMjo5ump6ebnvs2dlZRkZGFtx2+FS5W/SNXgFvfLPY4dsyV3eGNde6VSva/tyXtNjrsVe9vJ6H9e/xfLk2rl76negmJiYOZebYQtt6mXNfzXfeoUOr1G8FHgH+ICI2A8+0++bMnAKmAMbGxnJ8fLztgWZmZmi3ffuu/V3G7p8HN57l4cN9/9iiZ+bqzrDm2jO5su3PfUmLvR571cvreVj/Hs+X68S94wM5bt9HIjPfBH6u3/uVJHWul1MhTwHXzlle06zrWERsiYipM2fKTa1IUo16KffngesjYl1EXAZsBZ7uZgeZ+Uxm7ly1aulzTpKk79bpqZB7geeAGyLiZETsyMyzwAPAs8Ax4MnMPDq4qJKkTnU0556Z29qsPwAcWOrBI2ILsGX9+vVL3YUkaQFFLz/gtIwkDYbXlpGkChUtd8+WkaTB6Pg3VAcaIuLfgX9d5ClXAV9bpjjdMFd3zNUdc3XnYsz1A5l59UIbhqLczyciXmj3K7Ylmas75uqOubpjrrdzzl2SKmS5S1KFLpRynyodoA1zdcdc3TFXd8w1xwUx5y5J6s6F8s5dktQFy12SKjTU5d7mHq3FRcSJiDgcES9GxAuFs3zX/W0j4sqI+ExEfKX58/uHJNdDEXGqGbcXI+LOZc50bUQcjIh/ioijEfGLzfqi47VIrtLj9b0R8YWIeKnJ9VvN+nUR8fnmdfkXzVVhhyHXnoj4lznjddNy5pqTb0VEfCki9jXLZcYrM4fyC1gBfBW4DrgMeAnYUDpXk+0EcFXpHE2W9wE3A0fmrPttYFfzeBfw8SHJ9RDwKwXH6hrg5ubxO4Av07r/b9HxWiRX6fEKYKR5fCnweeDHgCeBrc36x4APD0muPcA9pcZrTr5fBv6c1m1JKTVew/zO/RbgeGa+mplvAdPA3YUzDZ3M/Bzw9Xmr7wY+0Tz+BPCB5cwEbXMVlZmvZ+YXm8f/TetS1aspPF6L5CoqW2abxUubrwTeDzzVrC8xXu1yFRcRa4DNwJ80y0Gh8Rrmcl/oHq3Ff+AbCXw6Ig41N/oeNqOZ+Xrz+N+A0ZJh5nkgIl5upm2WfbronOaG7++m9a5vaMZrXi4oPF7NFMOLwGngM7T+N/2NbN3PAQq9Lufnysxz4/WRZrx+NyIuX+5cwO8Bvwp8u1l+F4XGa5jLfZi9NzNvBu4AfiEi3lc6UDvZ+r/gULyrAf4I+EHgJuB14OESISJiBPgr4Jcy87/mbis5XgvkKj5emfmtzLyJ1m00bwF+aLkzLGR+roi4EdhNK9+PAlcCv7acmSLiJ4HTmXloOY/bzjCXe8/3aB2UzDzV/Hka+BtaP/TD5I2IuAag+fN04TwAZOYbzYvy28AfU2DcIuJSWgX6qcz862Z18fFaKNcwjNc5mfkN4CBwG/DOiDh3o5+ir8s5uSab6a3MzP8B/pTlH6/3AHdFxAla08jvB36fQuM1zOXe8z1aByEiVkbEO849Bn4COLL4dy27p4H7msf3AX9XMMv/O1egjZ9imcetmf98HDiWmb8zZ1PR8WqXawjG6+qIeGfz+Argx2l9HnAQuKd5WonxWijXP8/5BzpozWsv63hl5u7MXJOZa2n11Wcz815KjVfpT5bP86nznbTOHPgq8Oul8zSZrqN15s5LwNHSuYC9tP7L/r+05vN20Jrn+0fgK8A/AFcOSa4/Aw4DL9Mq1GuWOdN7aU25vAy82HzdWXq8FslVerx+BPhSc/wjwG82668DvgAcB/4SuHxIcn22Ga8jwCdpzqgp8QWM852zZYqMl5cfkKQKDfO0jCRpiSx3SaqQ5S5JFbLcJalClrskVchyl6QKWe6SVKH/A62lTKQ4qoCdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "candidates_new['num_sent'].hist(log=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 637,
   "id": "fabc6270",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "      <th>Italian</th>\n",
       "      <th>English</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_len</th>\n",
       "      <th>num_sent</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, bri...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, water</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>9</td>\n",
       "      <td>201909</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>THE KNIGHT WITH A GRIM LOOK   The Carnival of ...</td>\n",
       "      <td>222</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4106</th>\n",
       "      <td>51750158296</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175015829...</td>\n",
       "      <td>81438595@N00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Rialto Bridge, Venice, Italy, 1971</td>\n",
       "      <td>Scanned from a half frame Kodachrome slide (tr...</td>\n",
       "      <td>...</td>\n",
       "      <td>1971-01-01 00:00:00</td>\n",
       "      <td>53</td>\n",
       "      <td>197153</td>\n",
       "      <td>Scanned from a half frame Kodachrome slide (tr...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Rialto Bridge, Venice, Italy, ...</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4057</th>\n",
       "      <td>51813044138</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181304413...</td>\n",
       "      <td>61033692@N00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Piove</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2022-01-05 18:56:59</td>\n",
       "      <td>1</td>\n",
       "      <td>202201</td>\n",
       "      <td>Venice, Italy. It has title of Piove. It has t...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Rains. It has tags of venezia,...</td>\n",
       "      <td>25</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4058</th>\n",
       "      <td>51812495828</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181249582...</td>\n",
       "      <td>126584084@N05</td>\n",
       "      <td>Deutschland</td>\n",
       "      <td>slowly, quietly, Venezia</td>\n",
       "      <td>slowly, quietly ( langsam, leise ) in einer Au...</td>\n",
       "      <td>...</td>\n",
       "      <td>2017-09-03 13:02:45</td>\n",
       "      <td>35</td>\n",
       "      <td>201735</td>\n",
       "      <td>slowly, quietly ( langsam, leise ) in einer Au...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>Biennale 2017 Venezia  slowly, quietly in an e...</td>\n",
       "      <td>27</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4075</th>\n",
       "      <td>51757171151</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5175717115...</td>\n",
       "      <td>61579728@N05</td>\n",
       "      <td>London, UK</td>\n",
       "      <td>2021-11-FL-2111 Biennale Architecture Venice-6</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-11-19 18:02:18</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of 2021-11-FL-2111 Biennale Arc...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of architecturebiennale, biennale,...</td>\n",
       "      <td>11</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4087</th>\n",
       "      <td>51783729807</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5178372980...</td>\n",
       "      <td>141429878@N03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Murano -Venezia</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-12-28 13:02:42</td>\n",
       "      <td>52</td>\n",
       "      <td>202152</td>\n",
       "      <td>. It has title of Murano -Venezia. It has tags...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>It has title of Murano - Venice</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4093</th>\n",
       "      <td>51770600998</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5177060099...</td>\n",
       "      <td>123016824@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venise, le marché</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-09-29 18:55:59</td>\n",
       "      <td>39</td>\n",
       "      <td>202139</td>\n",
       "      <td>. It has title of Venise, le marché. It has ta...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>It has tags of whl. It has title of Venice, th...</td>\n",
       "      <td>12</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1761 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "4106  51750158296            1   \n",
       "...           ...          ...   \n",
       "4057  51813044138            1   \n",
       "4058  51812495828            1   \n",
       "4075  51757171151            1   \n",
       "4087  51783729807            1   \n",
       "4093  51770600998            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   \n",
       "...                                                 ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "4106  https://live.staticflickr.com/65535/5175015829...   81438595@N00   \n",
       "...                                                 ...            ...   \n",
       "4057  https://live.staticflickr.com/65535/5181304413...   61033692@N00   \n",
       "4058  https://live.staticflickr.com/65535/5181249582...  126584084@N05   \n",
       "4075  https://live.staticflickr.com/65535/5175717115...   61579728@N05   \n",
       "4087  https://live.staticflickr.com/65535/5178372980...  141429878@N03   \n",
       "4093  https://live.staticflickr.com/65535/5177060099...  123016824@N05   \n",
       "\n",
       "             owner_loc                                           title  \\\n",
       "3     Tallinn, Estonia                                             ...   \n",
       "4     Tallinn, Estonia                                             ...   \n",
       "5     Tallinn, Estonia                                             ...   \n",
       "6                  NaN                         VENEZIA. CARNEVALE 2019   \n",
       "4106               NaN              Rialto Bridge, Venice, Italy, 1971   \n",
       "...                ...                                             ...   \n",
       "4057               NaN                                           Piove   \n",
       "4058       Deutschland                        slowly, quietly, Venezia   \n",
       "4075        London, UK  2021-11-FL-2111 Biennale Architecture Venice-6   \n",
       "4087               NaN                                 Murano -Venezia   \n",
       "4093               NaN                               Venise, le marché   \n",
       "\n",
       "                                            description  ...  \\\n",
       "3                                         Venice, Italy  ...   \n",
       "4                                         Venice, Italy  ...   \n",
       "5                                         Venice, Italy  ...   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  ...   \n",
       "4106  Scanned from a half frame Kodachrome slide (tr...  ...   \n",
       "...                                                 ...  ...   \n",
       "4057                                      Venice, Italy  ...   \n",
       "4058  slowly, quietly ( langsam, leise ) in einer Au...  ...   \n",
       "4075                                                     ...   \n",
       "4087                                                     ...   \n",
       "4093                                                     ...   \n",
       "\n",
       "                    time week  unique_week  \\\n",
       "3    2021-10-05 14:45:41   40       202140   \n",
       "4    2021-10-05 14:46:05   40       202140   \n",
       "5    2021-10-05 15:13:13   40       202140   \n",
       "6    2019-02-28 14:11:02    9       201909   \n",
       "4106 1971-01-01 00:00:00   53       197153   \n",
       "...                  ...  ...          ...   \n",
       "4057 2022-01-05 18:56:59    1       202201   \n",
       "4058 2017-09-03 13:02:45   35       201735   \n",
       "4075 2021-11-19 18:02:18   46       202146   \n",
       "4087 2021-12-28 13:02:42   52       202152   \n",
       "4093 2021-09-29 18:55:59   39       202139   \n",
       "\n",
       "                                              text_data Italian  English  \\\n",
       "3     Venice, Italy. It has title of .... It has tag...       0        1   \n",
       "4     Venice, Italy. It has title of .... It has tag...       0        1   \n",
       "5     Venice, Italy. It has title of .... It has tag...       0        1   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...       1        1   \n",
       "4106  Scanned from a half frame Kodachrome slide (tr...       0        1   \n",
       "...                                                 ...     ...      ...   \n",
       "4057  Venice, Italy. It has title of Piove. It has t...       1        0   \n",
       "4058  slowly, quietly ( langsam, leise ) in einer Au...       0        1   \n",
       "4075  . It has title of 2021-11-FL-2111 Biennale Arc...       0        0   \n",
       "4087  . It has title of Murano -Venezia. It has tags...       1        0   \n",
       "4093  . It has title of Venise, le marché. It has ta...       0        1   \n",
       "\n",
       "      Other_Lang                                       revised_text text_len  \\\n",
       "3              0  It has tags of adriaticsea, italy, venice, bri...        9   \n",
       "4              0          It has tags of adriaticsea, italy, venice        7   \n",
       "5              0   It has tags of adriaticsea, italy, venice, water        8   \n",
       "6              1  THE KNIGHT WITH A GRIM LOOK   The Carnival of ...      222   \n",
       "4106           0  It has title of Rialto Bridge, Venice, Italy, ...        9   \n",
       "...          ...                                                ...      ...   \n",
       "4057           0  It has title of Rains. It has tags of venezia,...       25   \n",
       "4058           1  Biennale 2017 Venezia  slowly, quietly in an e...       27   \n",
       "4075           1  It has tags of architecturebiennale, biennale,...       11   \n",
       "4087           0                    It has title of Murano - Venice        7   \n",
       "4093           1  It has tags of whl. It has title of Venice, th...       12   \n",
       "\n",
       "      num_sent  \n",
       "3            1  \n",
       "4            1  \n",
       "5            1  \n",
       "6            9  \n",
       "4106         1  \n",
       "...        ...  \n",
       "4057         2  \n",
       "4058         4  \n",
       "4075         1  \n",
       "4087         1  \n",
       "4093         2  \n",
       "\n",
       "[1761 rows x 34 columns]"
      ]
     },
     "execution_count": 637,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "candidates_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf132ad4",
   "metadata": {},
   "source": [
    "#### Merging the Revised Text data back to the full Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "49fb6f2d",
   "metadata": {},
   "source": [
    "Merging first back to the subset with text data and copy revised text data to the duplicates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 638,
   "id": "67734048",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_use = candidates_new.columns.difference(df_new.columns)\n",
    "new = df_new.merge(candidates_new[cols_to_use], left_index=True, right_index=True, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 639,
   "id": "8cf6c0e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_dict = {}\n",
    "for i in range(len(new)):\n",
    "    if new.iloc[i]['text_data'] not in text_dict:\n",
    "        text_dict[new.iloc[i]['text_data']] = new.iloc[i]['revised_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 640,
   "id": "50a8e88f",
   "metadata": {},
   "outputs": [],
   "source": [
    "new['revised_text'] = new['text_data'].apply(lambda x: text_dict[x])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "55dd1d03",
   "metadata": {},
   "source": [
    "191 data points have a text data field, but do not have a revised text data field, since the texts they have are *invalid*"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 865,
   "id": "a58d9365",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>text_data</th>\n",
       "      <th>English</th>\n",
       "      <th>Italian</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>51820580920</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5182058092...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5182058092...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5182058092...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5182058092...</td>\n",
       "      <td>40575230@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1981 European Trafalgar tour #203 Venice, Ital...</td>\n",
       "      <td>Scanner (Kodak Slide n scan) from Kodachrome s...</td>\n",
       "      <td>...</td>\n",
       "      <td>1981-11-01 00:00:00</td>\n",
       "      <td>44</td>\n",
       "      <td>198144</td>\n",
       "      <td>Scanner (Kodak Slide n scan) from Kodachrome s...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>51818440638</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181844063...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181844063...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181844063...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181844063...</td>\n",
       "      <td>129254222@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_20211119_095036</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-11-19 09:50:36</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of IMG_20211119_095036. It has ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>51818666444</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866644...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866644...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866644...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866644...</td>\n",
       "      <td>129254222@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_20211119_113153</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-11-19 11:31:53</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of IMG_20211119_113153. It has ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>51817375982</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181737598...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181737598...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181737598...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181737598...</td>\n",
       "      <td>129254222@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_20211119_115156</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-11-19 11:51:56</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of IMG_20211119_115156. It has ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>51818666119</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866611...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866611...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866611...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5181866611...</td>\n",
       "      <td>129254222@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_20211119_120843</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-11-19 12:08:43</td>\n",
       "      <td>46</td>\n",
       "      <td>202146</td>\n",
       "      <td>. It has title of IMG_20211119_120843. It has ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4573</th>\n",
       "      <td>51594038030</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159403803...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159403803...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159403803...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159403803...</td>\n",
       "      <td>10419801@N06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_7913.jpg</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-10 18:52:36</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>. It has title of IMG_7913.jpg. It has tags of</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4574</th>\n",
       "      <td>51593363538</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159336353...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159336353...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159336353...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5159336353...</td>\n",
       "      <td>10419801@N06</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IMG_7896.jpg</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-10 17:18:22</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>. It has title of IMG_7896.jpg. It has tags of</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4808</th>\n",
       "      <td>51320597675</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5132059767...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5132059767...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5132059767...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5132059767...</td>\n",
       "      <td>22857403@N05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20210718_124440</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-07-18 12:44:40</td>\n",
       "      <td>28</td>\n",
       "      <td>202128</td>\n",
       "      <td>. It has title of 20210718_124440. It has tags...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4932</th>\n",
       "      <td>51228899889</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122889988...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122889988...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122889988...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122889988...</td>\n",
       "      <td>85941441@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20210605_160458-01</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-06-05 16:04:58</td>\n",
       "      <td>22</td>\n",
       "      <td>202122</td>\n",
       "      <td>. It has title of 20210605_160458-01. It has t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4933</th>\n",
       "      <td>51228346363</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122834636...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122834636...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122834636...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5122834636...</td>\n",
       "      <td>85941441@N04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>20210605_123731-01</td>\n",
       "      <td></td>\n",
       "      <td>...</td>\n",
       "      <td>2021-06-05 12:37:31</td>\n",
       "      <td>22</td>\n",
       "      <td>202122</td>\n",
       "      <td>. It has title of 20210605_123731-01. It has t...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>191 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "132   51820580920            1   \n",
       "141   51818440638            1   \n",
       "142   51818666444            1   \n",
       "143   51817375982            1   \n",
       "144   51818666119            1   \n",
       "...           ...          ...   \n",
       "4573  51594038030            1   \n",
       "4574  51593363538            1   \n",
       "4808  51320597675            1   \n",
       "4932  51228899889            1   \n",
       "4933  51228346363            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "132   https://live.staticflickr.com/65535/5182058092...   \n",
       "141   https://live.staticflickr.com/65535/5181844063...   \n",
       "142   https://live.staticflickr.com/65535/5181866644...   \n",
       "143   https://live.staticflickr.com/65535/5181737598...   \n",
       "144   https://live.staticflickr.com/65535/5181866611...   \n",
       "...                                                 ...   \n",
       "4573  https://live.staticflickr.com/65535/5159403803...   \n",
       "4574  https://live.staticflickr.com/65535/5159336353...   \n",
       "4808  https://live.staticflickr.com/65535/5132059767...   \n",
       "4932  https://live.staticflickr.com/65535/5122889988...   \n",
       "4933  https://live.staticflickr.com/65535/5122834636...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "132   https://live.staticflickr.com/65535/5182058092...   \n",
       "141   https://live.staticflickr.com/65535/5181844063...   \n",
       "142   https://live.staticflickr.com/65535/5181866644...   \n",
       "143   https://live.staticflickr.com/65535/5181737598...   \n",
       "144   https://live.staticflickr.com/65535/5181866611...   \n",
       "...                                                 ...   \n",
       "4573  https://live.staticflickr.com/65535/5159403803...   \n",
       "4574  https://live.staticflickr.com/65535/5159336353...   \n",
       "4808  https://live.staticflickr.com/65535/5132059767...   \n",
       "4932  https://live.staticflickr.com/65535/5122889988...   \n",
       "4933  https://live.staticflickr.com/65535/5122834636...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "132   https://live.staticflickr.com/65535/5182058092...   \n",
       "141   https://live.staticflickr.com/65535/5181844063...   \n",
       "142   https://live.staticflickr.com/65535/5181866644...   \n",
       "143   https://live.staticflickr.com/65535/5181737598...   \n",
       "144   https://live.staticflickr.com/65535/5181866611...   \n",
       "...                                                 ...   \n",
       "4573  https://live.staticflickr.com/65535/5159403803...   \n",
       "4574  https://live.staticflickr.com/65535/5159336353...   \n",
       "4808  https://live.staticflickr.com/65535/5132059767...   \n",
       "4932  https://live.staticflickr.com/65535/5122889988...   \n",
       "4933  https://live.staticflickr.com/65535/5122834636...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "132   https://live.staticflickr.com/65535/5182058092...   40575230@N05   \n",
       "141   https://live.staticflickr.com/65535/5181844063...  129254222@N04   \n",
       "142   https://live.staticflickr.com/65535/5181866644...  129254222@N04   \n",
       "143   https://live.staticflickr.com/65535/5181737598...  129254222@N04   \n",
       "144   https://live.staticflickr.com/65535/5181866611...  129254222@N04   \n",
       "...                                                 ...            ...   \n",
       "4573  https://live.staticflickr.com/65535/5159403803...   10419801@N06   \n",
       "4574  https://live.staticflickr.com/65535/5159336353...   10419801@N06   \n",
       "4808  https://live.staticflickr.com/65535/5132059767...   22857403@N05   \n",
       "4932  https://live.staticflickr.com/65535/5122889988...   85941441@N04   \n",
       "4933  https://live.staticflickr.com/65535/5122834636...   85941441@N04   \n",
       "\n",
       "     owner_loc                                              title  \\\n",
       "132        NaN  1981 European Trafalgar tour #203 Venice, Ital...   \n",
       "141        NaN                                IMG_20211119_095036   \n",
       "142        NaN                                IMG_20211119_113153   \n",
       "143        NaN                                IMG_20211119_115156   \n",
       "144        NaN                                IMG_20211119_120843   \n",
       "...        ...                                                ...   \n",
       "4573       NaN                                       IMG_7913.jpg   \n",
       "4574       NaN                                       IMG_7896.jpg   \n",
       "4808       NaN                                    20210718_124440   \n",
       "4932       NaN                                 20210605_160458-01   \n",
       "4933       NaN                                 20210605_123731-01   \n",
       "\n",
       "                                            description  ...  \\\n",
       "132   Scanner (Kodak Slide n scan) from Kodachrome s...  ...   \n",
       "141                                                      ...   \n",
       "142                                                      ...   \n",
       "143                                                      ...   \n",
       "144                                                      ...   \n",
       "...                                                 ...  ...   \n",
       "4573                                                     ...   \n",
       "4574                                                     ...   \n",
       "4808                                                     ...   \n",
       "4932                                                     ...   \n",
       "4933                                                     ...   \n",
       "\n",
       "                    time week  unique_week  \\\n",
       "132  1981-11-01 00:00:00   44       198144   \n",
       "141  2021-11-19 09:50:36   46       202146   \n",
       "142  2021-11-19 11:31:53   46       202146   \n",
       "143  2021-11-19 11:51:56   46       202146   \n",
       "144  2021-11-19 12:08:43   46       202146   \n",
       "...                  ...  ...          ...   \n",
       "4573 2021-10-10 18:52:36   40       202140   \n",
       "4574 2021-10-10 17:18:22   40       202140   \n",
       "4808 2021-07-18 12:44:40   28       202128   \n",
       "4932 2021-06-05 16:04:58   22       202122   \n",
       "4933 2021-06-05 12:37:31   22       202122   \n",
       "\n",
       "                                              text_data English  Italian  \\\n",
       "132   Scanner (Kodak Slide n scan) from Kodachrome s...     NaN      NaN   \n",
       "141   . It has title of IMG_20211119_095036. It has ...     NaN      NaN   \n",
       "142   . It has title of IMG_20211119_113153. It has ...     NaN      NaN   \n",
       "143   . It has title of IMG_20211119_115156. It has ...     NaN      NaN   \n",
       "144   . It has title of IMG_20211119_120843. It has ...     NaN      NaN   \n",
       "...                                                 ...     ...      ...   \n",
       "4573    . It has title of IMG_7913.jpg. It has tags of      NaN      NaN   \n",
       "4574    . It has title of IMG_7896.jpg. It has tags of      NaN      NaN   \n",
       "4808  . It has title of 20210718_124440. It has tags...     NaN      NaN   \n",
       "4932  . It has title of 20210605_160458-01. It has t...     NaN      NaN   \n",
       "4933  . It has title of 20210605_123731-01. It has t...     NaN      NaN   \n",
       "\n",
       "      Other_Lang num_sent revised_text  text_len  \n",
       "132          NaN      NaN          NaN       NaN  \n",
       "141          NaN      NaN          NaN       NaN  \n",
       "142          NaN      NaN          NaN       NaN  \n",
       "143          NaN      NaN          NaN       NaN  \n",
       "144          NaN      NaN          NaN       NaN  \n",
       "...          ...      ...          ...       ...  \n",
       "4573         NaN      NaN          NaN       NaN  \n",
       "4574         NaN      NaN          NaN       NaN  \n",
       "4808         NaN      NaN          NaN       NaN  \n",
       "4932         NaN      NaN          NaN       NaN  \n",
       "4933         NaN      NaN          NaN       NaN  \n",
       "\n",
       "[191 rows x 34 columns]"
      ]
     },
     "execution_count": 865,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new[new['revised_text'].isna()]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd2a671",
   "metadata": {},
   "source": [
    "Merging then back to the full dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 643,
   "id": "afe93988",
   "metadata": {},
   "outputs": [],
   "source": [
    "cols_to_use = new.columns.difference(df.columns)\n",
    "final_df = df.merge(new[cols_to_use], left_index=True, right_index=True, how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 644,
   "id": "c8d099c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>time</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>English</th>\n",
       "      <th>Italian</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_data</th>\n",
       "      <th>text_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 14:45:41</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, bri...</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 14:46:05</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>7.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-10-05 15:13:13</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, water</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>51870380541</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187038054...</td>\n",
       "      <td>64148082@N02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>VENEZIA. CARNEVALE 2019</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>...</td>\n",
       "      <td>2019-02-28 14:11:02</td>\n",
       "      <td>9</td>\n",
       "      <td>201909</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>THE KNIGHT WITH A GRIM LOOK   The Carnival of ...</td>\n",
       "      <td>IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...</td>\n",
       "      <td>222.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51863124066</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>186704588@N07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venice - Ponte dei Sospiri</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>...</td>\n",
       "      <td>2022-01-20 18:14:39</td>\n",
       "      <td>3</td>\n",
       "      <td>202203</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Bridge of Sighs. The Bridge of Sighs was built...</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>184.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>51233430558</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-05-20 14:58:59</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_145859. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>51234286630</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-05-20 14:57:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_145745. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>51234294135</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_164727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-05-20 16:47:27</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_164727. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>51234291635</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-05-20 15:56:46</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_155646. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>51233434888</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2021-05-20 15:55:25</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_155525. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2951 rows × 34 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "6     51870380541            1   \n",
       "13    51863124066            1   \n",
       "...           ...          ...   \n",
       "4981  51233430558            1   \n",
       "4982  51234286630            1   \n",
       "4983  51234294135            1   \n",
       "4984  51234291635            1   \n",
       "4985  51233434888            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "6     https://live.staticflickr.com/65535/5187038054...   64148082@N02   \n",
       "13    https://live.staticflickr.com/65535/5186312406...  186704588@N07   \n",
       "...                                                 ...            ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   83293103@N00   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   83293103@N00   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   83293103@N00   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   83293103@N00   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   83293103@N00   \n",
       "\n",
       "             owner_loc                       title  \\\n",
       "3     Tallinn, Estonia                         ...   \n",
       "4     Tallinn, Estonia                         ...   \n",
       "5     Tallinn, Estonia                         ...   \n",
       "6                  NaN     VENEZIA. CARNEVALE 2019   \n",
       "13                 NaN  Venice - Ponte dei Sospiri   \n",
       "...                ...                         ...   \n",
       "4981     Milano, Italy             20210520_145859   \n",
       "4982     Milano, Italy             20210520_145745   \n",
       "4983     Milano, Italy             20210520_164727   \n",
       "4984     Milano, Italy             20210520_155646   \n",
       "4985     Milano, Italy             20210520_155525   \n",
       "\n",
       "                                            description  ...  \\\n",
       "3                                         Venice, Italy  ...   \n",
       "4                                         Venice, Italy  ...   \n",
       "5                                         Venice, Italy  ...   \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...  ...   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...  ...   \n",
       "...                                                 ...  ...   \n",
       "4981                                                NaN  ...   \n",
       "4982                                                NaN  ...   \n",
       "4983                                                NaN  ...   \n",
       "4984                                                NaN  ...   \n",
       "4985                                                NaN  ...   \n",
       "\n",
       "                    time week  unique_week  English Italian  Other_Lang  \\\n",
       "3    2021-10-05 14:45:41   40       202140      1.0     0.0         0.0   \n",
       "4    2021-10-05 14:46:05   40       202140      1.0     0.0         0.0   \n",
       "5    2021-10-05 15:13:13   40       202140      1.0     0.0         0.0   \n",
       "6    2019-02-28 14:11:02    9       201909      1.0     1.0         1.0   \n",
       "13   2022-01-20 18:14:39    3       202203      0.0     1.0         0.0   \n",
       "...                  ...  ...          ...      ...     ...         ...   \n",
       "4981 2021-05-20 14:58:59   20       202120      0.0     1.0         0.0   \n",
       "4982 2021-05-20 14:57:46   20       202120      0.0     1.0         0.0   \n",
       "4983 2021-05-20 16:47:27   20       202120      0.0     1.0         0.0   \n",
       "4984 2021-05-20 15:56:46   20       202120      0.0     1.0         0.0   \n",
       "4985 2021-05-20 15:55:25   20       202120      0.0     1.0         0.0   \n",
       "\n",
       "      num_sent                                       revised_text  \\\n",
       "3          1.0  It has tags of adriaticsea, italy, venice, bri...   \n",
       "4          1.0          It has tags of adriaticsea, italy, venice   \n",
       "5          1.0   It has tags of adriaticsea, italy, venice, water   \n",
       "6          9.0  THE KNIGHT WITH A GRIM LOOK   The Carnival of ...   \n",
       "13        10.0  Bridge of Sighs. The Bridge of Sighs was built...   \n",
       "...        ...                                                ...   \n",
       "4981       1.0  It has tags of biennale, architecture, archite...   \n",
       "4982       1.0  It has tags of biennale, architecture, archite...   \n",
       "4983       1.0  It has tags of biennale, architecture, archite...   \n",
       "4984       1.0  It has tags of biennale, architecture, archite...   \n",
       "4985       1.0  It has tags of biennale, architecture, archite...   \n",
       "\n",
       "                                              text_data  text_len  \n",
       "3     Venice, Italy. It has title of .... It has tag...       9.0  \n",
       "4     Venice, Italy. It has title of .... It has tag...       7.0  \n",
       "5     Venice, Italy. It has title of .... It has tag...       8.0  \n",
       "6     IL CAVALIERE DALLO SGUARDO TORVO\\n\\n\\nIl Carne...     222.0  \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...     184.0  \n",
       "...                                                 ...       ...  \n",
       "4981  . It has title of 20210520_145859. It has tags...       9.0  \n",
       "4982  . It has title of 20210520_145745. It has tags...       9.0  \n",
       "4983  . It has title of 20210520_164727. It has tags...       9.0  \n",
       "4984  . It has title of 20210520_155646. It has tags...       9.0  \n",
       "4985  . It has title of 20210520_155525. It has tags...       9.0  \n",
       "\n",
       "[2951 rows x 34 columns]"
      ]
     },
     "execution_count": 644,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e1051b7",
   "metadata": {},
   "source": [
    "A few cases with super long text descriptions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 651,
   "id": "78282199",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['NOT ONLY THE SQUERO. The church is dedicated to the two Milanese martyrs Gervasio and Protasio (hence Trovaso) and was founded in the 9th century. As Giuseppe Tassini tells us, the church was built in the early days after the foundation of Venice, and immediately became a parish church. It was rebuilt in 1028 by the Barbarigo and Caravella families. From its foundation it was subjected to the authorities of the patriarchs of Grado and the bishops of Olivolo. Due to various controversies, a compromise was reached in 1041: both prelates would have elected and commanded the vicar of San Trovaso. This is an important testimony of the bad relations that existed in the Middle Ages between the Gradense clergy and the Venetian clergy. In 1105 the church was renovated. It collapsed in 1583 and from 1584 it was rebuilt in seven years. Completely renovated starting from 1585 in Palladian forms with a Greek cross plan, on the outside it has the particularity of having two almost identical facades with double facing onto the Rio di Ognissanti and the Rio di San Trovaso. It was consecrated in 1657. CHIESA DI SAN TROVASO. The church is dedicated to the two Milanese martyrs Gervasio and Protasio (hence Trovaso) and was founded in the 9th century. As Giuseppe Tassini tells us, the church was built in the early days after the foundation of Venice, and immediately became a parish. It was rebuilt in 1028 by the Barbarigo and Caravella families. From its foundation it was subject to the authorities of the patriarchs of Grado and the bishops of Olivolo. Due to various controversies in 1041 a compromise was reached: both prelates would have elected and commanded the vicar of San Trovaso. This is an important testimony of the bad relations that existed in the Middle Ages between the Gradense clergy and the Venetian clergy.. In 1105 the church was renovated. It collapsed in 1583 and from 1584 it was rebuilt in seven years. Completely renovated starting from 1585 in Palladian forms with a Greek cross plan, on the outside it has the particularity of having two almost identical facades with double facing onto the Rio di Ognissanti and the Rio di San Trovaso. It was consecrated in 1657. NOT ONLY THE SQUERO. It has title of VENEZIA',\n",
       " 'How many times has this framing of the Ri alto been and are still taken in photography. I found many, but with subtle differences for the stereos of those years. It is still the more or less defined association between Henri Plaut and Alexandre Bertrand, in 1865. (see comments on their panorama of Cadiz) If some questions have been raised I think rise, as for Cadiz the images have been retouched. I deliberately left the unfortunate alterations on the left center of the opening of the Ri alto. Retouched and trimmed. (as well as the roofs of the houses on the left) They used at least for this series the same tools to put back in the right direction the negatives before insolation of the positives. The visible separation bar (which should not be. ) is always identical. This makes the hyper-stereo a signature. Good photographers, but for one of them a lousy lab tech. I hesitated, but it would have been a shame to deprive myself of this particularly powerful stereography, a bit of today’s rendering with a Fuji réal 3 D, but it’s 1865. One of the sails on the second gondola to slide and the long exposure time gives the water this vaporous side in opposition to the sharpness of the whole picture. In view of the dust (of origin.) and other defects, I only dusted the whole. The difference between dust removal and  restored  is only in the process. For a «dust removal» I work on the image at max 150% magnification, for the other 300%, has this size nothing can normally escape you... Generally speaking my wish would be to be able to view the enlarged image, as a document. But eyelashes, hairs or treacherous dust can and surely must have already passed through the visual antiparasitic net... That’s my limit: I’m human. It has tags of stereophotography, stereophotos, stereoview, stereo, stererocard, stereograph, stereoscopic, stereography, stereoscopique, anaglyph, peppinanaglyphes, venise, bertrand, rialto. How many times this framing of the Ri alto has been and are still photographed. I found many, but with sometimes subtle differences for the stereos of those years. It is still the more or less defined association between Henri Plaut and Alexandre Bertrand, in 1865. (see the comments on their panorama of Cadiz) If some questions have been raised, I think, as for Cadiz the images have been retouched. I deliberately left the unfortunate retouching on the left center of the opening of the Ri alto. Retouched and cropped. (as well as the roofs of the houses on the left) They used at least for this series the same tools to put the negatives back in the right direction before insolation of the positives. The visible separation bar (which should not be. ) is always the same. This plus hyper stereo suddenly becomes a signature. Good photographers, but for one of them a poor lab technician. I hesitated, but it would have been a shame to deprive yourself of this particularly powerful stereography, a bit like today\\'s rendering with a Fuji real 3D, but we are in 1865. One of the sails on the second gondola to slide and the long exposure time gives the water this vaporous side in opposition to the sharpness of the whole image. In view of the dust (of origins.) and other defects, I only dusted the whole. The difference between dust removal and \"restored\" is only in the process. For a \"dusting\" I work on the image at maximum 150% enlargement, for the other 300%, at this size nothing can normally escape you... In general, my desire would be to be able to view the image enlarged, like a document. But eyelashes, hairs or treacherous dust can and surely must have already passed through the visual interference net... This is my limit: I am human. It has title of Venice Italy,N°377 Rialto Bridge. Alexander Bertrand 1865',\n",
       " 'The Peggy Guggenheim Collection is a modern art museum on the Grand Canal in the Dorsoduro sestiere of Venice, Italy. It is one of the most visited attractions in Venice. The collection is housed in the Palazzo Venier dei Leoni, an 18th-century palace, which was the home of the American heiress Peggy Guggenheim for three decades. She began displaying her private collection of modern artworks to the public seasonally in 1951. After her death in 1979, it passed to the Solomon R. Guggenheim Foundation, which opened the collection year-round from 1980. The collection includes works of prominent Italian futurists and American modernists working in such genres as Cubism, Surrealism and abstract expressionism. It also includes sculptural works. In 2017, Karole Vail, a granddaughter of Peggy Guggenheim, was appointed Director of the collection, succeeding Philip Rylands, who led the museum for 37 years. The collection is housed in the Palazzo Venier dei Leoni, which Peggy Guggenheim purchased in 1949. Although sometimes mistaken for a modern building, it is an 18th-century palace designed by the Venetian architect Lorenzo Boschetti. The building was unfinished, and has an unusually low elevation on the Grand Canal. After the Foundation took control of the building in 1979, it took steps to expand gallery space. by 1985, .all of the rooms on the main floor had been converted into galleries ... the white Istrian stone facade and the unique canal terrace had been restored. and a protruding arcade wing, called the barchessa, had been rebuilt by architect Giorgio Bellavitis. Since 1985, the museum has been open year-round. In 1993, apartments adjacent to the museum were converted to a garden annex, a shop and more galleries. In 1995, the Nasher Sculpture Garden was completed, additional exhibition rooms were added, and a café was opened. A few years later, in 1999 and in 2000, the two neighboring properties were acquired. In 2003, a new entrance and booking office opened to cope with the increasing number of visitors, which reached 350,000 in 2007. Since 1993, the museum has doubled in size, from 2,000 to 4,000 square meters. It has title of ALOA 372 Venice Tour - Electric Taxi Venice',\n",
       " \"A monument of Giosuè Carducci, an influencial poet, writer, literary critic and teacher. In Giardini della biennale an urban park in Castello, one of the six sestieri of Venice, Veneto, Italy. Carducci was a fierce critic of literature and society. In his youth he was an atheist, whose political views were vehemently hostile to the Catholic Church. During his life his views on religion shifted towards a socially oriented theism which he exposed in his famous .Discorso sulla libertà perpetua di San Marino. (.A Speech on San Marino's Perpetual Freedom.), pronounced on September the 30th, 1894 before the authorities and people of that ancient Republic and celebrating .the Universal God of Peoples, Mazzini's and Washington's God.. His anti-clerical revolutionary vehemence was prominently showcased in one famous poem, the deliberately blasphemous and provocative .Inno a Satana. was considered by Italian leftists of the time as a metaphor of the rebellious and freethinking spirit. The poem was composed in 1863 as a dinner party toast, published in 1865, then republished in 1869 by Bologna's radical newspaper, Il Popolo, as a provocation timed to coincide with the First Vatican Council, a time when revolutionary fervor directed against the papacy was running high as republicans pressed both politically and militarily for an end to the Vatican's domination over the papal states. had quite a revolutionary impact, Carducci's finest poetry came in later years. His collections Rime Nuove (New Rhymes) and Odi Barbare (Barbarian Odes) contain his greatest works. He was the first Italian to receive the Nobel Prize in Literature, in 1906. He was also appointed senator by the King of Italy (1890). In politics he remained a strong Liberal throughout his life. through the years he progressively evolved from republicanism to a sort of support to monarchy. Although his reputation rests primarily on his poetry, he also produced a large body of prose works. Indeed, his prose writings, including literary criticism, biographies, speeches and essays, fill some 20 volumes. Carducci was also an excellent translator and translated some of Goethe and Heine into Italian. While. Hymn to Satan. It has tags of castello, italy, italianrepublic, repubblicaitaliana, europe, mediterranean, venice, veneto, venezia, venesia, urban, city, sestieri, venetianrepublic, venetianghetto, giardinidellabiennale, venicegiardini, giosu, monument, sculpture, statue. (.Hymn to Satan..) .Satan\",\n",
       " \"Located in Central Venice this very large edifice catches your eye as you enter Campo San Zaccaria. In this image, the seven people heading into the church were almost perfectly arranged. History  The crypt The first church on the site was founded by Doge Giustiniano Participazio in the early 9th century to house the body of the saint to which it is dedicated, a gift of the Byzantine Emperor Leo V the Armenian, which it contains under the second altar on the right. The remains of various doges are buried in the crypt of the church. The original church was rebuilt in the 1170s (when the present campanile was built) and was replaced by a Gothic church in the 15th century. The remains of this building still stand, as the present church was built beside and not over it. The present church was built between 1458 and 1515. Antonio Gambello was the original architect, who started the building in the Gothic style, but the upper part of the facade with its arched windows and its columns, and the upper parts of the interior were completed by Mauro Codussi in early Renaissance style many years later. The facade is a harmonious Venetian mixture of late-Gothic and Renaissance styles. Monastery  View of the church and adjoining monastery by Gabriele Bella (1790), in Pinacoteca Querini Stampalia The church was originally attached to a Benedictine monastery of nuns also founded by Participazio and various other doges of the family. The nuns of this monastery mostly came from prominent noble families of the city and had a reputation for laxness in their observance of the monastic enclosure. The abbess was usually related to the doge. In 855, Pope Benedict III took refuge in the monastery while fleeing the violence of the Antipope Anastasius, whose election his supporters had challenged. Out of gratitude, Pope Benedict gave the nuns a large collection of relics which was the foundation of a large collection for which the monastery was famed, among which were those of Athanasius of Alexandria and a piece of the True Cross. A devastating fire destroyed the entire monastic complex in 1105. According to chronicles of the time, some one hundred nuns who had taken refuge in the cellars of the monastery died from smoke inhalation. Under the direction of Enrico Dandolo, the convent was reformed into a Cluniac house. The monastery had the tradition of being visited by the doge and his entire court annually at Easter in a ceremony which included presentation of the corno ducale (ducal cap), insignia of his office. This tradition is said to have begun in the 12th century after the nuns had donated land for the building of a ducal chapel, now St Mark's Basilica, and ended only in 1797, at the end of the Republic, when the monastery was suppressed by the invading forces of Napoleon's army. It has tags of venezia, religious, italy, architecture, summer, church, veneto. It has title of San Zaccaria\",\n",
       " \"The Rialto Bridge is the oldest of the four bridges on the Grand Canal. It is famous for both its architecture and its history. For years it was the hub of the economy of Venice. The Rialto Bridge was built between 1588 and 1591, based on a project by the architect Antonio da Ponte, to replace the previous wooden structure, which collapsed twice and burned on various occasions. The structure of the bridge is similar to the previous ones and consists of two inclined ramps joined by a portico in the center. The construction A single arch of over 28 meters of rope connects the two banks of the Grand Canal. More than twelve thousand elm wood poles were used to support the foundations of the structure, which houses 24 shops, distributed on both sides. The climb is divided into three ramps, a central one about ten meters wide with large steps divided into groups of five, and two lateral ramps about 3 meters wide. On the central arches, overlooking the Grand Canal, sculptures representing the Annunciation are visible on one side: a dove between the Archangel Gabriel and the Virgin Mary (by Agostino Rubini), on the other San Marco and San Teodoro (by Tiziano Aspetti ), from the end of the 16th century. On the bases the construction date of the bridge is engraved together with the legendary date of the foundation of Venice (which according to ancient tradition took place on March 25 of the year 421). The total cost for the realization of this daring engineering work, which managed to find a winning balance between technique and aesthetics, reached the fabulous figure of 250,000 ducats. The Rialto Bridge is the oldest of the four bridges on the Grand Canal. It is famous for both its architecture and its history. For years it was the hub of the economy of Venice. The Rialto Bridge was built between 1588 and 1591, based on a project by the architect Antonio da Ponte, to replace the previous wooden structure, which collapsed twice and burned on various occasions. The structure of the bridge is similar to the previous ones and consists of two inclined ramps joined by a portico in the center. The construction A single arch of over 28 meters of rope connects the two banks of the Grand Canal. More than twelve thousand elm wood poles were used to support the foundations of the structure, which houses 24 shops, distributed on both sides. The climb is divided into three ramps, a central one about ten meters wide with large steps divided into groups of five, and two lateral ramps about 3 meters wide. On the central arches, overlooking the Grand Canal, sculptures representing the Annunciation are visible on one side: a dove between the Archangel Gabriel and the Virgin Mary (by Agostino Rubini), on the other San Marco and San Teodoro (by Tiziano Aspetti ), from the end of the 16th century. On the bases the date of construction of the bridge is engraved together with the legendary date of the foundation of Venice (which according to ancient tradition took place on March 25 of the year 421). The total cost for the construction of this daring engineering work, which managed to find a winning balance between technique and aesthetics, reached the fabulous figure of 250,000 ducats.. It has title of Rialto bridge. It has tags of bridge, water, colors, venice, canal, hour, blue, gondolas, elitegalleryaoi, bestcapturesaoi, aoi. The Rialto Bridge is the oldest of the four bridges on the Grand Canal. It is famous both for its architecture and its history. For years it was the center of Venice's economy. The Rialto Bridge was built between 1588 and 1591, based on a project by the architect Antonio da Ponte, to replace the previous wooden structure, which collapsed twice and burned several times.. The structure of the bridge is similar to the previous ones and consists of two inclined ramps joined by a portal in the center. The construction A single arch of more than 28 meters of rope connects the two banks of the Grand Canal. More than twelve thousand elm wood poles were used to support the foundations of the structure, which houses 24 commercial premises, distributed on both sides. The ascent is divided into three ramps, a central one about ten meters wide with large steps divided into groups of five, and two side ramps about 3 meters wide.. In the central arches, overlooking the Grand Canal, there are sculptures representing the Annunciation on one side: a dove between the Archangel Gabriel and the Virgin Mary (by Agostino Rubini), on the other San Marco and San Teodoro (by Tiziano Aspetti ), late 16th century. The date of construction of the bridge is engraved on the bases together with the legendary date of the founding of Venice (which according to ancient tradition took place on March 25, 421).. The total cost for the realization of this daring work of engineering, which managed to find a winning balance between technique and aesthetics, reached the fabulous figure of 250,000 ducats\",\n",
       " \"San Giorgio Maggiore (Church of San Giorgio Maggiore) on the island of San Giorgio Maggiore, in Giudecca one of the six sestieri of Venice in Veneto, Italy. The first church on the island was built about 790, and in 982, the island was given to the Benedictine order by the Doge Tribuno Memmo. The Benedictines founded a monastery there, but in 1223, all the buildings on the island were destroyed by an earthquake. The church and monastery were rebuilt after the earthquake. The church, which had a nave with side chapels, was not in the same position as the present church, but farther back at the side of a small campo or square. There were cloisters in front of it, which were demolished in 1516. The monks were considering the rebuilding of the church from 1521. Palladio arrived in Venice in 1560, when the refectory of the monastery was being rebuilt. He made great improvements to this and in 1565, was asked to prepare a model for a new church. The model was completed and approved in 1566 and the foundation stone was laid in the presence of the Pope in the same year. The work was not finished before the death of Palladio in 1580, but the body of the church was complete by 1575, except for the choir behind the altar and the facade. The decoration of the interior was completed subsequently. The choir appears to have been designed in essentials by Palladio before his death and was built between 1580 and 1589. The façade, initially under the superintendence of Simone Sorella, was not commenced until 1599. The stonemason's contract provided that it was to follow Palladio's model and there were only minor changes. It was completed in 1610. The campanile (bell tower), first built in 1467, fell in 1774. it was rebuilt in neo-classic style by 1791. It was ascended by easy ramps and there is now also a lift. There is a fine view across Venice from the top. Information Source: (church),_Venice. It has tags of giudecca, italy, italianrepublic, repubblicaitaliana, europe, mediterranean, venice, veneto, venezia, venesia, urban, city, sestieri, venetianrepublic, sangiorgiomaggiore, churchofsangiorgiomaggiore, church, chapel, religion, christ, christian, christianity, dogetribunomemmo, worship benedictine, benedictines, palladio, simonesorella\",\n",
       " \"San Giorgio Maggiore (San Zorzi Mazor in Venetian) is a 16th-century Benedictine church on the island of the same name in Venice. The first church on the island was built about 790, and in 982 the island was given to the Benedictine order by the Doge Tribuno Memmo. The Benedictines founded a monastery there, but in 1223 all the buildings on the island were destroyed by an earthquake. The church and monastery were rebuilt after the earthquake. The church, which had a nave with side chapels, was not in the same position as the present church, but farther back at the side of a small campo or square. There were cloisters in front of it, which were demolished in 1516. The monks were considering the rebuilding of the church from 1521. Palladio arrived in Venice in 1560, when the refectory of the monastery was being rebuilt. He made great improvements to this and in 1565, was asked to prepare a model for a new church. The model was completed and approved in 1566 and the foundation stone was laid in the presence of the Pope in the same year. The work was not finished before the death of Palladio in 1580, but the body of the church was complete by 1575 except for the choir behind the altar and the facade. The decoration of the interior was completed subsequently. The choir appears to have been designed in essentials by Palladio before his death and was built between 1580 and 1589. The façade, initially under the superintendence of Simone Sorella, was not commenced until 1599. The stonemason's contract provided that it was to follow Palladio's model and there were only minor changes. It was completed in 1610. The campanile (bell tower), first built in 1467, fell in 1774. it was rebuilt in neo-classic style by 1791. It was ascended by easy ramps and there is now also a lift. There is a fine view across Venice from the top.. It has title of ALOA 372 Tour - Electric Taxi Venice\"]"
      ]
     },
     "execution_count": 651,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_df[final_df['num_sent']>20]['revised_text'].tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "399b1522",
   "metadata": {},
   "source": [
    "### Creating a one-hot Embedding for frequent Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "80ef994c",
   "metadata": {},
   "source": [
    "#### Generating a tag dictionary with appearance counts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bedba802",
   "metadata": {},
   "source": [
    "With the tags of the complete dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 660,
   "id": "5b097b0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict = {}\n",
    "for i in final_df.index:\n",
    "    list_now = final_df.loc[i]['tag_list']\n",
    "    for w in list_now:\n",
    "        if not w in tag_dict:\n",
    "            tag_dict[w] = {}\n",
    "            tag_dict[w]['count'] = 1\n",
    "            tag_dict[w]['origin'] = [i]\n",
    "        else:\n",
    "            tag_dict[w]['count'] += 1\n",
    "            tag_dict[w]['origin'].append(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b63c852d",
   "metadata": {},
   "source": [
    "Or alternatively, with the tags from the subset of unique and valid text data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 715,
   "id": "e9bd5029",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict = {}\n",
    "for i in candidates_new.index:\n",
    "    list_now = candidates_new.loc[i]['tag_list']\n",
    "    for w in list_now:\n",
    "        if not w in tag_dict:\n",
    "            tag_dict[w] = {}\n",
    "            tag_dict[w]['count'] = 1\n",
    "            tag_dict[w]['origin'] = [i]\n",
    "        else:\n",
    "            tag_dict[w]['count'] += 1\n",
    "            tag_dict[w]['origin'].append(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 866,
   "id": "750643d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2113"
      ]
     },
     "execution_count": 866,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tag_dict)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "434a6b4e",
   "metadata": {},
   "source": [
    "#### Converting tags to non-capitalized and translating tag words into English, if possible"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 718,
   "id": "24ff9373",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_dict_en = {}\n",
    "for k, v in tag_dict.items():\n",
    "    w = GoogleTranslator(source='auto', target='en').translate(k.lower())\n",
    "    if not w in tag_dict_en:\n",
    "        tag_dict_en[w] = {}\n",
    "        tag_dict_en[w]['count'] = v['count']\n",
    "        tag_dict_en[w]['origin'] = set(v['origin'])\n",
    "    else:\n",
    "        tag_dict_en[w]['count'] += v['count']\n",
    "        tag_dict_en[w]['origin'] = tag_dict_en[w]['origin'].union(set(v['origin']))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ec33bac",
   "metadata": {},
   "source": [
    "1865 out of 2113 tags remained after translating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 805,
   "id": "34944000",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2113, 1865)"
      ]
     },
     "execution_count": 805,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tag_dict),len(tag_dict_en)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a334aaee",
   "metadata": {},
   "source": [
    "#### Converting the tag dictionary to a pandas DataFrame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 803,
   "id": "298ade12",
   "metadata": {},
   "outputs": [],
   "source": [
    "tag_df = pd.DataFrame(tag_dict_en).T.sort_values(by='count',ascending=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df6978cd",
   "metadata": {},
   "source": [
    "It could be observed that this tag distribution has a very long tail, and the most frequent tags are only referring to the place, which may need to be filtered out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 817,
   "id": "5616a324",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>origin</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>italy</th>\n",
       "      <td>1935</td>\n",
       "      <td>{3, 4, 5, 4106, 13, 4124, 4129, 43, 4140, 46, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>venice</th>\n",
       "      <td>1730</td>\n",
       "      <td>{3, 4, 5, 4106, 13, 15, 16, 4129, 4140, 46, 41...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>venezia</th>\n",
       "      <td>919</td>\n",
       "      <td>{2054, 2055, 2057, 2058, 2059, 13, 15, 16, 207...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>veneto</th>\n",
       "      <td>237</td>\n",
       "      <td>{512, 513, 514, 1544, 1547, 2059, 2571, 3595, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>179</td>\n",
       "      <td>{1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>europe</th>\n",
       "      <td>155</td>\n",
       "      <td>{512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>church</th>\n",
       "      <td>155</td>\n",
       "      <td>{4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>architecture</th>\n",
       "      <td>112</td>\n",
       "      <td>{2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>canalgrande</th>\n",
       "      <td>101</td>\n",
       "      <td>{4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>water</th>\n",
       "      <td>83</td>\n",
       "      <td>{3, 5, 2570, 279, 2083, 2605, 2606, 2351, 2353...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>canal</th>\n",
       "      <td>83</td>\n",
       "      <td>{4613, 2057, 13, 527, 2324, 4375, 1815, 1816, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>urban</th>\n",
       "      <td>82</td>\n",
       "      <td>{4870, 4871, 4872, 4873, 2059, 2074, 3109, 311...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sanmarco</th>\n",
       "      <td>76</td>\n",
       "      <td>{1544, 2058, 1547, 3089, 3090, 3091, 3092, 309...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>dorsoduro</th>\n",
       "      <td>74</td>\n",
       "      <td>{2560, 2305, 2306, 2561, 2562, 2563, 2565, 256...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>provinzvenedig</th>\n",
       "      <td>74</td>\n",
       "      <td>{2569, 523, 2572, 535, 279, 537, 2331, 4382, 4...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gondola</th>\n",
       "      <td>72</td>\n",
       "      <td>{1542, 1543, 1546, 1550, 1809, 18, 1817, 1819,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>italianrepublic</th>\n",
       "      <td>67</td>\n",
       "      <td>{4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>italian republic</th>\n",
       "      <td>67</td>\n",
       "      <td>{4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mediterranean</th>\n",
       "      <td>67</td>\n",
       "      <td>{4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>museum</th>\n",
       "      <td>64</td>\n",
       "      <td>{2560, 2561, 2562, 2563, 2565, 2566, 4871, 487...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>venetianrepublic</th>\n",
       "      <td>63</td>\n",
       "      <td>{128, 129, 2178, 3200, 4870, 4871, 4872, 4873,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sestieri</th>\n",
       "      <td>63</td>\n",
       "      <td>{128, 129, 2178, 3200, 4870, 4871, 4872, 4873,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>byerwin</th>\n",
       "      <td>56</td>\n",
       "      <td>{2688, 900, 2436, 2437, 2696, 2569, 523, 2699,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>night</th>\n",
       "      <td>55</td>\n",
       "      <td>{4736, 4737, 3456, 1799, 2572, 13, 660, 3988, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ber</th>\n",
       "      <td>50</td>\n",
       "      <td>{2688, 900, 2436, 2696, 2569, 2699, 1933, 535,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>street</th>\n",
       "      <td>49</td>\n",
       "      <td>{1667, 1542, 1543, 2054, 2055, 1546, 2696, 155...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cannaregio</th>\n",
       "      <td>46</td>\n",
       "      <td>{128, 129, 2059, 3598, 3600, 3601, 3993, 3616,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>biennale</th>\n",
       "      <td>45</td>\n",
       "      <td>{4228, 3332, 3333, 4362, 3348, 283, 2078, 4659...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>cityscape</th>\n",
       "      <td>43</td>\n",
       "      <td>{3456, 1667, 1542, 1543, 2054, 2055, 1546, 13,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rialto</th>\n",
       "      <td>41</td>\n",
       "      <td>{1542, 1543, 1546, 1550, 1809, 1817, 1819, 182...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 count                                             origin\n",
       "italy             1935  {3, 4, 5, 4106, 13, 4124, 4129, 43, 4140, 46, ...\n",
       "venice            1730  {3, 4, 5, 4106, 13, 15, 16, 4129, 4140, 46, 41...\n",
       "venezia            919  {2054, 2055, 2057, 2058, 2059, 13, 15, 16, 207...\n",
       "veneto             237  {512, 513, 514, 1544, 1547, 2059, 2571, 3595, ...\n",
       "city               179  {1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...\n",
       "europe             155  {512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...\n",
       "church             155  {4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...\n",
       "architecture       112  {2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...\n",
       "canalgrande        101  {4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...\n",
       "water               83  {3, 5, 2570, 279, 2083, 2605, 2606, 2351, 2353...\n",
       "canal               83  {4613, 2057, 13, 527, 2324, 4375, 1815, 1816, ...\n",
       "urban               82  {4870, 4871, 4872, 4873, 2059, 2074, 3109, 311...\n",
       "sanmarco            76  {1544, 2058, 1547, 3089, 3090, 3091, 3092, 309...\n",
       "dorsoduro           74  {2560, 2305, 2306, 2561, 2562, 2563, 2565, 256...\n",
       "provinzvenedig      74  {2569, 523, 2572, 535, 279, 537, 2331, 4382, 4...\n",
       "gondola             72  {1542, 1543, 1546, 1550, 1809, 18, 1817, 1819,...\n",
       "italianrepublic     67  {4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...\n",
       "italian republic    67  {4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...\n",
       "mediterranean       67  {4870, 4871, 4872, 4873, 2059, 2074, 551, 552,...\n",
       "museum              64  {2560, 2561, 2562, 2563, 2565, 2566, 4871, 487...\n",
       "venetianrepublic    63  {128, 129, 2178, 3200, 4870, 4871, 4872, 4873,...\n",
       "sestieri            63  {128, 129, 2178, 3200, 4870, 4871, 4872, 4873,...\n",
       "byerwin             56  {2688, 900, 2436, 2437, 2696, 2569, 523, 2699,...\n",
       "night               55  {4736, 4737, 3456, 1799, 2572, 13, 660, 3988, ...\n",
       "ber                 50  {2688, 900, 2436, 2696, 2569, 2699, 1933, 535,...\n",
       "street              49  {1667, 1542, 1543, 2054, 2055, 1546, 2696, 155...\n",
       "cannaregio          46  {128, 129, 2059, 3598, 3600, 3601, 3993, 3616,...\n",
       "biennale            45  {4228, 3332, 3333, 4362, 3348, 283, 2078, 4659...\n",
       "cityscape           43  {3456, 1667, 1542, 1543, 2054, 2055, 1546, 13,...\n",
       "rialto              41  {1542, 1543, 1546, 1550, 1809, 1817, 1819, 182..."
      ]
     },
     "execution_count": 817,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_df.head(30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 806,
   "id": "a868062b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 806,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD8CAYAAACb4nSYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAbmElEQVR4nO3de3Bc5Z3m8e+vu3WXun2RLFstGxss21jtJIAgJMwlg4EYJhuYDcOQyQaHeNfFkjDZYWonsDW1mdqtmoS9DAlVW2RMcDC1WZIUyw7eCZchJiTMLDcZKGzZBt9iW7JlyZKt+73f/aOPQTaydWmpT3ef51OlUp/3vKfPT6dUT596zzlvm3MOEREJhpDfBYiISOYo9EVEAkShLyISIAp9EZEAUeiLiASIQl9EJEAifhdwMZWVlW758uV+lyEiklN27tx5yjlXNdG6rA795cuX09jY6HcZIiI5xcyOXGidhndERAJEoS8iEiAKfRGRAFHoi4gEiEJfRCRAFPoiIgGSl6HfNTDCC7tb6ewb9rsUEZGskpehf/hUH/f8z528ebjT71JERLJKXob+msUVhENG0/Euv0sREckqeRn6xQVh6haVs7tFoS8iMl5ehj5AIh5jV0s3+jpIEZGP5G/o10Q51TtEW8+Q36WIiGSNvA39dbUxAA3xiIiMk7ehf/mSKGawS6EvIvKhvA390sIIl1WVs7ul2+9SRESyxqShb2ZbzazNzHaPa1tgZi+Z2X7v93yv3czsETM7YGbvmdmV47bZ6PXfb2Yb5+bPOVeiJqrbNkVExpnKmf4TwIbz2h4Adjjn6oAd3jLAzUCd97MZeBRSHxLAd4BPA9cA3zn7QTGXEvEYJ7oGOdWri7kiIjCF0HfO/QY4/9HWW4Ft3uttwG3j2p90Ka8D88xsCfB54CXnXKdz7jTwEh//IJl1ibgu5oqIjDfTMf1q59wJ73UrUO29jgPHxvVr9tou1P4xZrbZzBrNrLG9vX2G5aWsrYkC0HRc4/oiIjALF3Jd6umnWXsCyjm3xTnX4JxrqKqa8Ht9pyxaXMCKyjJ2NetMX0QEZh76J71hG7zfbV57C7B0XL9ar+1C7XOuvibKbl3MFREBZh7624Gzd+BsBJ4d136XdxfPtUCXNwz0InCTmc33LuDe5LXNuUQ8RvPpAc70a5plEZGp3LL5FPAasNrMms1sE/A94EYz2w/c4C0DPAccAg4AjwH3AjjnOoH/DLzl/fwnr23OrfvwYq7G9UVEIpN1cM59+QKr1k/Q1wHfuMD7bAW2Tqu6WVDvXczdfbyL36mrzPTuRUSySt4+kXvWvNJCaueX6LZNERECEPqQGuJR6IuIBCT0E/EYv+3op3twxO9SRER8FYjQPzuuv0cPaYlIwAUi9DUdg4hISiBCv7K8iCWxYoW+iAReIEIfoL4mxm4N74hIwAUm9BPxKAfbe+kfHvW7FBER3wQm9NfFYzini7kiEmyBCX1dzBURCVDoL6ooorK8SOP6IhJogQl9M2NdPKozfREJtMCEPqSGePa39TI4MuZ3KSIivghU6NfXxBhLOva19vhdioiILwIV+utqUxdzd2mIR0QCKlChXxMrZn5pAU0KfREJqECFvpmRiMf0nbkiEliBCn1IXcx9v7WHoVFdzBWR4Ale6NfEGBlz7D/Z63cpIiIZF7zQj3vfmatxfREJoMCF/rIFpVQUR3QHj4gEUuBC38xIaJplEQmowIU+pIZ49p7oZmQs6XcpIiIZFdDQjzE8muRguy7mikiwBDb0AXY1a1xfRIIlkKG/YmEZZYVhmjSuLyIBE8jQD4WMtTWaZllEgieQoQ+pIZ6m492MJZ3fpYiIZExaoW9mf25mTWa228yeMrNiM1thZm+Y2QEz+5mZFXp9i7zlA9765bPyF8xQoibGwMgYh0/pYq6IBMeMQ9/M4sCfAQ3OuQQQBu4EHgIeds6tBE4Dm7xNNgGnvfaHvX6++eg7czWuLyLBke7wTgQoMbMIUAqcAK4HnvbWbwNu817f6i3jrV9vZpbm/mfssqoyigtCejJXRAJlxqHvnGsB/htwlFTYdwE7gTPOuVGvWzMQ917HgWPetqNe/4Xnv6+ZbTazRjNrbG9vn2l5k4qEQ1y+RBdzRSRY0hnemU/q7H0FUAOUARvSLcg5t8U51+Cca6iqqkr37S4qURNjz/FukrqYKyIBkc7wzg3AYedcu3NuBHgGuA6Y5w33ANQCLd7rFmApgLc+BnSksf+0JeJReoZGOdLZ72cZIiIZk07oHwWuNbNSb2x+PbAH+BVwu9dnI/Cs93q7t4y3/mXnnK+n2B9dzNUQj4gEQzpj+m+QuiD7NrDLe68twLeB+83sAKkx+8e9TR4HFnrt9wMPpFH3rKhbVEFhOKSvTxSRwIhM3uXCnHPfAb5zXvMh4JoJ+g4Cf5zO/mZbYSTE6sUVOtMXkcAI7BO5ZyXiMXa3dOPzSJOISEYo9ONRugZGaD494HcpIiJzTqFfk7qY26RxfREJgMCH/urFFURCpidzRSQQAh/6xQVh6qorNAePiARC4EMfIOHNra+LuSKS7xT6wLraGB19w7R2D/pdiojInFLoA/U1mmZZRIJBoQ+sXRIlZJqOQUTyn0IfKCkMs3JRuUJfRPKeQt+TqIlpDh4RyXsKfU99PMbJ7iHaenQxV0Tyl0Lfk6iJAtCki7kikscU+p56za0vIgGg0PeUF0W4tLJM4/oiktcU+uPUe9Msi4jkK4X+OOviUVrODNDZN+x3KSIic0KhP46mWRaRfKfQH+fsdAyaZllE8pVCf5xYaQHLFpTqtk0RyVsK/fMk4lHdwSMieUuhf576mhhHOvrpGhjxuxQRkVmn0D/Purgu5opI/lLon6de0zGISB5T6J9nYXkRNbFijeuLSF5S6E8gEY/ptk0RyUsK/Qkk4jEOn+qjd2jU71JERGaVQn8CiXgU52DvCY3ri0h+UehP4Ox0DLuaNcQjIvklrdA3s3lm9rSZ7TOzvWb2GTNbYGYvmdl+7/d8r6+Z2SNmdsDM3jOzK2fnT5h9i6LFLKoo0sVcEck76Z7p/wB4wTm3BvgksBd4ANjhnKsDdnjLADcDdd7PZuDRNPc9pxLxmG7bFJG8M+PQN7MY8HvA4wDOuWHn3BngVmCb120bcJv3+lbgSZfyOjDPzJbMdP9zLVETZX9bDwPDY36XIiIya9I5018BtAM/NrN3zOxHZlYGVDvnTnh9WoFq73UcODZu+2av7RxmttnMGs2ssb29PY3y0pOIx0g62Nuqs30RyR/phH4EuBJ41Dl3BdDHR0M5ADjnHOCm86bOuS3OuQbnXENVVVUa5aUncXY6Bt2vLyJ5JJ3QbwaanXNveMtPk/oQOHl22Mb73eatbwGWjtu+1mvLSktixSwoK9RDWiKSV2Yc+s65VuCYma32mtYDe4DtwEavbSPwrPd6O3CXdxfPtUDXuGGgrGNmJPSduSKSZyJpbn8f8BMzKwQOAXeT+iD5uZltAo4Ad3h9nwNuAQ4A/V7frJaoibLlN4cYGh2jKBL2uxwRkbSlFfrOuXeBhglWrZ+grwO+kc7+Mi0RjzGadLzf2sMnauf5XY6ISNr0RO5FnJ1bX0M8IpIvFPoXUTu/hGhxRE/mikjeUOhfxNmLubptU0TyhUJ/Eol4jL2tPYyMJf0uRUQkbQr9SSTiMYZHk+w/2et3KSIiaVPoTyLhfWeuxvVFJB8o9CexfGEZ5UURdmtcX0TygEJ/EqGQsbYmqtAXkbyg0J+CRE2MPSe6GUtOa+44EZGso9CfgkQ8yuBIkoPtupgrIrlNoT8FHz2ZqyEeEcltCv0puLSqnOKCkKZjEJGcp9CfgnDIWLtEF3NFJPcp9KdoXTxG0/EukrqYKyI5TKE/RfXxGH3DY/y2o8/vUkREZkyhP0WJmtTFXH19oojkMoX+FNVVl1MYCdF0XBdzRSR3KfSnqCAc4vLFFbqYKyI5TaE/DfXxGLtbukh986OISO5R6E9DoiZG9+AoxzoH/C5FRGRGFPrT8OGTuZpmWURylEJ/GlYtLicSMo3ri0jOUuhPQ1EkzKrqCt22KSI5S6E/Taknc7t1MVdEcpJCf5oS8SidfcOc6Br0uxQRkWlT6E9TfVxP5opI7lLoT9PaJVHCIaNJoS8iOUihP03FBWFWVpWzW9MxiEgOSjv0zSxsZu+Y2T94yyvM7A0zO2BmPzOzQq+9yFs+4K1fnu6+/VIfj2p4R0Ry0myc6X8L2Dtu+SHgYefcSuA0sMlr3wSc9tof9vrlpHXxGO09Q7R162KuiOSWtELfzGqBPwR+5C0bcD3wtNdlG3Cb9/pWbxlv/Xqvf85J6MlcEclR6Z7pfx/4SyDpLS8EzjjnRr3lZiDuvY4DxwC89V1e/3OY2WYzazSzxvb29jTLmxuXL4liBruaNa4vIrllxqFvZl8A2pxzO2exHpxzW5xzDc65hqqqqtl861lTXhRhRWWZzvRFJOdE0tj2OuCLZnYLUAxEgR8A88ws4p3N1wItXv8WYCnQbGYRIAZ0pLF/X62Lx3jrcKffZYiITMuMz/Sdcw8652qdc8uBO4GXnXNfAX4F3O512wg8673e7i3jrX/Z5fBcBomaGMe7BunoHfK7FBGRKZuL+/S/DdxvZgdIjdk/7rU/Diz02u8HHpiDfWdMfTwKoPv1RSSnpDO88yHn3CvAK97rQ8A1E/QZBP54NvaXDeq9L0rf3dLF76/KzmsPIiLn0xO5MxQrKeCShaWaW19EcopCPw2Jmpju4BGRnKLQT0MiHuNY5wBd/SN+lyIiMiUK/TQkvIu5TTrbF5EcodBPQ6JGc+uLSG5R6Kdhflkh8Xklum1TRHKGQj9NiXhUX6giIjlDoZ+mRE2MQ6f66BnUxVwRyX4K/TQlalPj+ns0xCMiOUChn6azF3M1ri8iuUChn6aqiiKqo0V6MldEcoJCfxYkamIKfRHJCQr9WZCIxzjY3kv/8OjknUVEfKTQnwVXL19A0sGz7x73uxQRkYtS6M+C61Yu5KpL5vPwSx/obF9EsppCfxaYGf/hljW09Qzxo1cP+12OiMgFKfRnyVWXLGBD/WL+7tcHae/RVyiKSHZS6M+iv9ywmsHRJI/s2O93KSIiE1Loz6JLq8r502uW8b/ePMrB9l6/yxER+RiF/iz71g11FEdC/NcX3ve7FBGRj1Hoz7LK8iLu+f3LeKGplZ1HOv0uR0TkHAr9ObDpd1ewqKKIv3luH845v8sREfmQQn8OlBZGuP/GVew8cpoXm1r9LkdE5EMK/Tly+1W11C0q56EX3mdkLOl3OSIigEJ/zkTCIR64eQ2HT/Xx0zeP+l2OiAig0J9T169ZxKdXLOD7v9xP75CmZxAR/yn051BqeobL6egbZsuvD/pdjoiIQn+ufXLpPP7FJ2t47NXDnOwe9LscEQm4GYe+mS01s1+Z2R4zazKzb3ntC8zsJTPb7/2e77WbmT1iZgfM7D0zu3K2/ohs9+9vWs1oMsn3f/mB36WISMClc6Y/CvyFc24tcC3wDTNbCzwA7HDO1QE7vGWAm4E672cz8Gga+84pyxaW8tVrl/Ozt47xwckev8sRkQCbceg750445972XvcAe4E4cCuwzeu2DbjNe30r8KRLeR2YZ2ZLZrr/XHPf9SspK4rw0PP7/C5FRAJsVsb0zWw5cAXwBlDtnDvhrWoFqr3XceDYuM2avbZAmF9WyL2fW8mOfW28drDD73JEJKDSDn0zKwf+N/DvnHPd49e51BwE05qHwMw2m1mjmTW2t7enW15Wufu65SyJFfPd5/eSTGp6BhHJvLRC38wKSAX+T5xzz3jNJ88O23i/27z2FmDpuM1rvbZzOOe2OOcanHMNVVVV6ZSXdYoLwvzFTat5r7mLX+w6MfkGIiKzLJ27dwx4HNjrnPvbcau2Axu91xuBZ8e13+XdxXMt0DVuGCgw/uiKOJcvifJfXtzH0OiY3+WISMCkc6Z/HfBV4Hoze9f7uQX4HnCjme0HbvCWAZ4DDgEHgMeAe9PYd84Kh4wHb17Dsc4BfvK6pmcQkcyKzHRD59w/AXaB1esn6O+Ab8x0f/nk91ZV8bt1lTzy8n6+dFUtsZICv0sSkYDQE7k++faGNXQNjPDoK5qeQUQyR6Hvk0Q8xh99Ks7Wfz5My5kBv8sRkYBQ6Pvo/ptWAfC3/6jpGUQkMxT6PqqdX8rdn13OM+80s+d49+QbiIikSaHvs3s/t5JocQHfe0HTM4jI3FPo+yxWWsB916/kNx+08/K+k36XIyJ5TqGfBb76mUtYVV3Ot556l70nNMwjInNHoZ8FiiJhfnz3NZQVRfjaj9/U3TwiMmcU+lkiPq+EJ75+Nf3DY2zc+iZn+of9LklE8pBCP4usWRzlsbsaONrRz6ZtjQyOaG4eEZldCv0sc+2lC/n+nZ/i7aOn+bOn3mFMUzCLyCxS6GehW9Yt4TtfWMs/7jnJf3x2N6lpi0RE0jfjCddkbn3tuhW0dg/xw18fZHG0mPvW1/ldkojkAYV+Fvv2htW0dQ/y31/6gOpoMXdcvXTyjURELkKhn8XMjIdu/wTtvUM8+H92UVlRyPVrqiffUETkAjSmn+UKwiEe/VdXsXZJlHt/8jbvHD3td0kiksMU+jmgvCjC1q9dzaKKYr7+xFscau/1uyQRyVEK/RxRVVHEk1+/hpAZd219k7aeQb9LEpEcpNDPIcsry9j6tavp6B3mTx97g5++eZTTfXpyV0SmzrL5HvCGhgbX2NjodxlZ59X97fzV3+/mSEc/4ZDx2csW8oVPLOGmtYuZX1bod3ki4jMz2+mca5hwnUI/NznnaDrezS92neC5XSc40tFPJGR8dmUlf7husT4ARAJMoZ/nxn8A/OK9ExztTH0A/E5dJbdfVcuNa6spioT9LlNEMkShHyBnPwD+4b0TbH+3heNdg8wrLeC2T8W5o2Epa2uifpcoInNMoR9QY0nH/zt4ip83NvPi7laGx5Ik4lHuaFjKrZ+MEyst8LtEEZkDCn3hTP8wz757nJ83HqPpeDeFkRBf+fQy7ru+jgUa+xfJKwp9Ocfuli6efO23PL2zmbLCCPd87jK+ft0KSgo17i+SDxT6MqH9J3t46IV9/HJvG4ujxdx/4yq+dFUt4ZB92Mc5R/fAKBaCaLGGg0RygUJfLuqNQx189/l9vHvsDKuqy1mzOEpr9yAnvZ/BkSQFYeOmtYu585qlXHdZJaFxHwwikl0U+jIp5xzP727lkR37GRgZo7qimOpYMYujRVRHiznRNcgzbzdzun+EpQtK+JOGpVyxbD7jo39BeSGrqysw0weCiJ+yKvTNbAPwAyAM/Mg5970L9VXoZ5eh0TFebDrJU28c5bVDHRP2WRwt5g/WVPEHqxdx5SXzWVhWOOGHgHOO90/28OoHpygpDHPJwlJq55eyoKyQaHFEHxwiacia0DezMPABcCPQDLwFfNk5t2ei/gr97HWss5/jZwbOaTvS2c+v9rXx6v5T9A6NAlAYDlEdK6K6ophoSQEVxREKwyHe/G0nRzr6J3zvcMioW1TO7VfVctsVcRaUfnR3kQN6h0Y51TtEUSREfF7JOR8QzrkPl8/+b5//ATIwPMap3iEGR8aIzy+htNCfr5UYX6vIbMqm0P8M8NfOuc97yw8COOe+O1F/hX5uGh5N0nikkw9ae2jtHqK1a4C2niG6B0foGRylb2iU+poYn69fzPrLF5F0jqMd/TSfHuB0/zCn+4f55wMdvHvszKT7ipUUcFlVGVUVRbR2D7H3RDdlhWGiJQWc7B7EMJbEiimMhOgfHqOjd4i+4bFz3uMTtTEuqyrn/PgdHkvy+qFOFpYV0j04QnlRhEQ89rF+F5N0joGRMQZGkgwOjzEwMkZJQZjB0TH2tfYQLS5g6YISFnn1L6oooqIo9SE0NJrk8Kk+ImGjIByiwPtdGA6lliMhIiE7p56kc4w5GEsmGUs6xpKOUe/32dfJ89qSzlFUEKYoEvJ+whQXpPahjyT/fGrZPO76zPIZbZtNoX87sME596+95a8Cn3bOfXNcn83AZoBly5ZddeTIkYzVJ9nlg5M97NjbxvBo8pz2sqIwleVF9A6N0nS8m6OdfbR1D7GgrJBEPMbgyBhdAyNUR4sBaO0aZDSZpCiS2q6yopDK8iKKIiEOtffxTwdOTThVtWEk4lFO9Q4TLS6ge3CEE10DH+t3MYZRUhCmuDBMSUGI4oIwnX3DlBdFuHxJlL6hUY6d7uekF/jtPUMMjo59uO0lC0uJhIyRMcfwWJIR72d4NMnImGM0mfzY/iIhIxRK/Q6P+4mEjJAZkbARDoUIG4RDIcxSH9RDo2MMjiQZ8l6PjCUn+pMkQ9avqeavv1g/o20vFvpZ93WJzrktwBZInen7XI74aFV1BauqK+Z8P39+46o534dItsj0fPotwPhv96712kREJAMyHfpvAXVmtsLMCoE7ge0ZrkFEJLAyOrzjnBs1s28CL5K6ZXOrc64pkzWIiARZxsf0nXPPAc9ler8iIqLvyBURCRSFvohIgCj0RUQCRKEvIhIgWT3Lppm1A2eArglWx85rP38ZoBI4NSfFTV7PXL/HVPpO1udC6ydqn0pbJo/3RPuf6+3TPebTOd4Xatcxn531+f4/folzrmrCNc65rP4BtkylfaJ+QKPfdc7Ve0yl72R9pnpsp9qWyeM9G8d8utune8ync7x1zKfWX//j0//JheGd/zvF9gv1y5TZ2P903mMqfSfrM9VjO522TEp3/9PdPt1jPp3jfaF2HfPZWR+U//GPyerhnXSZWaO7wKRDMvt0vDNPxzyz8uF458KZfjq2+F1AwOh4Z56OeWbl/PHO6zN9ERE5V76f6YuIyDgKfRGRAFHoi4gESGBC38zKzGybmT1mZl/xu54gMLNLzexxM3va71qCwMxu8/6/f2ZmN/ldTxCY2eVm9kMze9rM/q3f9UxFToe+mW01szYz231e+wYze9/MDpjZA17zvwSeds79G+CLGS82T0znmDvnDjnnNvlTaX6Y5vH+e+//+x7gT/yoNx9M85jvdc7dA9wBXOdHvdOV06EPPAFsGN9gZmHgfwA3A2uBL5vZWlJfzXjM6zaWwRrzzRNM/ZhL+p5g+sf7r7z1MjNPMI1jbmZfBH5BjnxPSE6HvnPuN0Dnec3XAAe8s8xh4KfArUAzqeCHHP+7/TTNYy5pms7xtpSHgOedc29nutZ8Md3/cefcdufczUBODBvnY/jF+eiMHlJhHweeAb5kZo/i/6PV+WbCY25mC83sh8AVZvagP6XlpQv9j98H3ADcbmb3+FFYHrvQ//jnzOwRM/s7cuRMP+Nfl+gX51wfcLffdQSJc66D1PiyZIBz7hHgEb/rCBLn3CvAKz6XMS35eKbfAiwdt1zrtcnc0THPLB3vzMubY56Pof8WUGdmK8ysELgT2O5zTflOxzyzdLwzL2+OeU6Hvpk9BbwGrDazZjPb5JwbBb4JvAjsBX7unGvys858omOeWTremZfvx1wTromIBEhOn+mLiMj0KPRFRAJEoS8iEiAKfRGRAFHoi4gEiEJfRCRAFPoiIgGi0BcRCRCFvohIgPx/BbajN5Zf2+MAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tag_df['count'].value_counts().sort_index().plot(logx=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1dd8fddb",
   "metadata": {},
   "source": [
    "#### Deciding on the dimension of the embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8e67626",
   "metadata": {},
   "source": [
    "The number of 3 seems to provide an elbow point for the remained tags for embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 874,
   "id": "22919e93",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\461701484.py:3: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  xs.append(len(tag_df[tag_df['count']>i][tag_df['count']<200]))\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 874,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAhVUlEQVR4nO3de3zcdZ3v8ddnJrcmae6T3kvSNumFW4FS7onIorCuAnp0YVeR9VI44LqrnrML7p7Vsx6O+1BXV4/CLgirrAjLAioPRREVWxAKpLSU0tI2vdGU0KZJ27RJc53P+WN+aaellzSZ5JfMvJ+Pxzzym+/vNzOfzKN9f3/5/i5fc3dERCQzRMIuQERERo9CX0Qkgyj0RUQyiEJfRCSDKPRFRDJIVtgFnExFRYVXVVWFXYaIyLixYsWK3e4eO9a6MR/6VVVVNDQ0hF2GiMi4YWbbjrdOwzsiIhlEoS8ikkEU+iIiGUShLyKSQRT6IiIZRKEvIpJBFPoiIhkkLUO/q7eff1u6iec27g67FBGRMSUtQz8nGuHeZzfz6IrtYZciIjKmpGXoRyLGZTUxlm3cTTyuSWJERAakZegD1NfGaOvo4fW32sMuRURkzEjb0L+0pgKAZRtbQq5ERGTsSNvQryjM5cxpxSxdr9AXERmQtqEPUFdbwStv7qG9qzfsUkRExoS0Dv362kr64s7zja1hlyIiMiakdeifM7OEwtwsjeuLiATSOvSzoxEunl3O0vUtuOvUTRGRtA59gPq5MXbsPcjm3R1hlyIiErqThr6Z3W9mu8xsTVLbf5rZquCx1cxWBe1VZnYwad2/Jr3mPDN7zcwazew7ZmYj8hsdpa4mMU2kzuIRERncnv4PgKuSG9z9T919obsvBB4DHk9avWlgnbvfktR+N/BpoCZ4HPGeI2VGWT6zYgUa1xcRYRCh7+7LgLZjrQv21j8CPHSi9zCzKUCRuy/3xOD6A8C1p1ztENXVxFi+uZWu3v7R+kgRkTFpuGP6lwE73X1jUlu1ma00s6VmdlnQNg1oStqmKWg7JjNbYmYNZtbQ0jL8PfT6uTG6euO8vPWYfZeISMYYbujfwJF7+c3ATHc/B/g88GMzKzrVN3X3e9x9kbsvisViwywRLqwuJycronF9Ecl4WUN9oZllAR8Ezhtoc/duoDtYXmFmm4BaYAcwPenl04O2UTEhJ8oF1WUa1xeRjDecPf0/At5w90PDNmYWM7NosDyLxAHbze7eDLSb2YXBcYAbgZ8N47NPWV1NjA07D9C87+BofqyIyJgymFM2HwJeAOaaWZOZfTJYdT3vPIBbB6wOTuF8FLjF3QcG0m8Fvg80ApuAXw6//MGrn5sYJlq2QXv7IpK5Tjq84+43HKf9pmO0PUbiFM5jbd8AnHGK9aVMTWUhk4vyWLZhN396/sywyhARCVXaX5E7wMyoq63g2Y0t9PXHwy5HRCQUGRP6kLjrZntXH6827Qu7FBGRUGRU6F86p4KIwVKN64tIhsqo0C/Oz2bhjBIdzBWRjJVRoQ9QVxvj1aa97OnoCbsUEZFRl3GhX18bwx2ea9wddikiIqMu40L/rOkllORna1xfRDJSxoV+NGJcOqeCZRs0m5aIZJ6MC31IjOvv2t/NG2/vD7sUEZFRlZGhX1+rWzKISGbKyNCfVJTHvMkTNa4vIhknI0MfEnv7DVv30NnTF3YpIiKjJmNDv642Rk9/nOWbW8MuRURk1GRs6C+qKmVCdlSzaYlIRsnY0M/NinLR7HKWbdRFWiKSOTI29AHqairYsruDN1s7wy5FRGRUZHTo18+tBGCp5s4VkQyR0aFfVZ7PjLIJGtcXkYyR0aFvZtTXxnhh0256+jSbloikv8FMjH6/me0yszVJbV82sx1mtip4/HHSujvMrNHM1pvZe5ParwraGs3s9tT/KkNTVxOjo6efFdv2hF2KiMiIG8ye/g+Aq47R/i13Xxg8ngQwswXA9cDpwWvuMrOomUWB7wFXAwuAG4JtQ3fxnAqyIsYyjeuLSAY4aei7+zKgbZDvdw3wsLt3u/sWoBFYHDwa3X2zu/cADwfbhq4wN4vzTivVuL6IZIThjOl/xsxWB8M/pUHbNGB70jZNQdvx2seE+rkx1ja3s2t/V9iliIiMqKGG/t3AbGAh0Az8c6oKAjCzJWbWYGYNLS0jvwdeV5O46+azG3ShloiktyGFvrvvdPd+d48D95IYvgHYAcxI2nR60Ha89uO9/z3uvsjdF8VisaGUeEoWTCmiojBH4/oikvaGFPpmNiXp6XXAwJk9TwDXm1mumVUDNcBLwMtAjZlVm1kOiYO9Twy97NSKRIy6mhjPbtxNPK7ZtEQkfWWdbAMzewh4F1BhZk3Al4B3mdlCwIGtwM0A7v66mT0CrAX6gNvcvT94n88ATwFR4H53fz3Vv8xw1NXGeHzlDta8tY+zppeEXY6IyIg4aei7+w3HaL7vBNvfCdx5jPYngSdPqbpRdFlNBWaJ2bQU+iKSrjL6itxk5YW5nDG1WLNpiUhaU+gnqa+N8cqbe2nv6g27FBGREaHQT1JXG6M/7jzfqFM3RSQ9KfSTnDOzhIm5WSzV+foikqYU+kmyoxEunlPOsg0tuOvUTRFJPwr9o9TXVrJj70E2tXSEXYqISMop9I9SV1sBoLN4RCQtKfSPMr00n9mxApYp9EUkDSn0j6GuNsbyza109faHXYqISEop9I+hrjZGd1+cl7YMdhoBEZHxQaF/DBdWl5OTFdG4voikHYX+MUzIiXJBdZnG9UUk7Sj0j6O+NsbGXQd4a+/BsEsREUkZhf5x1NUmJm/R3r6IpBOF/nHUVBYypThPs2mJSFpR6B+H2eHZtPr642GXIyKSEgr9E6ifG2N/Vx+vNu0NuxQRkZRQ6J/AJbMriBgsXa8hHhFJDwr9EyjOz+acmaUs3ahbLYtIelDon0RdTYzVTXtp6+gJuxQRkWE7aeib2f1mtsvM1iS1fd3M3jCz1Wb2EzMrCdqrzOygma0KHv+a9JrzzOw1M2s0s++YmY3Ib5RidbUVuMNzmk1LRNLAYPb0fwBcdVTb08AZ7n4WsAG4I2ndJndfGDxuSWq/G/g0UBM8jn7PMems6SWU5GdrXF9E0sJJQ9/dlwFtR7X92t37gqfLgekneg8zmwIUuftyT0xJ9QBw7ZAqHmXRiHHpnAqWbdRsWiIy/qViTP8TwC+Tnleb2UozW2pmlwVt04CmpG2agrZjMrMlZtZgZg0tLeHvYdfXxmjZ38265v1hlyIiMizDCn0z+zugD3gwaGoGZrr7OcDngR+bWdGpvq+73+Pui9x9USwWG06JKXHolgy6OldExrkhh76Z3QT8CfDnwZAN7t7t7q3B8gpgE1AL7ODIIaDpQdu4MKkoj3mTJ+o+PCIy7g0p9M3sKuBvgA+4e2dSe8zMosHyLBIHbDe7ezPQbmYXBmft3Aj8bNjVj6L62hgvb22jo7vv5BuLiIxRgzll8yHgBWCumTWZ2SeB7wITgaePOjWzDlhtZquAR4Fb3H3gIPCtwPeBRhJ/ASQfBxjz6mtj9PY7yze3hl2KiMiQZZ1sA3e/4RjN9x1n28eAx46zrgE445SqG0POqyplQnaUpRtauGL+pLDLEREZEl2RO0i5WVEunl2ucX0RGdcU+qegrjbG1tZOtrV2hF2KiMiQKPRPQb1m0xKRcU6hfwqqKgqYWZbPUoW+iIxTCv1TVFdbwfObWunp02xaIjL+KPRPUX1tJZ09/TRsazv5xiIiY4xC/xRdNLucrIixbINutSwi449C/xQV5maxqKpU4/oiMi4p9IegrjbGuuZ2drV3hV2KiMgpUegPwaFTNzV3roiMMwr9IZg/uYiKwlydry8i445CfwgiEaOutoJnN7bQH9dsWiIyfij0h6i+Nsaezl7W7NgXdikiIoOm0B+iS+dUYKZbMojI+KLQH6LywlzOnFasUzdFZFxR6A9DXU2Mldv3su9gb9iliIgMikJ/GOrnxuiPO8836tRNERkfFPrDsHBGCRNzs1i2UUM8IjI+KPSHITsa4ZI5FSxd34K7Tt0UkbFPoT9MdbUx3trXxaaWA2GXIiJyUoMKfTO738x2mdmapLYyM3vazDYGP0uDdjOz75hZo5mtNrNzk17z8WD7jWb28dT/OqOvrrYCgN+v1xCPiIx9g93T/wFw1VFttwO/dfca4LfBc4CrgZrgsQS4GxKdBPAl4AJgMfClgY5iPJtems/sWIHuwyMi48KgQt/dlwFHzxpyDfDDYPmHwLVJ7Q94wnKgxMymAO8Fnnb3NnffAzzNOzuScam+tpIXN7fS1dsfdikiIic0nDH9Se7eHCy/DUwKlqcB25O2awrajtf+Dma2xMwazKyhpWXsD5vU1VbQ3RfnxS2aTUtExraUHMj1xKkrKTt9xd3vcfdF7r4oFoul6m1HzIWzysnNiuiWDCIy5g0n9HcGwzYEP3cF7TuAGUnbTQ/ajtc+7uVlR1lcXaZbMojImDec0H8CGDgD5+PAz5LabwzO4rkQ2BcMAz0FvMfMSoMDuO8J2tJCfW2Mxl0H2LH3YNiliIgc12BP2XwIeAGYa2ZNZvZJ4J+AK81sI/BHwXOAJ4HNQCNwL3ArgLu3AV8BXg4e/xi0pYVDs2lpb19ExrCswWzk7jccZ9UVx9jWgduO8z73A/cPurpxZE5lIVOK81i2oYUbFs8MuxwRkWPSFbkpYmbU18Z4rnE3ff3xsMsRETkmhX4K1dXG2N/Vx6rte8MuRUTkmBT6KXTJnAqiEdNZPCIyZin0U6h4QjYLZ5ToYK6IjFkK/RSrr42xesc+2jp6wi5FROQdFPopVlcbwx2e1cQqIjIGKfRT7MxpxZTmZ2tcX0TGJIV+ikUjxqU1MZZt2E08rtm0RGRsUeiPgPraGLsPdLPu7fawSxEROYJCfwTU1SRm01q2QROriMjYotAfAZVFecyfUqRTN0VkzFHoj5C62goatrXR0d0XdikiIoco9EdIfU2M3n7nhU2tYZciInKIQn+EnFdVSn5OVKduisiYotAfIblZUS6aVc4yXaQlImOIQn8E1c+Nsa21k627O8IuRUQEUOiPqMvnVhIx+OJPXuNgT3/Y5YiIKPRH0oyyfL7x4bN5YXMrS/6jga5eBb+IhEuhP8I+eO50vvahs3iucTc3/8cKBb+IhEqhPwo+vGgGX73uTJZuaOHWB1+hu0/BLyLhGHLom9lcM1uV9Gg3s782sy+b2Y6k9j9Oes0dZtZoZuvN7L2p+RXGh+sXz+TO687gd2/s4rYHV9LTp3l0RWT0DTn03X29uy9094XAeUAn8JNg9bcG1rn7kwBmtgC4HjgduAq4y8yiw6p+nPnzC07jK9eczm/W7eQvH3qFXk2gLiKjLFXDO1cAm9x92wm2uQZ42N273X0L0AgsTtHnjxsfu6iKL79/AU+9vpPPPrRSwS8ioypVoX898FDS88+Y2Wozu9/MSoO2acD2pG2agrZ3MLMlZtZgZg0tLel3cdNNl1Tz9++bzy/XvM3n/nMVfQp+ERklww59M8sBPgD8V9B0NzAbWAg0A/98qu/p7ve4+yJ3XxSLxYZb4pj0qctm8cU/nsfPVzfzhf96lX5NuCIioyArBe9xNfCKu+8EGPgJYGb3Aj8Pnu4AZiS9bnrQlrGW1M2mL+587VfriZrx9Q+fTTRiYZclImksFaF/A0lDO2Y2xd2bg6fXAWuC5SeAH5vZN4GpQA3wUgo+f1y79V1ziMedb/x6A5GI8bUPnUVEwS8iI2RYoW9mBcCVwM1JzV8zs4WAA1sH1rn762b2CLAW6ANuc3edsA585t019MWdf/nNRqJmfPWDZyr4RWREDCv03b0DKD+q7WMn2P5O4M7hfGa6+qsrauiPO//vd41EIsad156h4BeRlEvF8I6kgJnx+Str6Y87d/1+E9EIfOWaMzBT8ItI6ij0xxAz43++dy79cefflm0mKxLhS+9foOAXkZRR6I8xZsbtV8+jP+58/7ktRMz4X38yX8EvIimh0B+DzIy/e998+uLO/X/YQlbUuOPqeQp+ERk2hf4YZWZ86f0LiLtzz7LNRMz426vmKvhFZFgU+mOYmfG/P3A6/XHnX5duIitifOE9tQp+ERkyhf4YZ2Z85Zoz6I87332mkWjE+NyVtWGXJSLjlEJ/HIhEjP973Zn0x51v/3Yj0Yjx2Stqwi5LRMYhhf44EYkY//Shs+h355tPbyAaMW67fE7YZYnIOKPQH0eiEePr/+1s4nHn60+tJyti3Fw/O+yyRGQcUeiPM9GI8Y0Pn02/w1d/+QbRiPGpy2aFXZaIjBMK/XEoKxrhWx9J7PH/n1+sI2LGJy6tDrssERkHFPrjVFY0wr9cv5D+uPOPP19LVtS48aKqsMsSkTEuVdMlSgiyoxG+c8M5XLlgEv/ws9f50fITTVEsIqLQH/dysiJ878/O5Yp5lfz9T9fw0Etvhl2SiIxhCv00kJMV4a6Pnsvlc2Pc8fhrPPLy9pO/SEQykkI/TeRmRbn7o+dxWU0Ff/v4ah5b0RR2SSIyBin000hedpR7b1zEJbMr+B+PvspPV2b0vPMicgwK/TQzEPwXVpfz+UdW8cSrb4VdkoiMIcMOfTPbamavmdkqM2sI2srM7Gkz2xj8LA3azcy+Y2aNZrbazM4d7ufLO03IiXLfTYs4v6qMz/3nKn6xujnskkRkjEjVnv7l7r7Q3RcFz28HfuvuNcBvg+cAVwM1wWMJcHeKPl+Okp+Txf03nc+5M0v47MMr+dUaBb+IjNzwzjXAD4PlHwLXJrU/4AnLgRIzmzJCNWS8gtws/v0vFnP29GJu+/FKvvDIqzTu2h92WSISolSEvgO/NrMVZrYkaJvk7gO7lm8Dk4LlaUDy+YRNQdsRzGyJmTWYWUNLS0sKSsxchblZ/PATi/n4RVX84rW3uPJby7jlP1awumlv2KWJSAhScRuGS919h5lVAk+b2RvJK93dzcxP5Q3d/R7gHoBFixad0mvlnSbmZfMP71/AbZfP5gfPb+WHz2/lV6+/zWU1Fdz6rjlcOKtMs3GJZIhh7+m7+47g5y7gJ8BiYOfAsE3wc1ew+Q5gRtLLpwdtMgrKC3P5wnvm8ofb383tV89jXfN+brh3OR+8+3l+s3Yn8bj6V5F0N6zQN7MCM5s4sAy8B1gDPAF8PNjs48DPguUngBuDs3guBPYlDQPJKJmYl80t9bN57m8v5yvXnkHL/m4+9UADV3/7WX62agd9/fGwSxSREWLuQ9+7M7NZJPbuITFU9GN3v9PMyoFHgJnANuAj7t5miTGE7wJXAZ3AX7h7w4k+Y9GiRd7QcMJNZJh6++P8fPVb3PXMJjbuOsDMsnxurp/Fh86dTl52NOzyROQUmdmKpLMpj1w3nNAfDQr90ROPO79Zt5Pv/X4Tr27fS+XEXD51WTV/dsFpFObqLtwi44VCX06Ju/P8plbu+n0jf2hspXhCNjddXMVNF1dRWpATdnkichIKfRmyVdv3ctczjfx67U7yc6L82eKZfOqyWUwuzgu7NBE5DoW+DNuGnfu5+/ebeOLVt4ia8aHzpnFz3WyqKgrCLk1EjqLQl5TZ3tbJvy3bxCMNTfT1x3nfWVP57/WzWTC1KOzSRCSg0JeU27W/i/ue28KPXthGR08/755Xya3vms2iqrKwSxPJeAp9GTH7Ont54IWt3P+HLezp7GVxdRm3XT6HupoKXeUrEhKFvoy4zp4+Hn5pO/cs28zb7V2cPrWI2y6fw3tPn0w0ovAXGU0KfRk1PX1xfrpyB3cv3cSW3R3MqijglnfN5tqF08jJ0pw9IqNBoS+jrj/u/HJNM3c9s4m1ze1MLc7j03WzuP78mUzI0VW+IiNJoS+hcXd+v6GFu5/ZxEtb2yjMzeK800pZXF3GBdVlnDm9mNwsdQIiqXSi0Ne19TKizIzL51Zy+dxKXtrSxk9X7eDlLW18/an1AORmRVg4o4QLqss4v7qMc2eWUqBbPoiMGP3vklGzuLqMxdWJUzrbOnp4eWsbL21p4+WtbXz3mUbiv4NoxDhjWjGLq0pZXF3O+VWllOTr1g8iqaLhHRkT9nf18sqbe3l5S6IjWLV9Lz3BLZ7nTpp4qMNYXF3GpCLdAkLkRDSmL+NOV28/q5v28dKWVl7c0sYr2/bQ0dMPwGnl+SyuSgwHXVBdxsyyfF0TIJJEoS/jXl9/nLXN7by05fCQ0J7OXgAmFeVyflWiA1hcXU5NZSERXRsgGUyhL2knHncaWw4c6gRe2tLG2+1dAJTkZ7PotIFOoIzTpxaRFdU1ApI5dPaOpJ1IxKidNJHaSRP56IWn4e407TnIi1vaEscFtrbxm3U7AcjPiSZOEw2GhBbOKNGMYJKxtKcvaWtXexcvb91z6LjA+p37cYecaIQFU4sSjymJn/MmTyQ/R/tAkh40vCNC4uZwDdsSQ0GvNu1l7VvttHf1AWAG1eUFzE/qCBZMKaJyYq4OEsu4o+EdEaA4P5sr5k/iivmTgMTVwm/t62LtW+2JR/M+Xmvaxy9WNx96TXlBzqEOYH7QGcyqKNAxAhm3hhz6ZjYDeACYBDhwj7t/28y+DHwaaAk2/aK7Pxm85g7gk0A/8Fl3f2oYtYsMi5kxrWQC00omcOWCSYfa27t6eaN5P2vf2sfa5nbWNe/n35/fSk9f4rqBnKwI8yZPZP7kokPDRPMmT2RiXnZYv4rIoA15eMfMpgBT3P0VM5sIrACuBT4CHHD3bxy1/QLgIWAxMBX4DVDr7v0n+hwN78hY0NsfZ3NLB2ub97GueX/wl0E7bR09h7aZWZZ/xNDQ/KlFTC3O0/CQjLoRGd5x92agOVjeb2brgGkneMk1wMPu3g1sMbNGEh3AC0OtQWS0ZEcjzJ08kbmTJ3LdOYk2d2dnezfrmhMdwEBH8NTatxnYlyqekH2oI5g/JdEZzKks1G2mJTQpGdM3syrgHOBF4BLgM2Z2I9AAfMHd95DoEJYnvayJ43QSZrYEWAIwc+bMVJQoknJmxuTiPCYX53H5vMpD7R3dfbzx9v5DHcG65nYefHEbXb2J4aHsqFFTOZH5QQdwWnk+M8vyOa08X0NEMuKGffaOmRUCS4E73f1xM5sE7CYxzv8VEkNAnzCz7wLL3f1HwevuA37p7o+e6P01vCPpoD/ubNndERwjOPxXQcv+7iO2K83PZmZ5AacFncDMsoEOoYDKibm60lgGZcTO3jGzbOAx4EF3fxzA3Xcmrb8X+HnwdAcwI+nl04M2kbQXjRhzKguZU1nIB86eeqh9f1cvb7Z18mZrJ9vaOtnW2sn2tk5Wbt/DL15rpj9+eKcsNytyqBOYWZ4fdAwFzCjLZ0bZBM1LIIMynLN3DLgPWOfu30xqnxKM9wNcB6wJlp8Afmxm3yRxILcGeGmony+SDibmZXP61GJOn1r8jnW9/XF27DnIm22JDuHN1g62tXbyZlsnL2xupbPn8DkQZjClKI+Z5Yf/MpiZ9NeCbk8tA4azp38J8DHgNTNbFbR9EbjBzBaSGN7ZCtwM4O6vm9kjwFqgD7jtZGfuiGSy7GiEqooCqioK3rHO3dl9oIc32zoSnULr4b8WnlnfQsv+piO2L8rLSnQEA53CwF8L5QVMLsrT5PUZRFfkiqShzp6+IzqD5L8WmvYcpC9p2Cg7apQX5FKSn01pfg6lBdmU5OdQGjw/tFyQk1ifn01RXraOL4xhuiJXJMPk52Qxb3IR8yYXvWNdX3+c5n1dhzuFtk5aD3Szp7OXvZ09rH97P3s7e9l7sPeIYwrJIpY4HTXRKRy/cygJOpGB7XTcIXwKfZEMkxWNBAd/87lkzvG3i8ed/d197O3sYU9nL3s6etgTLCfaDi837+tiXXM7ezp7Odh7/FHb/JzoUR1F9qEOorQgh7KCHCoKcykvzKG8IJfS/Gzd8iLFFPoickyRiFE8IZviCdmcVj7413X19ic6hI7ewx1GZ89Ry4mfO/YepK2jh/auXo410mwGJROyKS/MDTqERGdwaLkwl/KCnEOdRPEEDTudjEJfRFIqLzvKlOIJTCmeMOjX9MedfQd7aevoZveBHloP9Bxe7uim9UAPrR2JoafWjlb2BrOmHS0aMUrzBzqEd3YQyR1HeWEOhblZGXebDIW+iIQuGjHKguGdOZUn3763P86ezkTn0HpEx3C4g2g90M2re/bSeqCHA919x3yfnGgk0TkU5lBWkEtFQQ6lBYnOoDA3i4LcLApyo4eWj26bkB0dd52GQl9Exp3saITKiXlUTswb1PZdvf20dZy4g2jt6GHTrgO0dfSc8LhEsohBQc47O4fDHUSUwtxsCnOjR7VnHW7LOdw2GvdkUuiLSNrLy44ytWQCU0sGN+TUH3c6evro6E48DnT3Bz9P0NZzuK2tozNp2356+uOD+tycaISCoDOYWjyBR265aDi/9jEp9EVEjhKNGEV5iesRUqGnL364g+g5eUfS0d03Ynv9Cn0RkRGWkxUhJytxvCBsOgFWRCSDKPRFRDKIQl9EJIMo9EVEMohCX0Qkgyj0RUQyiEJfRCSDKPRFRDLImJ85y8xagG1DfHkFsDuF5Yxn+i6OpO/jSPo+DkuH7+I0d48da8WYD/3hMLOG400Zlmn0XRxJ38eR9H0clu7fhYZ3REQyiEJfRCSDpHvo3xN2AWOIvosj6fs4kr6Pw9L6u0jrMX0RETlSuu/pi4hIEoW+iEgGScvQN7OrzGy9mTWa2e1h1xMmM5thZs+Y2Voze93M/irsmsJmZlEzW2lmPw+7lrCZWYmZPWpmb5jZOjNL/fx844iZfS74f7LGzB4ys8FNwjuOpF3om1kU+B5wNbAAuMHMFoRbVaj6gC+4+wLgQuC2DP8+AP4KWBd2EWPEt4Ffufs84Gwy+Hsxs2nAZ4FF7n4GEAWuD7eq1Eu70AcWA43uvtnde4CHgWtCrik07t7s7q8Ey/tJ/KeeFm5V4TGz6cD7gO+HXUvYzKwYqAPuA3D3HnffG2pR4csCJphZFpAPvBVyPSmXjqE/Ddie9LyJDA65ZGZWBZwDvBhyKWH6F+BvgHjIdYwF1UAL8O/BcNf3zawg7KLC4u47gG8AbwLNwD53/3W4VaVeOoa+HIOZFQKPAX/t7u1h1xMGM/sTYJe7rwi7ljEiCzgXuNvdzwE6gIw9BmZmpSRGBaqBqUCBmX003KpSLx1DfwcwI+n59KAtY5lZNonAf9DdHw+7nhBdAnzAzLaSGPZ7t5n9KNySQtUENLn7wF9+j5LoBDLVHwFb3L3F3XuBx4GLQ64p5dIx9F8Gasys2sxySByIeSLkmkJjZkZizHadu38z7HrC5O53uPt0d68i8e/id+6edntyg+XubwPbzWxu0HQFsDbEksL2JnChmeUH/2+uIA0PbGeFXUCquXufmX0GeIrE0ff73f31kMsK0yXAx4DXzGxV0PZFd38yvJJkDPlL4MFgB2kz8Bch1xMad3/RzB4FXiFx1ttK0vCWDLoNg4hIBknH4R0RETkOhb6ISAZR6IuIZBCFvohIBlHoi4hkEIW+iEgGUeiLiGSQ/w+svgm7fobPrAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "xs = []\n",
    "for i in range(10):\n",
    "    xs.append(len(tag_df[tag_df['count']>i][tag_df['count']<200]))\n",
    "pd.Series(xs).plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 875,
   "id": "f1fbc1f5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\2378390123.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  tag_df[tag_df['count']>3][tag_df['count']<200]['count'].value_counts().sort_index().plot()\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:>"
      ]
     },
     "execution_count": 875,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAAsTAAALEwEAmpwYAAAekklEQVR4nO3de3zddZ3n8dcn55Jz0iZNStNQW0oLhQIi1ywiMChyGWBZipd1cVytK7MdHcbR1VmHER0dR13x7s7s6lZw7SByUajtACK1Isgo0PRmSy+0lF5p0rQ0TS+5nnzmj/M7ycn9NMlJ8st5Px+PPM7vfM/t01/Sd775/r6/39fcHRERCZ+isS5ARESGRgEuIhJSCnARkZBSgIuIhJQCXEQkpKKj+WHTpk3zOXPmjOZHioiE3urVqw+6e2XP9lEN8Dlz5lBTUzOaHykiEnpmtquvdg2hiIiElAJcRCSkFOAiIiGlABcRCSkFuIhISCnARURCSgEuIhJSoQjwpWv38pMX+pwGKSJSsAYNcDObb2brsr4azeyTZjbVzFaY2bbgtiJfRT6+fj8PvrQ7X28vIhJKgwa4u29194vc/SLgUuAEsBS4C1jp7mcBK4P7eZGIR2hqS+Xr7UVEQulkh1CuBV51913AAmBJ0L4EuG0E6+omGYvQ0taRr7cXEQmlkw3w24EHg+0qd98fbNcCVX29wMwWmVmNmdXU19cPqchkTD1wEZGecg5wM4sDtwI/6/mYpxfW7HNxTXdf7O7V7l5dWdnrYlo5ScSKaGpVgIuIZDuZHvhNwBp3rwvu15nZDIDg9sBIF5eR6YFrAWYRkS4nE+Dvp2v4BGA5sDDYXggsG6miekrEIwC0tGscXEQkI6cAN7NJwPXAY1nNXwOuN7NtwHXB/bxIxtIBrmEUEZEuOS3o4O7HgVN6tB0iPSsl7zIB3tyuABcRyQjFmZjJuHrgIiI9hSLAi6NBgGsqoYhIp1AEeKYH3qwAFxHpFI4A7zyIqVkoIiIZ4Qpw9cBFRDqFI8Dj6TIV4CIiXUIR4ImYxsBFRHpSgIuIhFQoAlxnYoqI9BaKAE/oIKaISC+hCPBIkRGPFinARUSyhCLAIT2M0qwhFBGRTuEKcC2rJiLSKTwBroWNRUS6CU2AF2sMXESkm9AEeDIe0TxwEZEs4QnwWETzwEVEsoQrwNUDFxHpFJoAT2gIRUSkm1wXNS43s5+b2RYz22xmbzOzqWa2wsy2BbcV+SxU0whFRLrLtQf+PeApdz8HuBDYDNwFrHT3s4CVwf28ScQ0C0VEJNugAW5mU4CrgfsA3L3V3RuABcCS4GlLgNvyU2KaDmKKiHSXSw98LlAP/H8zW2tm95rZJKDK3fcHz6kFqvp6sZktMrMaM6upr68fcqGZg5juPuT3EBGZSHIJ8ChwCfB9d78YOE6P4RJPp2qfyerui9292t2rKysrh1xoIljYuKVd4+AiIpBbgO8F9rr7i8H9n5MO9DozmwEQ3B7IT4lpuia4iEh3gwa4u9cCe8xsftB0LbAJWA4sDNoWAsvyUmEgE+DN7QpwERFID4/k4uPAA2YWB3YA/410+D9iZncAu4D35afEtIR64CIi3eQU4O6+Dqju46FrR7SaAWhVHhGR7kJzJmYyroWNRUSyhSfAO4dQNAtFRATCGODqgYuIAGEK8Hi6VA2hiIikhSbAi6PqgYuIZAtNgOsgpohId+EJcM0DFxHpJjQBrnngIiLdhSbAI0VGPFqkHriISCA0AQ6ZVXkU4CIiELIAL4lHOKEeuIgIELIA18r0IiJdwhXgcS2rJiKSEa4AVw9cRKRTuAJcY+AiIp3CFeCahSIi0ilUAa5ZKCIiXUIV4Mm4xsBFRDLCFeCxqGahiIgEcloT08x2AkeBFNDu7tVmNhV4GJgD7ATe5+6H81NmWknQA3d3zCyfHyUiMu6dTA/8Gne/yN0zixvfBax097OAlcH9vErGI6Q6nNaUllUTERnOEMoCYEmwvQS4bdjVDCJzSdlmrYspIpJzgDvwtJmtNrNFQVuVu+8PtmuBqhGvrofMog4n2trz/VEiIuNeTmPgwFXuvs/MpgMrzGxL9oPu7mbmfb0wCPxFALNnzx5WsSVxLeogIpKRUw/c3fcFtweApcBlQJ2ZzQAIbg/089rF7l7t7tWVlZXDKjazqIPmgouI5BDgZjbJzEoz28ANwEZgObAweNpCYFm+iswo0bqYIiKdchlCqQKWBtP2osBP3f0pM1sFPGJmdwC7gPflr8y0pHrgIiKdBg1wd98BXNhH+yHg2nwU1Z/MQUydjSkiErozMXUQU0QkI1QBXhJP/8GgHriISMgCXGPgIiJdwhXgmoUiItIpVAEeixiRIuNEq87EFBEJVYCbGSWxCE26FoqISLgCHCARj9Cka6GIiIQvwEviEU0jFBEhhAGejGldTBERCGOAa11MEREghAGuIRQRkbTQBXgyph64iAiEMcDjWpleRATCGOCxIh3EFBEhhAFeEo9qCEVEhBAGeCKmg5giIhDCAC+JR2hNddCe0un0IlLYQhfgnYs6aBhFRApc+AJcy6qJiABhDHAtqyYiApxEgJtZxMzWmtnjwf25ZvaimW03s4fNLJ6/MruUqAcuIgKcXA/8E8DmrPv3AN9x93nAYeCOkSysP4m4llUTEYEcA9zMZgH/Ebg3uG/AO4GfB09ZAtyWh/p6KdEQiogIkHsP/LvAZ4DM3L1TgAZ3z6yssBeY2dcLzWyRmdWYWU19ff1wagXS88ABWtoV4CJS2AYNcDO7BTjg7quH8gHuvtjdq929urKycihv0U0mwJvbNA9cRApbNIfnXAncamY3AwmgDPgeUG5m0aAXPgvYl78yuyRi6d85WpleRArdoD1wd/87d5/l7nOA24HfuPsHgGeA9wZPWwgsy1uVWYqj6oGLiMDw5oH/LfApM9tOekz8vpEpaWCZHrjGwEWk0OUyhNLJ3X8L/DbY3gFcNvIlDUxj4CIiaaE7E7M4qjFwEREIYYCbGfFoEc0aQhGRAhe6AAdIRIto0RCKiBS4cAZ4LKKDmCJS8EIb4DqIKSKFLpQBXhwt0kFMESl4oQzw9BCKeuAiUthCGuDqgYuIhDTAIwpwESl4oQzw9Bi4hlBEpLCFM8BjEZ3IIyIFL5QBnohGdCKPiBS8cAZ4rEgn8ohIwQtlgBdHdSKPiEgoA1zTCEVEQhvgEdo7nPaUeuEiUrhCGuCZVXkU4CJSuEIa4JlVeTSMIiKFK5QB3rkqj3rgIlLABg1wM0uY2Utmtt7MXjazfwja55rZi2a23cweNrN4/stNy/TAW9QDF5EClksPvAV4p7tfCFwE3GhmlwP3AN9x93nAYeCOvFXZQ3FUCxuLiAwa4J52LLgbC74ceCfw86B9CXBbPgrsS+Ygpk6nF5FCltMYuJlFzGwdcABYAbwKNLh7e/CUvcDMfl67yMxqzKymvr5+BErO7oErwEWkcOUU4O6ecveLgFnAZcA5uX6Auy9292p3r66srBxalT1oGqGIyEnOQnH3BuAZ4G1AuZlFg4dmAftGtrT+6SCmiEhus1Aqzaw82E4C1wObSQf5e4OnLQSW5anGXrrmgasHLiKFKzr4U5gBLDGzCOnAf8TdHzezTcBDZvZlYC1wXx7r7KZzHrh64CJSwAYNcHf/I3BxH+07SI+Hj7rOIRSNgYtIAQvlmZid0wjVAxeRAhbKANeJPCIiIQ3wSJERi5hO5BGRghbKAAetiykiEtoA18r0IlLoQhvgWlZNRApdaAO8OFqkIRQRKWihDfBELEJLe4rfbj3As6+MzEWyRETCJJczMcelRCxCc1sHX3p8E+XJGG8/e2QulCUiEhYh7oEXcfBYCzvqj9PY3D74C0REJpjQBnhxNMLWuqMAHG1uG+NqRERGX2gDPBErwj293dikHriIFJ7wBnhwOj1AU1uKVl3YSkQKTGgDvDgW6XZfwygiUmjCG+DBNcHPObUUQAcyRaTghDbAM9cEv3LeNEA9cBEpPKEN8PKSGLGIccWZpwA6kCkihSe0J/J84K2zuWreNCJFBkCjeuAiUmBC2wMvTcQ4f+YUypIxABqbFOAiUlhyWZX+NDN7xsw2mdnLZvaJoH2qma0ws23BbUX+y+2tLJH+I+KoDmKKSIHJpQfeDnza3c8DLgfuNLPzgLuAle5+FrAyuD/qJsWjFJmGUESk8Awa4O6+393XBNtHgc3ATGABsCR42hLgtjzVOKCiIqM0EdMQiogUnJMaAzezOcDFwItAlbvvDx6qBar6ec0iM6sxs5r6+vxc9rU0EdU8cBEpODkHuJlNBh4FPunujdmPubsD3tfr3H2xu1e7e3VlZX4u+VqWiGkeuIgUnJwC3MxipMP7AXd/LGiuM7MZweMzgAP5KXFwZcmo5oGLSMHJZRaKAfcBm93921kPLQcWBtsLgWUjX15uyhIxHcQUkYKTy4k8VwIfBDaY2bqg7bPA14BHzOwOYBfwvrxUmIOypA5iikjhGTTA3f15wPp5+NqRLWdoShNRzQMXkYIT2jMxs5UlYhxtaSfV0edxVBGRCWliBHhwOv0x9cJFpIBMjAAPTqfXgUwRKSQTI8AzF7RSgItIAZkQAV6a6YFrLriIFJAJEeBlCfXARaTwTIgAn6JrgotIAZoQAZ7pgf/+1UM8tbGWDk0nFJECMCECfHIiSnlJjKVr9/HRn6xm7Z7DY12SiEjeTYgAjxQZz33mGu6/4zIA9jU0j3FFIiL5NyECHNLDKBfMKgfgQKMCXEQmvgkT4JA+oScZi1B7RAEuIhPfhApwM6OqrJi6oy1jXYqISN5NqAAHmF6WoE5DKCJSACZcgJ+qABeRAjHhAryqrJi6xmbSy3SKiExcEzDAEzS3dWiVehGZ8CZcgE8vSwBoGEVEJrwJF+CnDhDgGlYRkYkkl1Xpf2RmB8xsY1bbVDNbYWbbgtuK/JaZu6qyYgDqGrtPJVy7+zDnf+FX7Dx4fCzKEhEZcbn0wH8M3Nij7S5gpbufBawM7o8LVf30wNfvaeB4a4onNuwfi7JEREbcoAHu7s8Bb/RoXgAsCbaXALeNbFlDl4hFmJKM9QrwfQ1NAKzYVDcWZYmIjLihjoFXuXumK1sLVI1QPSMiM5Uw297D6QBft6dB10oRkQlh2AcxPX1ksN+jg2a2yMxqzKymvr5+uB+Xk6qyBLU9xsD3NTQxszwJwK83HxiVOkRE8mmoAV5nZjMAgtt+E9HdF7t7tbtXV1ZWDvHjTk5VWaJXL3vv4SauPruS06YmWbGpdlTqEBHJp6EG+HJgYbC9EFg2MuWMjNMqSqhrbObw8VYATrS288bxVmZVJLn5/Bk8t+0gm/c3jnGVIiLDk8s0wgeBPwDzzWyvmd0BfA243sy2AdcF98eNt8+vpMPhma3pPwz2BePfsyqSfOwdZzIlGePupRu09JqIhFous1De7+4z3D3m7rPc/T53P+Tu17r7We5+nbv3nKUypi6YOYXppcWdM072NnQFeHlJnLtvPpc1uxv42eo9Y1mmiMiwTLgzMQGKiozrzqvi2VfqaW5Ldc5AmVVRAsC7L5nJvOmTeXKDxsJFJLwmZIADXH9eFSdaU/zh1UPsO9xEPFJE5eT0WZpmxrzKyZ1zw0VEwmjCBvjbzjiFkniEX27cz97DJ3hTeYKiIut8fGZFkr2HT+j6KCISWhM2wBOxCO+6eCaPrtnHqp1vMLMi2e3xWRVJmts6eCOYqSIiEjYTNsAB/uaG+cFp9S3MKi/p9ljmpJ7M+LiISNhM6ACvmBTnszefC6R73NkyBzQ1Di4iYRUd6wLy7T2XzKS5LcV153a/XEtmSGXv4RNjUZaIyLBN+AA3M/7r5af3ap+SjFGaiHae5CMiEjYTeghlMLMqSjQGLiKhVdABPrM8qTFwEQmtgg7wWRVJ9h5u6jUXfOfB450XwhqqptYU2w8cG9Z7iIgMpOAD/FhLO41N7Z1ttUeaueWfnudzyzYO8MqBnWht5wP3vsAN33mWjfuOjESpIiK9FHyAA+zJmonyj49v4lhLO89sOUBzW+qk37O1vYOP/WQN6/Y0MLk4yt1LN5DSVQ9FJA8KOsBnBif3/OjfXuOff7ONLz++iSc27OeKM09JX0dlx6HO526pbeT32w8C0J7qYPn61znR2t7t/To6nE//bD3PvlLPV9/1Fr604HzW7z3CT1/anXNNuw+d4NlX0isXuTvL1u3jSFPbcP+po6491cEjq/aw/4iOMYjkS0EH+NzKSZwyKc5ja/bxzadf4d7nX+Pi2eX8vw9eyqR4pNsCyJ99bAMf/vEqXjt4nPuef42/fnAtX3lic+fj7s4Xlr/Mv65/nbtuOofbL5vNgoveRPXpFdz7ux05X3PlK09u4s+XrKKxuY0N+47wiYfWce/vdoz4vz2fOjqcv310A5959I9c/+3neODFXbr2ukgeTPh54AOZXBxl1d3XkcoK12iRYWa8fX4lv95Ux5cXnM+h462s3dOAO3zqkXVs2X+UycVRfvrSbt576Swunl3Bd369jftf2MVfXH0GH337mUB6DvqCi2fy+V9sZNuBY5xdVTpgPc1tKZ575SBtKefZrfVsrT0KwIpNdXz6hvn52xEjyN356pObeXTNXj5y5Vy21DZy99KNLF/3Ol97zwXMnTZprEsUmTAKOsAhfe3wIqxX+/XnVfHkhlrW7W1gW91R3OH9l53Ggy/tIRmL8NhfXsGH7nuJjz+4lrOrSvnNlgP8l+rTuOumc7q/z7lVfP4XG1mxqW7QAP+37Qdpakthlg7trbVHMYMttUfZ88YJTptaMuDrR0uqw3ml7ig1uw6z6fUj3HHVGcybPhmA//vbV7n3+df48BVz+Pwt6csYPLxqD195YjM3fvc5Pn3D2XzkyrlEI93/+GtuS/HPv9nOpadXcM0500+6ptojzXzr6a1MKo4ybXKcaZOL01+lxVSWFjNtcpziaGT4/3gZMakOp6ktRXNbiqbW9G1zWwdNbanO9uzHmoLHWoLHm1pTtLR3UBKPMCUZoywZo7wkxpRk+qs8Ge/cLk1Eu12NdKIo+ADvzzvPqaIsEeWbv9pKMhZhZnmSL9/2FlrbnavOOoWzq0r55n++kC8/sYmdh47zZ2+dzZdufTNm3X9ITp2S4IJZU1ixqY47r5k34Geu2FRHaXGUa8+dzlMv19Lc1sEHLz+d+1/YxYpNdXzkqrn5/Cf3q6k1xbo9DdTsfIOaXYdZs/swR5u7xv837z/KYx+7ggdX7eYbv9rKgovexN/fcl7nvrj9stm8Y/50PveLjXz1yS08/sf93POeCzh3RhkArzc08Rf3r2bDviOYwaevP5s7r5nXa18O5HO/2Mhvtx4gGYtwtKW9z+eUJqJUdgZ7V8inA764M/grS4tJxAoz7N2dlvYOWnoEaVNbiubWFM3tKZpaO3oHbFZ7Z8C2dQVv12tTwXM6aE11DKnGRKyIRCxCMhahOFrEidYUDU1ttLb3/35mUJYIgj0I+bJkjPJkVuB3a48zJbg/KR45qZ/F0WSjeT3s6upqr6mpGbXPG66fvLCLz/0iPZ3ww1fM4Yu3vnlI7/NPK7fxrRWv8NJnr2V6WaLP53R0OJd9dSWXnzGVWy98E4vuXw3As//zHfz3f6nhlEnFPLjo8qH9Q07SgaPNrN55mJpdh6nZ+QYvv95IezCGPb+qlEvnVFB9egX/Yc5UVu18g089sp5bLpjBExv2846zK1n8oWpikd6HV9ydJzbs5wvLXuZIUxt/+Y4zufyMU/jrh9bS3NbB1997ASs21bF07T5ufsupfOO9FzKpePA+xtMv17Lo/tV89uZzWHT1mTS3pag/2sLBYy0cPNaavs26X3+spbOtsbnvsJ+c1ZPvCvgewR/cL4nnvx+U6vDOIE33PLsHac9AbWrr6LqfFaRdr81+Tff3GUokRIqMZCySDtZ4EYlohGQ8fT8dtkWdj6efEwmek24vDgK523t0vrarvTha1G9PurktxZGmNhpOtHGkqS3YbuVIUxuNTW00NGW3p9uOBO0DzRSLFllnyPfu5afb0r8A4r1+GYxUR8DMVrt7da92BXj/Ojqcd3//96zb08ADf/5Wrpw3bUjvs6W2kRu/+ztmlicpiff9DU11ODsOHud7t1/EDeedysX/+DSzp5bw9P94O19/ags/ePZVzqycPJx/Tk5OtKY6z04tjhZx4WnlnWF9yewKppTEuj3f3fmzH77IH3Ycovr0Cu6/460k+/k3Zhw+3sqXHt/E0rX7AJg7bRI//NClzJteirtz7+9e43/9cjNTJxVT0ePz+rL/SDOzKpL868ev6vMXx0Ba2lMcOtaaFfhByPe4f/BYCw0n+p4NVBKPMG1yMcXRkZkT4EBbqiNr+GDovdXiaBHJeKR7eGb1YBM9gjQ7NDNB2mfAxiIk4l3vc7L7fTxxd463psP/yIk2Gppau8I9+5dBU1uv9sbmtgF/4cWjRZ29/MUfqh7yMaC8BLiZ3Qh8D4gA97r7gKvThy3AAV6tP8bDq/bwmT+d32vcNlfuzj1PbWX3G8cHfF5pcYwv3HoeJfEoD720m6qyBNecM529h0/wjV9tpW2I/4lPRrSoiLfMnMKlcyo4/01TiOcQSnveOMH9L+zizmvmMSU5eOBmPLPlAL/bdpBPXHtWr18Mz287yEOrdtORw89nLFLEndfMG/QYw3C1tqcXADl4rIX6oy1ZPflWDh1vGdHvTyzSO1B79kqz24uDHm92+0C9VRkZHR3O0eb2zpBPB31r1/2sXwD/sODNTC/t+y/wwYx4gJtZBHgFuB7YC6wC3u/um/p7TRgDXERkrPUX4MP5u+cyYLu773D3VuAhYMEw3k9ERE7CcAJ8JrAn6/7eoK0bM1tkZjVmVlNfXz+MjxMRkWx5P/Lg7ovdvdrdqysrK/P9cSIiBWM4Ab4POC3r/qygTURERsFwAnwVcJaZzTWzOHA7sHxkyhIRkcEM+QwEd283s78CfkV6GuGP3P3lEatMREQGNKxTyNz9SeDJEapFREROQnhPnxIRKXCjeiq9mdUDuwZ4yjTg4CiVMxLCVG+YaoVw1ata8ydM9eaz1tPdvdc0vlEN8MGYWU1fZxuNV2GqN0y1QrjqVa35E6Z6x6JWDaGIiISUAlxEJKTGW4AvHusCTlKY6g1TrRCuelVr/oSp3lGvdVyNgYuISO7GWw9cRERypAAXEQmpcRPgZnajmW01s+1mdtdY15PNzE4zs2fMbJOZvWxmnwjav2hm+8xsXfB181jXmmFmO81sQ1BXTdA21cxWmNm24LZiHNQ5P2v/rTOzRjP75Hjat2b2IzM7YGYbs9r63JeW9r+Dn+M/mtkl46DWb5jZlqCepWZWHrTPMbOmrH38g9GsdYB6+/3em9nfBft2q5n96Tio9eGsOnea2bqgfXT2rbuP+Rfpa6m8CpwBxIH1wHljXVdWfTOAS4LtUtIrEZ0HfBH4m7Gur5+adwLTerR9Hbgr2L4LuGes6+zj56AWOH087VvgauASYONg+xK4GfglYMDlwIvjoNYbgGiwfU9WrXOynzeO9m2f3/vg/9x6oBiYG2RGZCxr7fH4t4C/H819O1564ON6dR933+/ua4Lto8Bm+li8IgQWAEuC7SXAbWNXSp+uBV5194HO1h117v4c8EaP5v725QLgXzztBaDczGaMSqH0Xau7P+3u7cHdF0hf+nlc6Gff9mcB8JC7t7j7a8B20tkxKgaq1cwMeB/w4GjVA+NnCCWn1X3GAzObA1wMvBg0/VXwp+mPxsOQRBYHnjaz1Wa2KGircvf9wXYtUDU2pfXrdrr/Bxiv+xb635fj/Wf5I6T/QsiYa2ZrzexZM/uTsSqqD31978fzvv0ToM7dt2W15X3fjpcADwUzmww8CnzS3RuB7wNnAhcB+0n/CTVeXOXulwA3AXea2dXZD3r677xxM4c0uKb8rcDPgqbxvG+7GW/7sj9mdjfQDjwQNO0HZrv7xcCngJ+aWdlY1ZclNN/7LO+ne+djVPbteAnwcb+6j5nFSIf3A+7+GIC717l7yt07gB8yin/ODcbd9wW3B4ClpGury/w5H9weGLsKe7kJWOPudTC+922gv305Ln+WzezDwC3AB4JfOARDEYeC7dWkx5TPHrMiAwN878frvo0C7wYezrSN1r4dLwE+rlf3Cca37gM2u/u3s9qzxzbfBWzs+dqxYGaTzKw0s036INZG0vt0YfC0hcCysamwT916MON132bpb18uBz4UzEa5HDiSNdQyJszsRuAzwK3ufiKrvdLMIsH2GcBZwI6xqbLLAN/75cDtZlZsZnNJ1/vSaNfXh+uALe6+N9Mwavt2tI7g5nCE92bSszteBe4e63p61HYV6T+R/wisC75uBu4HNgTty4EZY11rUO8ZpI/WrwdezuxP4BRgJbAN+DUwdaxrDeqaBBwCpmS1jZt9S/oXy36gjfS46x397UvSs0/+T/BzvAGoHge1bic9dpz52f1B8Nz3BD8f64A1wH8aJ/u23+89cHewb7cCN411rUH7j4GP9njuqOxbnUovIhJS42UIRURETpICXEQkpBTgIiIhpQAXEQkpBbiISEgpwEVEQkoBLiISUv8OYRTEssX4NxAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "tag_df[tag_df['count']>3][tag_df['count']<200]['count'].value_counts().sort_index().plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "974a80c6",
   "metadata": {},
   "source": [
    "A total of 366 tags remained in the embedding after filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 808,
   "id": "dcebc479",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\3099482604.py:1: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  tag_emb = tag_df[tag_df['count']>3][tag_df['count']<200]\n"
     ]
    }
   ],
   "source": [
    "tag_emb = tag_df[tag_df['count']>3][tag_df['count']<200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 809,
   "id": "504c9607",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\zzbn1\\AppData\\Local\\Temp\\ipykernel_23740\\1866734054.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tag_emb['edge_l'] = tag_emb['origin'].apply(lambda x: list(x))\n"
     ]
    }
   ],
   "source": [
    "tag_emb['edge_l'] = tag_emb['origin'].apply(lambda x: list(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 876,
   "id": "8287e5e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>origin</th>\n",
       "      <th>edge_l</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>city</th>\n",
       "      <td>179</td>\n",
       "      <td>{1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...</td>\n",
       "      <td>[1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>europe</th>\n",
       "      <td>155</td>\n",
       "      <td>{512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...</td>\n",
       "      <td>[512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>church</th>\n",
       "      <td>155</td>\n",
       "      <td>{4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...</td>\n",
       "      <td>[4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>architecture</th>\n",
       "      <td>112</td>\n",
       "      <td>{2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...</td>\n",
       "      <td>[2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>canalgrande</th>\n",
       "      <td>101</td>\n",
       "      <td>{4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...</td>\n",
       "      <td>[4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>r</th>\n",
       "      <td>4</td>\n",
       "      <td>{2632, 482, 483, 213}</td>\n",
       "      <td>[2632, 482, 483, 213]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>rust</th>\n",
       "      <td>4</td>\n",
       "      <td>{2393, 3005, 2903}</td>\n",
       "      <td>[2393, 3005, 2903]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sesteresanmarco</th>\n",
       "      <td>4</td>\n",
       "      <td>{3992, 4154, 1749, 3597}</td>\n",
       "      <td>[3992, 4154, 1749, 3597]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>verde</th>\n",
       "      <td>4</td>\n",
       "      <td>{2393, 3005, 2734, 2519}</td>\n",
       "      <td>[2393, 3005, 2734, 2519]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sightseeing</th>\n",
       "      <td>4</td>\n",
       "      <td>{3112, 3129, 2083, 2173}</td>\n",
       "      <td>[3112, 3129, 2083, 2173]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>366 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                count                                             origin  \\\n",
       "city              179  {1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...   \n",
       "europe            155  {512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...   \n",
       "church            155  {4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...   \n",
       "architecture      112  {2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...   \n",
       "canalgrande       101  {4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...   \n",
       "...               ...                                                ...   \n",
       "r                   4                              {2632, 482, 483, 213}   \n",
       "rust                4                                 {2393, 3005, 2903}   \n",
       "sesteresanmarco     4                           {3992, 4154, 1749, 3597}   \n",
       "verde               4                           {2393, 3005, 2734, 2519}   \n",
       "sightseeing         4                           {3112, 3129, 2083, 2173}   \n",
       "\n",
       "                                                            edge_l  \n",
       "city             [1544, 1547, 2059, 13, 2572, 1559, 2074, 1563,...  \n",
       "europe           [512, 513, 514, 1544, 2057, 1547, 2059, 529, 5...  \n",
       "church           [4873, 2058, 18, 787, 3107, 2605, 2614, 2360, ...  \n",
       "architecture     [2320, 787, 2605, 4144, 3901, 4926, 4414, 4928...  \n",
       "canalgrande      [4608, 4609, 4610, 3588, 1542, 1543, 1546, 155...  \n",
       "...                                                            ...  \n",
       "r                                            [2632, 482, 483, 213]  \n",
       "rust                                            [2393, 3005, 2903]  \n",
       "sesteresanmarco                           [3992, 4154, 1749, 3597]  \n",
       "verde                                     [2393, 3005, 2734, 2519]  \n",
       "sightseeing                               [3112, 3129, 2083, 2173]  \n",
       "\n",
       "[366 rows x 3 columns]"
      ]
     },
     "execution_count": 876,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag_emb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dcac4e5a",
   "metadata": {},
   "source": [
    "#### Constructing a Bipartite Graph for effective mapping of tag embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 738,
   "id": "3dd02b7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 810,
   "id": "73c33ebb",
   "metadata": {},
   "outputs": [],
   "source": [
    "G=nx.from_dict_of_lists(tag_emb.to_dict()['edge_l'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 811,
   "id": "8da75005",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 811,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nx.is_bipartite(G)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 812,
   "id": "b7790ba7",
   "metadata": {},
   "outputs": [],
   "source": [
    "emb_temp = nx.to_pandas_adjacency(G, dtype=int)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0592cbe4",
   "metadata": {},
   "source": [
    "Filter out the tags with only one letter would result a tag embedding dimension of 347"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 813,
   "id": "cd61114e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags = list(tag_emb.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 924,
   "id": "6e9d186c",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_n = [w for w in tags if len(w)>2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 926,
   "id": "f4c238d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "347"
      ]
     },
     "execution_count": 926,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(pd.Series(tags_n).apply(lambda x : x.lower()).unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 925,
   "id": "00c1357c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['city',\n",
       " 'europe',\n",
       " 'church',\n",
       " 'architecture',\n",
       " 'canalgrande',\n",
       " 'water',\n",
       " 'canal',\n",
       " 'urban',\n",
       " 'sanmarco',\n",
       " 'dorsoduro',\n",
       " 'provinzvenedig',\n",
       " 'gondola',\n",
       " 'italianrepublic',\n",
       " 'italian republic',\n",
       " 'mediterranean',\n",
       " 'museum',\n",
       " 'venetianrepublic',\n",
       " 'sestieri',\n",
       " 'byerwin',\n",
       " 'night',\n",
       " 'ber',\n",
       " 'street',\n",
       " 'cannaregio',\n",
       " 'biennale',\n",
       " 'cityscape',\n",
       " 'rialto',\n",
       " 'canon',\n",
       " 'travel',\n",
       " 'channel',\n",
       " 'reflection',\n",
       " 'santamariadellacarit',\n",
       " 'accademia',\n",
       " 'bridge',\n",
       " 'galleriedell',\n",
       " 'art',\n",
       " 'murano',\n",
       " 'nikon',\n",
       " 'giudecca',\n",
       " 'fuji movie',\n",
       " 'town',\n",
       " 'fujinon',\n",
       " 'car',\n",
       " 'marci',\n",
       " 'mosaic',\n",
       " 'boat',\n",
       " 'piazzasanmarco',\n",
       " 'basilicadisanmarco',\n",
       " 'sony',\n",
       " 'sanpolo',\n",
       " 'sonyalpha',\n",
       " 'baroque',\n",
       " 'san',\n",
       " 'basilica',\n",
       " 'sunset',\n",
       " 'markusdom',\n",
       " 'watercourse',\n",
       " 'sangiorgiomaggiore',\n",
       " 'castello',\n",
       " 'sestierecastello',\n",
       " 'eos',\n",
       " 'ponte',\n",
       " 'house',\n",
       " 'gothic',\n",
       " 'holidays',\n",
       " 'people',\n",
       " 'palaces',\n",
       " 'fullframe',\n",
       " 'palazzo',\n",
       " 'gothicstyle',\n",
       " 'leica',\n",
       " 'select',\n",
       " 'canoneos',\n",
       " 'unesco',\n",
       " 'landscape',\n",
       " 'herbst',\n",
       " 'outside',\n",
       " 'ducal palace',\n",
       " 'sonya',\n",
       " 'santamariadellasalute',\n",
       " 'marco',\n",
       " 'color',\n",
       " 'grandcanal',\n",
       " 'palace',\n",
       " 'exhibition',\n",
       " 'unescoworldheritagesite',\n",
       " 'venetian',\n",
       " 'monastery',\n",
       " 'blackandwhite',\n",
       " 'venexia',\n",
       " 'may',\n",
       " 'dogespalace',\n",
       " 'armenia',\n",
       " 'barca',\n",
       " 'yellow',\n",
       " 'sanlazzarodegliarmeni',\n",
       " 'stmarksbasilica',\n",
       " 'black and white',\n",
       " 'laguna',\n",
       " 'square',\n",
       " 'euro',\n",
       " 'gita',\n",
       " 'venicebiennale',\n",
       " 'palladio',\n",
       " 'reflections',\n",
       " \"st. mark's square\",\n",
       " 'nikond',\n",
       " 'market',\n",
       " 'baldassarelonghena',\n",
       " 'canonef',\n",
       " 'river',\n",
       " 'glass',\n",
       " 'batis',\n",
       " 'blackwhite',\n",
       " 'photography',\n",
       " 'boot',\n",
       " 'sculpture',\n",
       " 'red',\n",
       " 'colors',\n",
       " 'venetiasantalucia',\n",
       " 'stmarkssquare',\n",
       " 'film',\n",
       " 'great',\n",
       " 'alpha',\n",
       " 'boats',\n",
       " 'window',\n",
       " 'quartersanmarco',\n",
       " 'and',\n",
       " 'pont',\n",
       " 'show',\n",
       " 'wall',\n",
       " 'table',\n",
       " 'zeiss',\n",
       " 'sestieresanpolo',\n",
       " 'light',\n",
       " 'sky',\n",
       " 'mask',\n",
       " 'home',\n",
       " 'music',\n",
       " 'family',\n",
       " 'creek',\n",
       " 'trenitalia',\n",
       " 'railways of the italian state',\n",
       " 'summer',\n",
       " 'tourism',\n",
       " 'campanile',\n",
       " 'sestieredorsoduro',\n",
       " 'arsenal',\n",
       " 'santacroce',\n",
       " 'vaporetto',\n",
       " 'carnival',\n",
       " 'postcard',\n",
       " 'renaissance',\n",
       " 'italian',\n",
       " 'rosso',\n",
       " 'tamron',\n",
       " 'piazzadisanmarco',\n",
       " 'stream',\n",
       " 'brook',\n",
       " 'models',\n",
       " 'biennial architecture',\n",
       " 'district',\n",
       " 'reflex',\n",
       " 'brick',\n",
       " 'fog',\n",
       " 'july',\n",
       " 'revision',\n",
       " 'iconic',\n",
       " 'venetie',\n",
       " 'iphone',\n",
       " 'jacoporobusti',\n",
       " 'rivadeglischiavoni',\n",
       " 'leicaxtyp',\n",
       " 'jacopotintoretto',\n",
       " 'bacinodisanmarco',\n",
       " 'gondolas',\n",
       " 'longexposure',\n",
       " 'ghetto',\n",
       " 'isola',\n",
       " 'lagoon',\n",
       " 'romanesque',\n",
       " 'sonysel',\n",
       " 'building',\n",
       " 'rain',\n",
       " 'piazza',\n",
       " 'white',\n",
       " 'shadow',\n",
       " 'monument',\n",
       " 'winter',\n",
       " 'woman',\n",
       " 'statue',\n",
       " 'mist',\n",
       " 'rialtobridge',\n",
       " 'sea',\n",
       " 'black',\n",
       " 'crusaderknight',\n",
       " 'covid',\n",
       " 'silicon',\n",
       " 'dioxide',\n",
       " 'detail',\n",
       " 'puente',\n",
       " 'history',\n",
       " 'gwenaelpiaser',\n",
       " 'unlimitedphotos',\n",
       " 'provinceofvenice',\n",
       " 'giovanni',\n",
       " 'hungariannobleman',\n",
       " 'giambattistatiepolo',\n",
       " 'door',\n",
       " 'saintmarc',\n",
       " 'nopeople',\n",
       " 'cloister',\n",
       " 'analog',\n",
       " 'hat',\n",
       " 'darkness',\n",
       " 'panorama',\n",
       " 'stmarkscampanile',\n",
       " 'picturesque',\n",
       " 'galleriafranchetti',\n",
       " 'monochrome',\n",
       " 'clouds',\n",
       " 'campanzanipolo',\n",
       " 'cadoro',\n",
       " 'man',\n",
       " 'west facade',\n",
       " 'giorgiofranchetti',\n",
       " 'galleries',\n",
       " 'ocean',\n",
       " 'hat bridge',\n",
       " 'skyline',\n",
       " 'outdoors',\n",
       " 'pontedellacanonica',\n",
       " 'riodelpalazzo',\n",
       " 'usm',\n",
       " 'prime',\n",
       " 'triptych',\n",
       " 'places',\n",
       " 'streetphotography',\n",
       " 'crucifix oratory',\n",
       " 'rawtherapee',\n",
       " 'christian',\n",
       " 'direct market',\n",
       " 'mustsee',\n",
       " 'from',\n",
       " 'famous',\n",
       " 'green',\n",
       " 'salute',\n",
       " 'best',\n",
       " 'portal',\n",
       " 'clock',\n",
       " 'christ',\n",
       " 'giandomenicotiepolo',\n",
       " 'morning',\n",
       " 'travelphotography',\n",
       " 'janelea',\n",
       " 'joelbatham',\n",
       " 'chapel',\n",
       " 'white black',\n",
       " 'travel photography',\n",
       " 'worship',\n",
       " 'blue',\n",
       " 'facade',\n",
       " 'sestierecannaregio',\n",
       " 'religion',\n",
       " 'schoolgrandedisanrocco',\n",
       " 'sanmarcocampanile',\n",
       " 'christianity',\n",
       " 'lidodivenezia',\n",
       " 'dogenpalast',\n",
       " 'cloth',\n",
       " 'modernart',\n",
       " 'opera',\n",
       " 'abstract',\n",
       " 'camerabody',\n",
       " 'drasphotography',\n",
       " 'institutovenetodiscienze',\n",
       " 'meta',\n",
       " 'etr',\n",
       " 'eveofdestruction',\n",
       " 'long',\n",
       " 'triptychon',\n",
       " 'fomapan',\n",
       " 'long exposure',\n",
       " 'fusbodenmosaik',\n",
       " 'reflected',\n",
       " 'carlzeiss',\n",
       " 'sacristy',\n",
       " 'year',\n",
       " 'clothesline',\n",
       " 'palazzogrimani',\n",
       " 'highaltar',\n",
       " 'mirrorless',\n",
       " 'isoladisangio',\n",
       " 'bluesky',\n",
       " 'venetianlagoon',\n",
       " 'sangiorgio',\n",
       " 'small square',\n",
       " 'buchmalerei',\n",
       " 'gradual',\n",
       " 'grades',\n",
       " 'tintoretto',\n",
       " 'madonnadellorto',\n",
       " 'choralbooks',\n",
       " 'books',\n",
       " 'corona',\n",
       " 'locomotivafse',\n",
       " 'laundry',\n",
       " 'nature',\n",
       " 'weather',\n",
       " 'naturalhistorymuseum',\n",
       " 'naturalhistorymuseumofvenice',\n",
       " 'germany',\n",
       " 'niccol',\n",
       " 'whl',\n",
       " 'canals',\n",
       " \"doge's palace\",\n",
       " 'belfry',\n",
       " 'garden',\n",
       " 'nikkor',\n",
       " 'altar',\n",
       " 'adriaticsea',\n",
       " 'hukalo',\n",
       " 'val',\n",
       " 'portrait',\n",
       " 'romantic',\n",
       " 'buildings',\n",
       " 'gondolier',\n",
       " 'girl',\n",
       " 'wide',\n",
       " 'holiday',\n",
       " 'stone',\n",
       " 'giovannibellini',\n",
       " 'riodisantacaterina',\n",
       " 'colours',\n",
       " 'santa',\n",
       " 'riodellamisericordia',\n",
       " 'housing',\n",
       " 'puntadelladogana',\n",
       " 'painting',\n",
       " 'veniceitaly',\n",
       " 'cell',\n",
       " 'symmetry',\n",
       " 'sunrise',\n",
       " 'graphics',\n",
       " 'rust',\n",
       " 'sesteresanmarco',\n",
       " 'verde',\n",
       " 'sightseeing']"
      ]
     },
     "execution_count": 925,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40d59858",
   "metadata": {},
   "source": [
    "#### Tags embedding for data samples"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8ddb2d7",
   "metadata": {},
   "source": [
    "As results, the 347-dimensional tags embedding could be mapped back to 1104 data points. Note that the duplicated data points in terms of text data are not considered."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 927,
   "id": "750e9eac",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_emb = emb_temp.loc[tags_n].T.loc[~emb_temp.index.isin(tags)].sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 928,
   "id": "e9db5bad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>europe</th>\n",
       "      <th>church</th>\n",
       "      <th>architecture</th>\n",
       "      <th>canalgrande</th>\n",
       "      <th>water</th>\n",
       "      <th>canal</th>\n",
       "      <th>urban</th>\n",
       "      <th>sanmarco</th>\n",
       "      <th>dorsoduro</th>\n",
       "      <th>...</th>\n",
       "      <th>painting</th>\n",
       "      <th>veniceitaly</th>\n",
       "      <th>cell</th>\n",
       "      <th>symmetry</th>\n",
       "      <th>sunrise</th>\n",
       "      <th>graphics</th>\n",
       "      <th>rust</th>\n",
       "      <th>sesteresanmarco</th>\n",
       "      <th>verde</th>\n",
       "      <th>sightseeing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1104 rows × 347 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      city  europe  church  architecture  canalgrande  water  canal  urban  \\\n",
       "3        0       0       0             0            0      1      0      0   \n",
       "4        0       0       0             0            0      0      0      0   \n",
       "5        0       0       0             0            0      1      0      0   \n",
       "13       1       0       0             0            0      0      1      0   \n",
       "15       0       0       0             0            0      0      0      0   \n",
       "...    ...     ...     ...           ...          ...    ...    ...    ...   \n",
       "4981     0       0       0             1            0      0      0      0   \n",
       "4982     0       0       0             1            0      0      0      0   \n",
       "4983     0       0       0             1            0      0      0      0   \n",
       "4984     0       0       0             1            0      0      0      0   \n",
       "4985     0       0       0             1            0      0      0      0   \n",
       "\n",
       "      sanmarco  dorsoduro  ...  painting  veniceitaly  cell  symmetry  \\\n",
       "3            0          0  ...         0            0     0         0   \n",
       "4            0          0  ...         0            0     0         0   \n",
       "5            0          0  ...         0            0     0         0   \n",
       "13           0          0  ...         0            0     0         0   \n",
       "15           0          0  ...         0            0     0         0   \n",
       "...        ...        ...  ...       ...          ...   ...       ...   \n",
       "4981         0          0  ...         0            0     0         0   \n",
       "4982         0          0  ...         0            0     0         0   \n",
       "4983         0          0  ...         0            0     0         0   \n",
       "4984         0          0  ...         0            0     0         0   \n",
       "4985         0          0  ...         0            0     0         0   \n",
       "\n",
       "      sunrise  graphics  rust  sesteresanmarco  verde  sightseeing  \n",
       "3           0         0     0                0      0            0  \n",
       "4           0         0     0                0      0            0  \n",
       "5           0         0     0                0      0            0  \n",
       "13          0         0     0                0      0            0  \n",
       "15          0         0     0                0      0            0  \n",
       "...       ...       ...   ...              ...    ...          ...  \n",
       "4981        0         0     0                0      0            0  \n",
       "4982        0         0     0                0      0            0  \n",
       "4983        0         0     0                0      0            0  \n",
       "4984        0         0     0                0      0            0  \n",
       "4985        0         0     0                0      0            0  \n",
       "\n",
       "[1104 rows x 347 columns]"
      ]
     },
     "execution_count": 928,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_emb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 929,
   "id": "9c6c95b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_VEC = tags_emb.apply(lambda x: np.array(x), axis=1).rename('VECTOR')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 930,
   "id": "ce9755f3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3       [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4       [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "5       [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "13      [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "15      [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "                              ...                        \n",
       "4981    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4982    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4983    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4984    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "4985    [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...\n",
       "Name: VECTOR, Length: 1104, dtype: object"
      ]
     },
     "execution_count": 930,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags_VEC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9f5573d5",
   "metadata": {},
   "source": [
    "#### A subset with complet tag embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 931,
   "id": "0c9b1bba",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub = final_df.merge(tags_VEC, how='right', left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 932,
   "id": "3ac94721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>candownload</th>\n",
       "      <th>url_c</th>\n",
       "      <th>url_q</th>\n",
       "      <th>url_n</th>\n",
       "      <th>url_largest</th>\n",
       "      <th>owner</th>\n",
       "      <th>owner_loc</th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>...</th>\n",
       "      <th>week</th>\n",
       "      <th>unique_week</th>\n",
       "      <th>English</th>\n",
       "      <th>Italian</th>\n",
       "      <th>Other_Lang</th>\n",
       "      <th>num_sent</th>\n",
       "      <th>revised_text</th>\n",
       "      <th>text_data</th>\n",
       "      <th>text_len</th>\n",
       "      <th>VECTOR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>51870743082</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074308...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, bri...</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>51871789478</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187178947...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>7.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>51870742942</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5187074294...</td>\n",
       "      <td>47954272@N06</td>\n",
       "      <td>Tallinn, Estonia</td>\n",
       "      <td>...</td>\n",
       "      <td>Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>40</td>\n",
       "      <td>202140</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of adriaticsea, italy, venice, water</td>\n",
       "      <td>Venice, Italy. It has title of .... It has tag...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>51863124066</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186312406...</td>\n",
       "      <td>186704588@N07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Venice - Ponte dei Sospiri</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>...</td>\n",
       "      <td>3</td>\n",
       "      <td>202203</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>Bridge of Sighs. The Bridge of Sighs was built...</td>\n",
       "      <td>Ponte dei Sospiri. Il ponte dei Sospiri fu rea...</td>\n",
       "      <td>184.0</td>\n",
       "      <td>[1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>51860573333</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186057333...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186057333...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186057333...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5186057333...</td>\n",
       "      <td>61033692@N00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>A Murano</td>\n",
       "      <td>Murano, Venice, Italy</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>202201</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>Murano, Venice, Italy. It has tags of murano, ...</td>\n",
       "      <td>Murano, Venice, Italy. It has title of A Muran...</td>\n",
       "      <td>34.0</td>\n",
       "      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4981</th>\n",
       "      <td>51233430558</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343055...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145859</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_145859. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4982</th>\n",
       "      <td>51234286630</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123428663...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_145745</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_145745. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4983</th>\n",
       "      <td>51234294135</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429413...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_164727</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_164727. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4984</th>\n",
       "      <td>51234291635</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123429163...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_155646. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4985</th>\n",
       "      <td>51233434888</td>\n",
       "      <td>1</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>https://live.staticflickr.com/65535/5123343488...</td>\n",
       "      <td>83293103@N00</td>\n",
       "      <td>Milano, Italy</td>\n",
       "      <td>20210520_155525</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>20</td>\n",
       "      <td>202120</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>It has tags of biennale, architecture, archite...</td>\n",
       "      <td>. It has title of 20210520_155525. It has tags...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>[0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1104 rows × 35 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            index  candownload  \\\n",
       "3     51870743082            1   \n",
       "4     51871789478            1   \n",
       "5     51870742942            1   \n",
       "13    51863124066            1   \n",
       "15    51860573333            1   \n",
       "...           ...          ...   \n",
       "4981  51233430558            1   \n",
       "4982  51234286630            1   \n",
       "4983  51234294135            1   \n",
       "4984  51234291635            1   \n",
       "4985  51233434888            1   \n",
       "\n",
       "                                                  url_c  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "15    https://live.staticflickr.com/65535/5186057333...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_q  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "15    https://live.staticflickr.com/65535/5186057333...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                                  url_n  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   \n",
       "13    https://live.staticflickr.com/65535/5186312406...   \n",
       "15    https://live.staticflickr.com/65535/5186057333...   \n",
       "...                                                 ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   \n",
       "\n",
       "                                            url_largest          owner  \\\n",
       "3     https://live.staticflickr.com/65535/5187074308...   47954272@N06   \n",
       "4     https://live.staticflickr.com/65535/5187178947...   47954272@N06   \n",
       "5     https://live.staticflickr.com/65535/5187074294...   47954272@N06   \n",
       "13    https://live.staticflickr.com/65535/5186312406...  186704588@N07   \n",
       "15    https://live.staticflickr.com/65535/5186057333...   61033692@N00   \n",
       "...                                                 ...            ...   \n",
       "4981  https://live.staticflickr.com/65535/5123343055...   83293103@N00   \n",
       "4982  https://live.staticflickr.com/65535/5123428663...   83293103@N00   \n",
       "4983  https://live.staticflickr.com/65535/5123429413...   83293103@N00   \n",
       "4984  https://live.staticflickr.com/65535/5123429163...   83293103@N00   \n",
       "4985  https://live.staticflickr.com/65535/5123343488...   83293103@N00   \n",
       "\n",
       "             owner_loc                       title  \\\n",
       "3     Tallinn, Estonia                         ...   \n",
       "4     Tallinn, Estonia                         ...   \n",
       "5     Tallinn, Estonia                         ...   \n",
       "13                 NaN  Venice - Ponte dei Sospiri   \n",
       "15                 NaN                    A Murano   \n",
       "...                ...                         ...   \n",
       "4981     Milano, Italy             20210520_145859   \n",
       "4982     Milano, Italy             20210520_145745   \n",
       "4983     Milano, Italy             20210520_164727   \n",
       "4984     Milano, Italy             20210520_155646   \n",
       "4985     Milano, Italy             20210520_155525   \n",
       "\n",
       "                                            description  ...  week  \\\n",
       "3                                         Venice, Italy  ...    40   \n",
       "4                                         Venice, Italy  ...    40   \n",
       "5                                         Venice, Italy  ...    40   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...  ...     3   \n",
       "15                                Murano, Venice, Italy  ...     1   \n",
       "...                                                 ...  ...   ...   \n",
       "4981                                                NaN  ...    20   \n",
       "4982                                                NaN  ...    20   \n",
       "4983                                                NaN  ...    20   \n",
       "4984                                                NaN  ...    20   \n",
       "4985                                                NaN  ...    20   \n",
       "\n",
       "     unique_week  English  Italian Other_Lang  num_sent  \\\n",
       "3         202140      1.0      0.0        0.0       1.0   \n",
       "4         202140      1.0      0.0        0.0       1.0   \n",
       "5         202140      1.0      0.0        0.0       1.0   \n",
       "13        202203      0.0      1.0        0.0      10.0   \n",
       "15        202201      0.0      1.0        1.0       3.0   \n",
       "...          ...      ...      ...        ...       ...   \n",
       "4981      202120      0.0      1.0        0.0       1.0   \n",
       "4982      202120      0.0      1.0        0.0       1.0   \n",
       "4983      202120      0.0      1.0        0.0       1.0   \n",
       "4984      202120      0.0      1.0        0.0       1.0   \n",
       "4985      202120      0.0      1.0        0.0       1.0   \n",
       "\n",
       "                                           revised_text  \\\n",
       "3     It has tags of adriaticsea, italy, venice, bri...   \n",
       "4             It has tags of adriaticsea, italy, venice   \n",
       "5      It has tags of adriaticsea, italy, venice, water   \n",
       "13    Bridge of Sighs. The Bridge of Sighs was built...   \n",
       "15    Murano, Venice, Italy. It has tags of murano, ...   \n",
       "...                                                 ...   \n",
       "4981  It has tags of biennale, architecture, archite...   \n",
       "4982  It has tags of biennale, architecture, archite...   \n",
       "4983  It has tags of biennale, architecture, archite...   \n",
       "4984  It has tags of biennale, architecture, archite...   \n",
       "4985  It has tags of biennale, architecture, archite...   \n",
       "\n",
       "                                              text_data text_len  \\\n",
       "3     Venice, Italy. It has title of .... It has tag...      9.0   \n",
       "4     Venice, Italy. It has title of .... It has tag...      7.0   \n",
       "5     Venice, Italy. It has title of .... It has tag...      8.0   \n",
       "13    Ponte dei Sospiri. Il ponte dei Sospiri fu rea...    184.0   \n",
       "15    Murano, Venice, Italy. It has title of A Muran...     34.0   \n",
       "...                                                 ...      ...   \n",
       "4981  . It has title of 20210520_145859. It has tags...      9.0   \n",
       "4982  . It has title of 20210520_145745. It has tags...      9.0   \n",
       "4983  . It has title of 20210520_164727. It has tags...      9.0   \n",
       "4984  . It has title of 20210520_155646. It has tags...      9.0   \n",
       "4985  . It has title of 20210520_155525. It has tags...      9.0   \n",
       "\n",
       "                                                 VECTOR  \n",
       "3     [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "5     [0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "13    [1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "15    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "...                                                 ...  \n",
       "4981  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4982  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4983  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4984  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "4985  [0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, ...  \n",
       "\n",
       "[1104 rows x 35 columns]"
      ]
     },
     "execution_count": 932,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "45766782",
   "metadata": {},
   "source": [
    "### Save the Datasets generated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 847,
   "id": "5c12615a",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences.to_csv('Venezia/data_storage/sentences.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 819,
   "id": "ad08e9fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "candidates_new.to_csv('Venezia/data_storage/metadata_unique_text.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 818,
   "id": "4d512ad4",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_df.to_csv('Venezia/data_storage/metadata.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 933,
   "id": "d8544941",
   "metadata": {},
   "outputs": [],
   "source": [
    "tags_emb.to_csv('Venezia/data_storage/tags_embedding.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 934,
   "id": "9942398f",
   "metadata": {},
   "outputs": [],
   "source": [
    "sub.to_csv('Venezia/data_storage/meta_with_tags.csv',sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ce22692",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (env)",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}