03_google2022_inference.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef632465",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install threadpoolctl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2490d91c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from joblib import load"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f60d2f63",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Inputs\n",
    "path_g2022_vars = \"g2022_adid_01062021_11082022_text.csv.gz\"\n",
    "# this is the output table from part of data-post-production repo that merges preprocessed results.\n",
    "# Source: data-post-production/01-merge-results/01_merge_preprocessed_results\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1ba5a54f",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(path_g2022_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4418bb9f",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ad_id</th>\n",
       "      <th>wmp_creative_id</th>\n",
       "      <th>ad_type</th>\n",
       "      <th>ad_title</th>\n",
       "      <th>google_asr_text</th>\n",
       "      <th>aws_ocr_video_text</th>\n",
       "      <th>aws_ocr_img_text</th>\n",
       "      <th>csum_agg</th>\n",
       "      <th>advertiser_id</th>\n",
       "      <th>advertiser_name</th>\n",
       "      <th>ad_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>CR00000257354440376321</td>\n",
       "      <td>cid_16941</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Democratic candidate for governor. Stacey Abra...</td>\n",
       "      <td>» RECORDED\\nFOX31\\nSTACEYABRAMS\\nGOVERNOR\\nSpe...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3a0b45af177cfef8675852003ebb1e57838275d50d20b6...</td>\n",
       "      <td>AR08588079303567081473</td>\n",
       "      <td>Abrams For Governor</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>CR00000354386341527553</td>\n",
       "      <td>cid_1742</td>\n",
       "      <td>TEXT</td>\n",
       "      <td>Colorado Proposition GG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AR03715945093920718849</td>\n",
       "      <td>PROGRESSNOW COLORADO</td>\n",
       "      <td>Learn About Colorado Proposition GG Concerning...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>CR00000387990165651457</td>\n",
       "      <td>cid_3457</td>\n",
       "      <td>TEXT</td>\n",
       "      <td>Fight for Better Schools</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AR02250229023886016513</td>\n",
       "      <td>Mo Kashmiri for Sacramento County Office of Ed...</td>\n",
       "      <td>Mo Kashmiri for Sacramento County Board of Edu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>CR00000440354406924289</td>\n",
       "      <td>cid_39402</td>\n",
       "      <td>IMAGE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Better Jobs, Higher Wages, Yang York For New F...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AR11849446020951834625</td>\n",
       "      <td>YANG FOR NEW YORK INC</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>CR00000501995777556481</td>\n",
       "      <td>cid_38087</td>\n",
       "      <td>IMAGE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2 9 10 8 7 11 1.2 6 5 1 4 3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>AR01324533691786985473</td>\n",
       "      <td>KEVIN COMBS</td>\n",
       "      <td>1 Message Pending\\nView Pending Messages\\nNow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179151</th>\n",
       "      <td>CR07194247036013641729</td>\n",
       "      <td>cid_4339</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>How Much Energy Will the World Need?</td>\n",
       "      <td>We're headed toward an exciting. All renewable...</td>\n",
       "      <td>MATALO\\nRARE EARTH\\nPragerU;TWO DECADES\\n+\\nPr...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>a9430b5a86efa75ef2afe22d2389a6a858e242806c306f...</td>\n",
       "      <td>AR16784630625381384193</td>\n",
       "      <td>PRAGER UNIVERSITY FOUNDATION</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179152</th>\n",
       "      <td>CR04756463662258978817</td>\n",
       "      <td>cid_12037</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>We Can't Afford To Be Silent</td>\n",
       "      <td>One of my favorite lines on the couch on the a...</td>\n",
       "      <td>WE CAN'T AFFORD\\nTO BE SILENT\\nTURNING POINT U...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ea8f4e6d7ef123bdc508c39cb30aceabff3d347e42096c...</td>\n",
       "      <td>AR12552007562821632001</td>\n",
       "      <td>TURNING POINT USA, NFP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179153</th>\n",
       "      <td>CR12343661791669125121</td>\n",
       "      <td>cid_63</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>4-How to Beat Big Money</td>\n",
       "      <td>I've heard good things about Connecticut. So t...</td>\n",
       "      <td>FROM PAGE ONE\\nRell Keeps Focus On Reform\\nPAG...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>nan 604ebf1b6d00ec3ec53177baa6bbe3e74fcbecda20...</td>\n",
       "      <td>AR16131524841651044353</td>\n",
       "      <td>RECLAIMTHEAMERICANDREAM ORG</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179154</th>\n",
       "      <td>CR13232244551881588737</td>\n",
       "      <td>cid_7284</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>Not LeesBurk Ad: Inflation</td>\n",
       "      <td>This is our town of Leesburg, not Leesburg. Ho...</td>\n",
       "      <td>SUZANNE FOX VOTED AGAINST IT\\nSUZANNE D. FOX\\n...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>a5154514917f57ab4e8243301c9d5576656b6197babb76...</td>\n",
       "      <td>AR11905426212371038209</td>\n",
       "      <td>Suzanne Fox for Leesburg Mayor</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179155</th>\n",
       "      <td>CR15382706983039664129</td>\n",
       "      <td>cid_1321</td>\n",
       "      <td>VIDEO</td>\n",
       "      <td>California Is Burning: Inside The Collapse Of ...</td>\n",
       "      <td>California is burning. The state's been set, A...</td>\n",
       "      <td>SOCIALISM SUCKS\\nCALIFORNIA\\nTURNING\\nPOINT US...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>c81bfbe01dad5b878f12fa6e017aa844f7ffe29d32fddf...</td>\n",
       "      <td>AR12552007562821632001</td>\n",
       "      <td>TURNING POINT USA, NFP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>179156 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                         ad_id wmp_creative_id ad_type  \\\n",
       "0       CR00000257354440376321       cid_16941   VIDEO   \n",
       "1       CR00000354386341527553        cid_1742    TEXT   \n",
       "2       CR00000387990165651457        cid_3457    TEXT   \n",
       "3       CR00000440354406924289       cid_39402   IMAGE   \n",
       "4       CR00000501995777556481       cid_38087   IMAGE   \n",
       "...                        ...             ...     ...   \n",
       "179151  CR07194247036013641729        cid_4339   VIDEO   \n",
       "179152  CR04756463662258978817       cid_12037   VIDEO   \n",
       "179153  CR12343661791669125121          cid_63   VIDEO   \n",
       "179154  CR13232244551881588737        cid_7284   VIDEO   \n",
       "179155  CR15382706983039664129        cid_1321   VIDEO   \n",
       "\n",
       "                                                 ad_title  \\\n",
       "0                                                     NaN   \n",
       "1                                 Colorado Proposition GG   \n",
       "2                                Fight for Better Schools   \n",
       "3                                                     NaN   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "179151               How Much Energy Will the World Need?   \n",
       "179152                       We Can't Afford To Be Silent   \n",
       "179153                            4-How to Beat Big Money   \n",
       "179154                         Not LeesBurk Ad: Inflation   \n",
       "179155  California Is Burning: Inside The Collapse Of ...   \n",
       "\n",
       "                                          google_asr_text  \\\n",
       "0       Democratic candidate for governor. Stacey Abra...   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3                                                     NaN   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "179151  We're headed toward an exciting. All renewable...   \n",
       "179152  One of my favorite lines on the couch on the a...   \n",
       "179153  I've heard good things about Connecticut. So t...   \n",
       "179154  This is our town of Leesburg, not Leesburg. Ho...   \n",
       "179155  California is burning. The state's been set, A...   \n",
       "\n",
       "                                       aws_ocr_video_text  \\\n",
       "0       » RECORDED\\nFOX31\\nSTACEYABRAMS\\nGOVERNOR\\nSpe...   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3                                                     NaN   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "179151  MATALO\\nRARE EARTH\\nPragerU;TWO DECADES\\n+\\nPr...   \n",
       "179152  WE CAN'T AFFORD\\nTO BE SILENT\\nTURNING POINT U...   \n",
       "179153  FROM PAGE ONE\\nRell Keeps Focus On Reform\\nPAG...   \n",
       "179154  SUZANNE FOX VOTED AGAINST IT\\nSUZANNE D. FOX\\n...   \n",
       "179155  SOCIALISM SUCKS\\nCALIFORNIA\\nTURNING\\nPOINT US...   \n",
       "\n",
       "                                         aws_ocr_img_text  \\\n",
       "0                                                     NaN   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3       Better Jobs, Higher Wages, Yang York For New F...   \n",
       "4                             2 9 10 8 7 11 1.2 6 5 1 4 3   \n",
       "...                                                   ...   \n",
       "179151                                                NaN   \n",
       "179152                                                NaN   \n",
       "179153                                                NaN   \n",
       "179154                                                NaN   \n",
       "179155                                                NaN   \n",
       "\n",
       "                                                 csum_agg  \\\n",
       "0       3a0b45af177cfef8675852003ebb1e57838275d50d20b6...   \n",
       "1                                                     NaN   \n",
       "2                                                     NaN   \n",
       "3                                                     NaN   \n",
       "4                                                     NaN   \n",
       "...                                                   ...   \n",
       "179151  a9430b5a86efa75ef2afe22d2389a6a858e242806c306f...   \n",
       "179152  ea8f4e6d7ef123bdc508c39cb30aceabff3d347e42096c...   \n",
       "179153  nan 604ebf1b6d00ec3ec53177baa6bbe3e74fcbecda20...   \n",
       "179154  a5154514917f57ab4e8243301c9d5576656b6197babb76...   \n",
       "179155  c81bfbe01dad5b878f12fa6e017aa844f7ffe29d32fddf...   \n",
       "\n",
       "                 advertiser_id  \\\n",
       "0       AR08588079303567081473   \n",
       "1       AR03715945093920718849   \n",
       "2       AR02250229023886016513   \n",
       "3       AR11849446020951834625   \n",
       "4       AR01324533691786985473   \n",
       "...                        ...   \n",
       "179151  AR16784630625381384193   \n",
       "179152  AR12552007562821632001   \n",
       "179153  AR16131524841651044353   \n",
       "179154  AR11905426212371038209   \n",
       "179155  AR12552007562821632001   \n",
       "\n",
       "                                          advertiser_name  \\\n",
       "0                                     Abrams For Governor   \n",
       "1                                    PROGRESSNOW COLORADO   \n",
       "2       Mo Kashmiri for Sacramento County Office of Ed...   \n",
       "3                                   YANG FOR NEW YORK INC   \n",
       "4                                             KEVIN COMBS   \n",
       "...                                                   ...   \n",
       "179151                       PRAGER UNIVERSITY FOUNDATION   \n",
       "179152                             TURNING POINT USA, NFP   \n",
       "179153                        RECLAIMTHEAMERICANDREAM ORG   \n",
       "179154                     Suzanne Fox for Leesburg Mayor   \n",
       "179155                             TURNING POINT USA, NFP   \n",
       "\n",
       "                                                  ad_text  \n",
       "0                                                     NaN  \n",
       "1       Learn About Colorado Proposition GG Concerning...  \n",
       "2       Mo Kashmiri for Sacramento County Board of Edu...  \n",
       "3                                                     NaN  \n",
       "4           1 Message Pending\\nView Pending Messages\\nNow  \n",
       "...                                                   ...  \n",
       "179151                                                NaN  \n",
       "179152                                                NaN  \n",
       "179153                                                NaN  \n",
       "179154                                                NaN  \n",
       "179155                                                NaN  \n",
       "\n",
       "[179156 rows x 11 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1ab3c2ac",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Replace newline characters with spaces\n",
    "df['aws_ocr_video_text'] = df['aws_ocr_video_text'].str.replace('\\n', ' ')\n",
    "df['ad_text'] = df['ad_text'].str.replace('\\n', ' ')\n",
    "df['aws_ocr_img_text'] = df['aws_ocr_img_text'].str.replace('\\n', ' ')\n",
    "df['ad_title'] = df['ad_title'].str.replace('\\n', ' ')\n",
    "df['google_asr_text'] = df['google_asr_text'].str.replace('\\n', ' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d39ecb5c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1fd6eb2c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cols = ['advertiser_name', 'ad_title', 'google_asr_text', 'aws_ocr_img_text', \n",
    "        'aws_ocr_video_text', 'ad_text']\n",
    "df['combined'] = df[cols].apply(lambda row: ' '.join(row.values.astype(str)), axis=1)\n",
    "df['combined'] = df['combined'].str.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4d823b02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(179156, 12)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "75fe989a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5134"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df.advertiser_name.unique())\n",
    "#len(df.wmp_creative_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "03af8c3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt = df.groupby(['advertiser_id'])['combined'].apply(lambda x: ' '.join(x)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "c05e27fc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>advertiser_id</th>\n",
       "      <th>combined</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AR00000475401340059649</td>\n",
       "      <td>VINCE LEACH FOR SENATE nan nan A CHAMPION FOR ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AR00008638175664668673</td>\n",
       "      <td>TINA SMITH FOR MINNESOTA Protect Our Majority ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AR00013006535361822721</td>\n",
       "      <td>NICK LALOTA FOR CONGRESS Join Nick LaLota nan ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR00015213186479292417</td>\n",
       "      <td>Tom Lackey For Assembly 2012 nan nan PARKING -...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AR00017645374919409665</td>\n",
       "      <td>Democratic Party of Oregon nan Oil companies, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5143</th>\n",
       "      <td>AR18437367850864738305</td>\n",
       "      <td>AMERICANS FOR PROSPERITY nan Farm started in 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5144</th>\n",
       "      <td>AR18437938703557984257</td>\n",
       "      <td>Vote No On Amendment 1 Workers Rights Amendmen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5145</th>\n",
       "      <td>AR18440333439883280385</td>\n",
       "      <td>Kane County Democratic Central Cmte Vote for K...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5146</th>\n",
       "      <td>AR18441871450492108801</td>\n",
       "      <td>SAM GRAVES VICTORY FUND Elect Sam Graves nan n...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5147</th>\n",
       "      <td>AR18445881919154421761</td>\n",
       "      <td>DEBRA ANN GARDNER Paid for by Vance Victory na...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5148 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               advertiser_id  \\\n",
       "0     AR00000475401340059649   \n",
       "1     AR00008638175664668673   \n",
       "2     AR00013006535361822721   \n",
       "3     AR00015213186479292417   \n",
       "4     AR00017645374919409665   \n",
       "...                      ...   \n",
       "5143  AR18437367850864738305   \n",
       "5144  AR18437938703557984257   \n",
       "5145  AR18440333439883280385   \n",
       "5146  AR18441871450492108801   \n",
       "5147  AR18445881919154421761   \n",
       "\n",
       "                                               combined  \n",
       "0     VINCE LEACH FOR SENATE nan nan A CHAMPION FOR ...  \n",
       "1     TINA SMITH FOR MINNESOTA Protect Our Majority ...  \n",
       "2     NICK LALOTA FOR CONGRESS Join Nick LaLota nan ...  \n",
       "3     Tom Lackey For Assembly 2012 nan nan PARKING -...  \n",
       "4     Democratic Party of Oregon nan Oil companies, ...  \n",
       "...                                                 ...  \n",
       "5143  AMERICANS FOR PROSPERITY nan Farm started in 1...  \n",
       "5144  Vote No On Amendment 1 Workers Rights Amendmen...  \n",
       "5145  Kane County Democratic Central Cmte Vote for K...  \n",
       "5146  SAM GRAVES VICTORY FUND Elect Sam Graves nan n...  \n",
       "5147  DEBRA ANN GARDNER Paid for by Vance Victory na...  \n",
       "\n",
       "[5148 rows x 2 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "55b51ba8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Load the best model weights\n",
    "mnb_clf = load('models/party_clf_pdid_rf.joblib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "826b0da6",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=12)]: Using backend ThreadingBackend with 12 concurrent workers.\n",
      "[Parallel(n_jobs=12)]: Done  26 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=12)]: Done 176 tasks      | elapsed:    0.1s\n",
      "[Parallel(n_jobs=12)]: Done 426 tasks      | elapsed:    0.4s\n",
      "[Parallel(n_jobs=12)]: Done 500 out of 500 | elapsed:    0.4s finished\n"
     ]
    }
   ],
   "source": [
    "pred = mnb_clf.predict(df_txt['combined'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6ae82f26",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt['party_all_clf'] = pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "db922e42",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt = df_txt[['advertiser_id', 'party_all_clf']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "5ef5ec7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>advertiser_id</th>\n",
       "      <th>party_all_clf</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AR00000475401340059649</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AR00008638175664668673</td>\n",
       "      <td>DEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AR00013006535361822721</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AR00015213186479292417</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AR00017645374919409665</td>\n",
       "      <td>DEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5143</th>\n",
       "      <td>AR18437367850864738305</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5144</th>\n",
       "      <td>AR18437938703557984257</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5145</th>\n",
       "      <td>AR18440333439883280385</td>\n",
       "      <td>DEM</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5146</th>\n",
       "      <td>AR18441871450492108801</td>\n",
       "      <td>REP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5147</th>\n",
       "      <td>AR18445881919154421761</td>\n",
       "      <td>DEM</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5148 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "               advertiser_id party_all_clf\n",
       "0     AR00000475401340059649           REP\n",
       "1     AR00008638175664668673           DEM\n",
       "2     AR00013006535361822721           REP\n",
       "3     AR00015213186479292417           REP\n",
       "4     AR00017645374919409665           DEM\n",
       "...                      ...           ...\n",
       "5143  AR18437367850864738305           REP\n",
       "5144  AR18437938703557984257           REP\n",
       "5145  AR18440333439883280385           DEM\n",
       "5146  AR18441871450492108801           REP\n",
       "5147  AR18445881919154421761           DEM\n",
       "\n",
       "[5148 rows x 2 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "21fde0fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "party_all_clf\n",
       "DEM      2966\n",
       "REP      2180\n",
       "OTHER       2\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_txt.party_all_clf.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8f1e8a39",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_txt.to_csv(\"party_all_clf_google_2022_advertiser_id.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1dec06d",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}