withFeatureSelectionTrials.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "2053bca5",
   "metadata": {},
   "source": [
    "# Pre-requisites"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e3a96335",
   "metadata": {},
   "outputs": [],
   "source": [
    "#********************** to be run only once ************************************\n",
    "#pip install -U sentence-transformers\n",
    "#pip install --user annoy\n",
    "#export CC=/usr/bin/clang ; export CXX=/usr/bin/clang++"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc34074e",
   "metadata": {},
   "source": [
    "# Code Starts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 304,
   "id": "11050fb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "import edaSQL\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot as plt\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bbc77c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#DB connection\n",
    "edasql = edaSQL.SQL()\n",
    "edasql.connectToDataBase(server='', #Info removed to comply with the NDA\n",
    "                         database='', #Info removed to comply with the NDA\n",
    "                         user='', #Info removed to comply with the NDA\n",
    "                         password='',#Info removed to comply with the NDA\n",
    "                         sqlDriver='ODBC Driver 17 for SQL Server')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ceea3ad1",
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type1s data:\n",
    "ctrQuery = \"select * from <> ctr\" #Info removed to comply with the NDA\n",
    "ctrData = pd.read_sql(ctrQuery, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79587116",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctrData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "id": "32d51b83",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'No Info'"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Missing values\n",
    "ctrData['Type1ingProcedureType'] = ctrData['Type1ingProcedureType'].replace('', 'No Info')\n",
    "ctrData['Type1ingProcedureType'][13833]\n",
    "\n",
    "ctrData['Type1Type'] = ctrData['Type1Type'].replace('', 'No Info')\n",
    "ctrData['Type1Type'][13833]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2c10223",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set seaborn plotting aesthetics\n",
    "sns.set(rc = {'figure.figsize':(30,20)})\n",
    "\n",
    "#Plot\n",
    "plot1 = sns.countplot(data=ctrData, x=\"Type1ingProcedureType\")\n",
    "plot1.bar_label(plot1.containers[0], size=20)\n",
    "plt.title('Type1s - Type1ing Procedure Type', fontsize=35)\n",
    "plt.xticks(fontsize=20, rotation ='vertical')\n",
    "plt.yticks(fontsize=20)\n",
    "plt.xlabel('Type1ing Procedure Type', fontsize=25)\n",
    "plt.ylabel('Type1s Count', fontsize=25)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "id": "662b8651",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 2.46 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Getting Type1s data: parsedPriceBracket, executionDeadlineDays\n",
    "ctrQuery_1 = \"select * \" #Info removed to comply with the NDA\n",
    "ctrData_1 = pd.read_sql(ctrQuery_1, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "813cbe1f",
   "metadata": {},
   "source": [
    "# parsedPriceBracket \n",
    "#   0 --'<0' \n",
    "#   1 --'0-100000' \n",
    "#\t2 --'100001-500000' \n",
    "#\t3 --'500001-1000000' \n",
    "#\t4 --'1000001-3000000' \n",
    "#\t5 --'3000001-9000000' \n",
    "#\t6 --'9000000-20000000' \n",
    "#\t7 --'20000001-50000000' \n",
    "#\t8 --'50000001-100000000' \n",
    "#\t9 --'100000001-200000000' \n",
    "#\t10 --'200000001-400000000' \n",
    "#\t11 --'400000001-1000000000' \n",
    "#\t12 --'>1000000000'\n",
    "\n",
    "# executionDeadlineDays (Brackets)\n",
    "#   1 --'0-15' \n",
    "#   2 --'15-30' \n",
    "#   3 --'31-45' \n",
    "#   4 --'46-60' \n",
    "#   5 --'61-90' \n",
    "#   6 --'91-120' \n",
    "#   7 --'121-150' \n",
    "#   8 --'151-180' \n",
    "#   9 --'181-270' \n",
    "#   10 --'271-365' \n",
    "#   11 --'366-730' \n",
    "#   12 --'731-1095' \n",
    "#   13 --'1096-1460' \n",
    "#   14 --'1461-1825' \n",
    "#   15 --'>1825'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bdea8d7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctrData_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "id": "ecdeb3d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 1.35 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type1s data: with parsedPriceBracket, executionDeadlineDays\n",
    "Data_ctr = []\n",
    "Data_ctr = ctrData.merge(ctrData_1, how='left', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3fe68ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d855d1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set seaborn plotting aesthetics as default\n",
    "sns.set()\n",
    "\n",
    "#define plotting region (2 rows, 2 columns)\n",
    "fig, axs = plt.subplots(1, 2)\n",
    "\n",
    "#Plot\n",
    "plot2 = sns.histplot(data=Data_ctr, x=\"parsedPriceBracket\", ax=axs[0])\n",
    "plot2.axes.set_title(\"Type1s - Parsed Price\",fontsize=50)\n",
    "plot2.set_xlabel(\"Parsed Price Bracket\",fontsize=30)\n",
    "plot2.set_ylabel(\"Type1s Count\",fontsize=20)\n",
    "plot2.bar_label(plot2.containers[0], size=20)\n",
    "plot2.tick_params(labelsize=20)\n",
    "\n",
    "plot3 = sns.histplot(data=Data_ctr, x=\"executionDeadlineDays\", ax=axs[1])\n",
    "plot3.axes.set_title(\"Type1s - Execution DeadLine\",fontsize=50)\n",
    "plot3.bar_label(plot3.containers[0], size=20)\n",
    "plot3.set_xlabel(\"Execution Deadline Days\",fontsize=30)\n",
    "plot3.set_ylabel(\"Type1s Count\",fontsize=20)\n",
    "plot3.bar_label(plot3.containers[0], size=20)\n",
    "plot3.tick_params(labelsize=20)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "14dd77ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lemm                           0\n",
       "ctr_ads                        0\n",
       "id                             0\n",
       "contractingProcedureType       0\n",
       "contractType                   0\n",
       "objectBriefDescription         0\n",
       "contractingCount               0\n",
       "country_orig                9878\n",
       "parsedPriceBracket             0\n",
       "executionDeadlineDays          0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Data_ctr.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "59a7d4f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 2.77 s to compute.\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Getting Type1s data: category1sSplit\n",
    "ctrQuery_2 = \"select * from <>. ctr cross apply string_split(ctr.category1s,'|')) abc cross apply string_split(abc.category1s,'>')\" #Info removed to comply with the NDA\n",
    "ctrData_2 = pd.read_sql(ctrQuery_2, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec3fb3b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctrData_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "e27231b2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 2.11 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type1s data: with category1sSplit\n",
    "Data_ctr = Data_ctr.merge(ctrData_2, how='inner', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5b7b19f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "lemm                            0\n",
       "ctr_ads                         0\n",
       "id                              0\n",
       "contractingProcedureType        0\n",
       "contractType                    0\n",
       "objectBriefDescription          0\n",
       "contractingCount                0\n",
       "country_orig                10135\n",
       "parsedPriceBracket              0\n",
       "executionDeadlineDays           0\n",
       "cpvsSplit                       0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Data_ctr.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69712347",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4b05e5b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Plot\n",
    "sns.scatterplot(data=Data_ctr, x=\"Type1ingProcedureType\", y=\"parsedPriceBracket\", size=\"id\", sizes=(40, 400))\n",
    "plt.title('Type1s distribution', fontsize=35)\n",
    "plt.xticks(fontsize=20, rotation ='vertical')\n",
    "plt.yticks(fontsize=20)\n",
    "plt.xlabel('Type1ing Procedure Type', fontsize=25)\n",
    "plt.ylabel('Parsed price Bracket', fontsize=25)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5cd01f3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Plot\n",
    "df2 = pd.concat([Data_ctr['Type1ingProcedureType'],Data_ctr['category1sSplit']], axis=1)\n",
    "df2 = df2.drop_duplicates()\n",
    "\n",
    "sns.set(rc = {'figure.figsize':(30,20)})\n",
    "plot1 = sns.countplot(data=df2, x=\"Type1ingProcedureType\")\n",
    "plot1.bar_label(plot1.containers[0], size=20)\n",
    "plt.title('Distinct category1sSplit count per Type1ing Procedure Type', fontsize=35)\n",
    "plt.xticks(fontsize=20, rotation ='vertical')\n",
    "plt.yticks(fontsize=20)\n",
    "plt.xlabel('Type1ing Procedure Type', fontsize=25)\n",
    "plt.ylabel('Distinct category1sSplit Count', fontsize=25)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "529aedcb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 2.70 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type1s data: nif\n",
    "ctrQuery_nif = \"select * from  <> ctr union select * from <> nif\" #Info removed to comply with the NDA\n",
    "ctrData_nif = pd.read_sql(ctrQuery_nif, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "573ceb50",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ctrData_nif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "id": "4815f83c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 1.04 s to compute.\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type1s data: with nif\n",
    "Data_ctr = Data_ctr.merge(ctrData_nif, how='left', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ce5b45f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "beb7ae95",
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = Data_ctr[[\"Type1ingProcedureType\",\"nif\"]]\n",
    "df3 = df3.drop_duplicates()\n",
    "\n",
    "#Plot\n",
    "sns.set(rc = {'figure.figsize':(30,20)})\n",
    "plot1 = sns.countplot(data=df3, x=\"Type1ingProcedureType\")\n",
    "plot1.bar_label(plot1.containers[0], size=20)\n",
    "plt.title('Distinct Buyers per Type1ing Procedure Type', fontsize=35)\n",
    "plt.xticks(fontsize=20, rotation ='vertical')\n",
    "plt.yticks(fontsize=20)\n",
    "plt.xlabel('Type1ing Procedure Type', fontsize=25)\n",
    "plt.ylabel('Distinct Buyers', fontsize=25)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5da4533",
   "metadata": {},
   "outputs": [],
   "source": [
    "df4 = Data_ctr[[\"id\", \"parsedPriceBracket\",\"executionDeadlineDays\"]].reset_index(drop=True)\n",
    "df4 = df4.drop_duplicates().reset_index(drop=True)\n",
    "df4 = df4.pivot_table(index='executionDeadlineDays', columns='parsedPriceBracket', values='id', aggfunc='count')\n",
    "\n",
    "#Plot\n",
    "ax = sns.heatmap(df4,cmap=\"PiYG\", annot=True, fmt=\".0f\", annot_kws={\"fontsize\":20})\n",
    "cbar = ax.collections[0].colorbar\n",
    "cbar.ax.tick_params(labelsize=20)\n",
    "plt.title('Type1s Data', fontsize=35)\n",
    "plt.xticks(fontsize=20)\n",
    "plt.yticks(fontsize=20, rotation ='vertical')\n",
    "plt.xlabel('Execution Deadline Days Bracket', fontsize=25)\n",
    "plt.ylabel('Parsed Price Bracket', fontsize=25)\n",
    "plt.show()\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a9f6a77",
   "metadata": {},
   "source": [
    "# Fetching Data from the Db : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "1533c7cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 7.44 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type2s data:\n",
    "Type2sQuery = \"select * from <> Type2s\" #Info removed to comply with the NDA\n",
    "Type2sData = pd.read_sql(Type2sQuery, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35e69f09",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2sData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "97e39bf0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.69 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type2s data: parsedPriceBracket, executionDeadlineDays\n",
    "Type2sQuery_1 = \"select * from <> Type2s\" #Info removed to comply with the NDA\n",
    "Type2sData_1 = pd.read_sql(Type2sQuery_1, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3703eaa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2sData_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "id": "4c2ee9e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.22 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type2s data: with parsedPriceBracket, executionDeadlineDays\n",
    "Data_Type2s = []\n",
    "Data_Type2s = Type2sData.merge(Type2sData_1, how='left', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eac25d4f",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "id": "13a1f793",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.64 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type2s data: Category1Split\n",
    "Type2sQuery_2 = \"select * from <> Type2s cross apply string_split(Type2s.Category1,'|')) abc cross apply string_split(abc.Category1,'>')\" #Info removed to comply with the NDA\n",
    "Type2sData_2 = pd.read_sql(Type2sQuery_2, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0efaa19",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2sData_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "id": "b4dfd4f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.09 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type2s data: with category1sSplit\n",
    "Data_Type2s = Data_Type2s.merge(Type2sData_2, how='inner', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff82faa7",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "id": "d6e634be",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.75 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Type2s data: nif\n",
    "Type2sQuery_nif = \"select * from <> Type2s\" #Info removed to comply with the NDA\n",
    "Type2sData_nif = pd.read_sql(Type2sQuery_nif, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d940f44d",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "Type2sData_nif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de71ba33",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2sData_nif.drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "id": "5f490f2d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.11 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Type2s data: with nif\n",
    "Data_Type2s = Data_Type2s.merge(Type2sData_nif, how='left', on='id')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77ce620c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "de1f6e57",
   "metadata": {},
   "outputs": [],
   "source": [
    "#set seaborn plotting aesthetics as default\n",
    "sns.set()\n",
    "\n",
    "#define plotting region (2 rows, 2 columns)\n",
    "fig, axs = plt.subplots(1, 2)\n",
    "\n",
    "#Plot\n",
    "plot2 = sns.countplot(data=Data_Type2s, x=\"parsedPriceBracket\", ax=axs[0])\n",
    "plot2.axes.set_title(\"Type2s - Parsed Price\",fontsize=50)\n",
    "plot2.set_xlabel(\"Parced Price Bracket\",fontsize=30)\n",
    "plot2.set_ylabel(\"Type2s Count\",fontsize=20)\n",
    "plot2.bar_label(plot2.containers[0], size=20)\n",
    "plot2.tick_params(labelsize=20)\n",
    "\n",
    "df5 = Data_Type2s[[\"parsedPriceBracket\",\"nif\"]].reset_index(drop=True)\n",
    "df5 = df5.drop_duplicates().reset_index(drop=True)\n",
    "\n",
    "#Plot\n",
    "plot3 = sns.countplot(data=df5, x=\"parsedPriceBracket\", ax=axs[1])\n",
    "plot3.axes.set_title(\"Buyers - Parsed Price\",fontsize=50)\n",
    "plot3.bar_label(plot3.containers[0], size=20)\n",
    "plot3.set_xlabel(\"Parced Price Bracket\",fontsize=30)\n",
    "plot3.set_ylabel(\"Distinct Buyers Count\",fontsize=20)\n",
    "plot3.bar_label(plot3.containers[0], size=20)\n",
    "plot3.tick_params(labelsize=20)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "id": "aeb14b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Combine Type1s and Type2s into single df for Location info merging\n",
    "fullData =  pd.concat([Data_ctr, Data_Type2s], ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5867620f",
   "metadata": {},
   "outputs": [],
   "source": [
    "fullData"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3713d86d",
   "metadata": {},
   "outputs": [],
   "source": [
    "fullData.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3847d227",
   "metadata": {},
   "outputs": [],
   "source": [
    "fullData[ (fullData.ctr_Type2s =='Type2s') & (fullData.nif.isnull())]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "id": "c377656d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.76 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting data: loc\n",
    "Query_loc = \"select * \" #Info removed to comply with the NDA\n",
    "Data_loc = pd.read_sql(Query_loc, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93080aaf",
   "metadata": {},
   "outputs": [],
   "source": [
    "Data_loc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "f55bc92a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 1.02 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge loc data:\n",
    "Full_Data = fullData.merge(Data_loc, how='left', on='nif')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "4e96c0b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data['country_nif'] = Full_Data['country_nif'].str.capitalize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d96632e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c5b9d39e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data[((Full_Data.country_orig =='NA') | (Full_Data.country_orig ==None))]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c8896c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "d80c54ce",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "#Full_Data.drop(['country'], axis = 1)\n",
    "#Full_Data[\"country\"] = Full_Data['country_orig'].apply(lambda x: x if x!='NA' )\n",
    "Full_Data['country'] = np.where((Full_Data['country_orig'] == 'NA'), Full_Data['country_nif'], Full_Data['country_orig'])\n",
    "Full_Data['country'] = np.where((Full_Data['country'] == None), Full_Data['country_nif'], Full_Data['country'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "id": "b10937bb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.10 s to compute.\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "\n",
    "# Getting Geo data\n",
    "Query_geo = \"select *\" #Info removed to comply with the NDA\n",
    "Data_geo = pd.read_sql(Query_geo, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "id": "b3d53473",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.81 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge Geo data:\n",
    "Full_Data = Full_Data.merge(Data_geo, how='left', on='county')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "acb156cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data[\"country_geo\"].fillna(\"No Info\", inplace = True)\n",
    "Full_Data[\"country_orig\"].fillna(\"No Info\", inplace = True)\n",
    "Full_Data.country_orig[Full_Data['country_orig'].str.contains('Portugal')] = 'Portugal'\n",
    "Full_Data.country_orig[(Full_Data['country_orig'].str.contains('<BR/>'))& (Full_Data['country_geo']!= 'Portugal')] = 'Multiple'\n",
    "Full_Data.country_orig[(Full_Data['country_orig'].str.contains('<BR/>'))& (Full_Data['country_geo']== 'Portugal')] = 'Portugal'\n",
    "Full_Data.country_orig[Full_Data['country_orig'] == ''] = 'No Info'\n",
    "Full_Data.country_orig[Full_Data['country_orig'] == 'NA'] = 'No Info'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eba7cd7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "Full_Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5a76d187",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Plot\n",
    "sns.set(rc = {'figure.figsize':(30,20)})\n",
    "plot1 = sns.countplot(data=Full_Data, x=\"ctr_Type2s\")\n",
    "plot1.bar_label(plot1.containers[0], size=20)\n",
    "plt.title('Type1s and Type2s per Country of Origin', fontsize=35)\n",
    "plt.xticks(fontsize=20, rotation ='vertical')\n",
    "plt.yticks(fontsize=20)\n",
    "plt.xlabel('Type1s and Type2s', fontsize=25)\n",
    "plt.ylabel('Count', fontsize=25)\n",
    "#plt.legend(loc='upper right')\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "c44d906d",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = Full_Data.drop(columns = ['country', 'country_nif', 'country_geo', 'Type1ingCount'])\n",
    "data_df.rename(columns = {'country_orig':'country'}, inplace = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba1d7544",
   "metadata": {},
   "source": [
    "# Ordinal Encoding for Categorical Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "31221195",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df[\"ctr_Type2s\"] = data_df[\"ctr_Type2s\"].astype('category')\n",
    "data_df[\"Type1ingProcedureType\"] = data_df[\"Type1ingProcedureType\"].astype('category')\n",
    "data_df[\"Type1Type\"] = data_df[\"Type1Type\"].astype('category')\n",
    "data_df[\"parsedPriceBracket\"] = data_df[\"parsedPriceBracket\"].astype('category')\n",
    "data_df[\"executionDeadlineDays\"] = data_df[\"executionDeadlineDays\"].astype('category')\n",
    "#data_df[\"category1sSplit\"] = data_df[\"category1sSplit\"].astype('category')\n",
    "data_df[\"nif\"] = data_df[\"nif\"].astype('category')\n",
    "data_df[\"county\"] = data_df[\"county\"].astype('category')\n",
    "data_df[\"country\"] = data_df[\"country\"].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "edc6ff7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df[\"ctr_Type2s\"] = data_df[\"ctr_Type2s\"].cat.codes\n",
    "#data_df[\"Type1ingProcedureType\"] = data_df[\"Type1ingProcedureType\"].cat.codes\n",
    "#data_df[\"Type1Type\"] = data_df[\"Type1Type\"].cat.codes\n",
    "data_df[\"parsedPriceBracket\"] = data_df[\"parsedPriceBracket\"].cat.codes\n",
    "data_df[\"executionDeadlineDays\"] = data_df[\"executionDeadlineDays\"].cat.codes\n",
    "#data_df[\"category1sSplit\"] = data_df[\"category1sSplit\"].cat.codes\n",
    "data_df[\"nif\"] = data_df[\"nif\"].cat.codes\n",
    "data_df[\"county\"] = data_df[\"county\"].cat.codes\n",
    "data_df[\"country\"] = data_df[\"country\"].cat.codes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "85c9bb8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "58d26e63",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['lemm',\n",
       " 'contractingProcedureType',\n",
       " 'contractType',\n",
       " 'objectBriefDescription',\n",
       " 'cpvsSplit']"
      ]
     },
     "execution_count": 174,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.select_dtypes(include=['object','category']).columns.tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "id": "fd2a9da7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = data_df.drop_duplicates()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "463cad3c",
   "metadata": {},
   "source": [
    "# Trying category1s text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e3d88e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Getting Type1s data: category1sSplit and text\n",
    "ctr_category1s_query = \"select * from <> ctr cross apply string_split(ctr.category1s,'|')) abc cross apply string_split(abc.category1s,'>')) def where category1s_text is NOT NULL\" #Info removed to comply with the NDA\n",
    "ctr_category1s_data = pd.read_sql(ctr_category1s_query, edasql.dbConnection)\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "id": "aa908a3b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.72 s to compute.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "# Merge category1S Text data:\n",
    "data_df_new = data_df.merge(ctr_category1s_data, how='left', on='category1sSplit')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "398c824e",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df_new"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ba187cc",
   "metadata": {},
   "source": [
    "# Sampling for Faster Processing"
   ]
  },
  {
   "cell_type": "raw",
   "id": "d153c278",
   "metadata": {},
   "source": [
    "#Using Subset of the data due to dataset being huge while processing capacity on my laptop is limited.\n",
    "#This same code can be deployed using scalable power for the whole dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ce25531",
   "metadata": {},
   "outputs": [],
   "source": [
    "#using Type1 nos from the test set\n",
    "ctr_list = [] #Info removed to comply with the NDA\n",
    "\n",
    "#plus 10K Type1s\n",
    "samp_ctr = data_df_new[(data_df_new.ctr_Type2s == 1) & (data_df_new.category1s_text != None)].head(10000)\n",
    "samp_ctr = samp_ctr.append(data_df_new[data_df_new['id'].isin(ctr_list)]).reset_index(drop=True) \n",
    "samp_ctr "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efa64bde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#using Type2s nos from the test set\n",
    "Type2s_list = [] #Info removed to comply with the NDA\n",
    "\n",
    "#plus 10K Type2s\n",
    "samp_Type2s = data_df[(data_df.ctr_Type2s == 0)].head(10000)\n",
    "samp_Type2s = samp_Type2s.append(data_df[data_df['id'].isin(Type2s_list)]).reset_index(drop=True) \n",
    "samp_Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "1e25ed8a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the data\n",
    "samp_ctr.to_csv(\"./samp_ctr.csv\")\n",
    "samp_Type2s.to_csv(\"./samp_Type2s.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a0138c84",
   "metadata": {},
   "source": [
    "# Text Embedding Using all-MiniLM-L6-v2"
   ]
  },
  {
   "cell_type": "raw",
   "id": "3d7519f3",
   "metadata": {},
   "source": [
    "# Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 823,
   "id": "c8efd097",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10203 records to process today\n",
      "10203 records are added\n",
      "\n",
      "This function took 148.96 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Type1 Titles\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_ctr_1 = []\n",
    "\n",
    "n = len(samp_ctr['lemm'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_ctr_1 = model.encode(samp_ctr['lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4929bf39",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr_embeds_df_1 = pd.DataFrame(embeddings_ctr_1)\n",
    "ctr_embeds_df_1.columns = ['col_'+str(num) for num in range(embeddings_ctr_1.shape[1])]\n",
    "ctr_embeds_df_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 951,
   "id": "d0017a78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10224 records to process today\n",
      "10224 records are added\n",
      "\n",
      "This function took 169.62 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Ad Titles\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_Type2s_1 = []\n",
    "\n",
    "n = len(samp_Type2s['lemm'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_Type2s_1 = model.encode(samp_Type2s['lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5667640",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2s_embeds_df_1 = pd.DataFrame(embeddings_Type2s_1)\n",
    "Type2s_embeds_df_1.columns = ['col_'+str(num) for num in range(embeddings_Type2s_1.shape[1])]\n",
    "Type2s_embeds_df_1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4988d71f",
   "metadata": {},
   "source": [
    "# Build the ANNOY index Using all-MiniLM-L6-v2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 956,
   "id": "25d00de4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 12.17 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 384\n",
    "\n",
    "t_ctr_txt_1 = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in ctr_embeds_df_1.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_txt_1.add_item(index, row)\n",
    "\n",
    "t_ctr_txt_1.build(10) # 10 trees\n",
    "t_ctr_txt_1.save('test_ctr_txt_1.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 957,
   "id": "c5bf5630",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 13.50 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type2s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 384\n",
    "\n",
    "t_Type2s_txt_1 = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in Type2s_embeds_df_1.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_Type2s_txt_1.add_item(index, row)\n",
    "\n",
    "t_Type2s_txt_1.build(10) # 10 trees\n",
    "t_Type2s_txt_1.save('test_Type2s_txt_1.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "895dab12",
   "metadata": {},
   "source": [
    "# Testing the embeddings Using all-MiniLM-L6-v2"
   ]
  },
  {
   "cell_type": "raw",
   "id": "f2718b25",
   "metadata": {},
   "source": [
    "# Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7042d3b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <> #Info removed to comply with the NDA\n",
    "c_list = [<>] #Info removed to comply with the NDA\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_1.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 1000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cce27aaa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_1.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors #Info removed to comply with the NDA\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbf08492",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[samp_ctr.index.isin([<>])][['id','lemm', 'objectBriefDescription']]#Info removed to comply with the NDA\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "7ae557e8",
   "metadata": {},
   "source": [
    "# Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ac7fc76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "a_no = <> #Info removed to comply with the NDA\n",
    "a_list = [<>] #Info removed to comply with the NDA\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_1.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a786532",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "a_no = <> #Info removed to comply with the NDA\n",
    "a_list = [<>] #Info removed to comply with the NDA\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_1.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c576d1ca",
   "metadata": {},
   "source": [
    "# Removing Numerals and Special Characters from the lemmetised text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 311,
   "id": "a5ceb61f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('pc\\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('pc-\\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('pc \\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('pc - \\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('pcd\\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('id \\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('p\\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('\\d+', '', regex=True)\n",
    "#samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('_', ' ', regex=True)\n",
    "samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('\\w*\\d\\w*', ' ', regex=True)\n",
    "samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('_', ' ', regex=True)\n",
    "samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('\\W', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 313,
   "id": "1c02d94d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#trim(preg_replace(\"/(^|\\s+)(\\S(\\s+|$))+/\", \" \", $input));\n",
    "samp_ctr['lemm'] = samp_ctr['lemm'].str.replace('\\\\b\\\\w{1,2}\\\\b', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "id": "a3bf0607",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('pc\\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('pc-\\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('pc \\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('pc - \\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('pcd\\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('id \\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('p\\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('\\d+', '', regex=True)\n",
    "samp_Type2s['lemm'] = samp_Type2s['lemm'].str.replace('\\W', ' ', regex=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a533beb7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "samp_ctr[samp_ctr['objectBriefDescription'].str.contains('CP\\d+')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "id": "318abbd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemm</th>\n",
       "      <th>ctr_ads</th>\n",
       "      <th>id</th>\n",
       "      <th>contractingProcedureType</th>\n",
       "      <th>contractType</th>\n",
       "      <th>objectBriefDescription</th>\n",
       "      <th>country</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>executionDeadlineDays</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "      <th>county</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [lemm, ctr_ads, id, contractingProcedureType, contractType, objectBriefDescription, country, parsedPriceBracket, executionDeadlineDays, cpvsSplit, nif, county]\n",
       "Index: []"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "samp_Type2s[samp_Type2s['objectBriefDescription'].str.contains('dm\\d+')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6859d278",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "samp_Type2s"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ccf7444",
   "metadata": {},
   "source": [
    "# Text Embedding Using distiluse-base-multilingual-cased-v1"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b7b5eb58",
   "metadata": {},
   "source": [
    "# Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "id": "749ef745",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr['lemm'] = samp_ctr['lemm'].str.strip()\n",
    "samp_ctr = samp_ctr[samp_ctr['lemm'] != '']\n",
    "samp_ctr.reset_index(inplace = True,drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 387,
   "id": "ed0f56da",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr['category1s_text'] = samp_ctr['category1s_text'].str.strip()\n",
    "samp_ctr = samp_ctr[samp_ctr['category1s_text'] ]\n",
    "samp_ctr.reset_index(inplace = True,drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 395,
   "id": "6ee6d65d",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr = samp_ctr.fillna('No Info')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01f18324",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr[samp_ctr['category1s_text'] == 'No Info']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "717d4ae7",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr['category1s_text'][samp_ctr.category1s_text.str.contains('Equipamento')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 333,
   "id": "415db39f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10168 records to process today\n",
      "10168 records are added\n",
      "\n",
      "This function took 354.93 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Type1 Titles\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('distiluse-base-multilingual-cased-v1')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_ctr_2 = []\n",
    "\n",
    "n = len(samp_ctr['lemm'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_ctr_2 = model.encode(samp_ctr['lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "e510351c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the data\n",
    "embeddings_ctr_2.tofile(\"./embeddings_ctr_2.csv\", sep = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "846126f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr_embeds_df_2 = pd.DataFrame(embeddings_ctr_2)\n",
    "ctr_embeds_df_2.columns = ['col_'+str(num) for num in range(embeddings_ctr_2.shape[1])]\n",
    "ctr_embeds_df_2\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 398,
   "id": "c80a794e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/uniquenetra/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import nltk\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('wordnet')\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "lemmatizer = WordNetLemmatizer()\n",
    "def lemmatize_words(text):\n",
    "    text = re.sub('[^a-zA-Z]', ' ', text)\n",
    "    #print(text)\n",
    "    text = text.lower()\n",
    "    words = text.split()\n",
    "    words = [lemmatizer.lemmatize(word,pos='v') for word in words if not word in set(stopwords.words('portuguese'))]\n",
    "    return ' '.join(words)\n",
    "samp_ctr['category1s_text_lemm'] = samp_ctr['category1s_text'].apply(lemmatize_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8e7e880f",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 401,
   "id": "8100e2a1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10168 records to process today\n",
      "10168 records are added\n",
      "\n",
      "This function took 146.46 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Type1 category1s\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('distiluse-base-multilingual-cased-v1')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_ctr_category1s = []\n",
    "\n",
    "n = len(samp_ctr['category1s_text'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_ctr_category1s = model.encode(samp_ctr['category1s_text_lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d2169581",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr_embeds_df_category1s = pd.DataFrame(embeddings_ctr_category1s)\n",
    "ctr_embeds_df_category1s.columns = ['col_'+str(num) for num in range(embeddings_ctr_category1s.shape[1])]\n",
    "ctr_embeds_df_category1s\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "6b3e23e7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 10224 records to process today\n",
      "10224 records are added\n",
      "\n",
      "This function took 502.24 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Ad Titles\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('distiluse-base-multilingual-cased-v1')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_Type2s_2 = []\n",
    "\n",
    "n = len(samp_Type2s['lemm'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_Type2s_2 = model.encode(samp_Type2s['lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))\n",
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "7d426553",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Saving the data\n",
    "embeddings_Type2s_2.tofile(\"./embeddings_Type2s_2.csv\", sep = ',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf2a20da",
   "metadata": {},
   "outputs": [],
   "source": [
    "Type2s_embeds_df_2 = pd.DataFrame(embeddings_Type2s_2)\n",
    "Type2s_embeds_df_2.columns = ['col_'+str(num) for num in range(embeddings_Type2s_2.shape[1])]\n",
    "Type2s_embeds_df_2\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fba67335",
   "metadata": {},
   "source": [
    "# Build the ANNOY Index Using distiluse-base-multilingual-cased-v1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "id": "68745e21",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 16.37 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 512\n",
    "\n",
    "t_ctr_txt_2 = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in ctr_embeds_df_2.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_txt_2.add_item(index, row)\n",
    "\n",
    "t_ctr_txt_2.build(10) # 10 trees\n",
    "t_ctr_txt_2.save('test_ctr_txt_2.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b4edce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type2s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 512\n",
    "\n",
    "t_Type2s_txt_2 = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in Type2s_embeds_df_2.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_Type2s_txt_2.add_item(index, row)\n",
    "\n",
    "t_Type2s_txt_2.build(10) # 10 trees\n",
    "t_Type2s_txt_2.save('test_Type2s_txt_2.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6e581681",
   "metadata": {},
   "source": [
    "# Testing the embeddings distiluse-base-multilingual-cased-v1: Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "a3456593",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Contract No.: 9054597 index: [10168]\n",
      "\n",
      "Indexes of Contracts to be matched: \n",
      "10054    7666846\n",
      "10061    7724046\n",
      "10164    9253429\n",
      "Name: id, dtype: int64\n",
      "\n",
      "Match Results\n",
      "1 records match\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "2    10164\n",
       "Name: Col1, dtype: int64"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#TC1\n",
    "c_no = <> #Info removed to comply with the NDA \n",
    "c_list = [<>] #Info removed to comply with the NDA\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "78058357",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors #Info removed to comply with the NDA\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b2dd066",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[samp_ctr.index.isin([<>])] #Info removed to comply with the NDA\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5fd6c7f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "c_no = <> #Info removed to comply with the NDA\n",
    "c_list = [<>] #Info removed to comply with the NDA\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 5000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b7536da3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors #Info removed to comply with the NDA\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 850,
   "id": "5f04c490",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemm</th>\n",
       "      <th>ctr_ads</th>\n",
       "      <th>id</th>\n",
       "      <th>contractingProcedureType</th>\n",
       "      <th>contractType</th>\n",
       "      <th>objectBriefDescription</th>\n",
       "      <th>country</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>executionDeadlineDays</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "      <th>county</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10082</th>\n",
       "      <td>servico mapeamento necessidade competencia are...</td>\n",
       "      <td>1</td>\n",
       "      <td>6407988</td>\n",
       "      <td>Ajuste Direto Regime Geral</td>\n",
       "      <td>Aquisição de serviços</td>\n",
       "      <td>serviços de mapeamento de necessidades de comp...</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>6680</td>\n",
       "      <td>4375</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10128</th>\n",
       "      <td>subscricao catalogo atividade parar desenvolvi...</td>\n",
       "      <td>1</td>\n",
       "      <td>7373595</td>\n",
       "      <td>Ajuste Direto Regime Geral</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>Subscrição de catálogo de atividades para dese...</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>786</td>\n",
       "      <td>121</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10130</th>\n",
       "      <td>adesao rede parceiro parar desenvolvimento loc...</td>\n",
       "      <td>1</td>\n",
       "      <td>7495110</td>\n",
       "      <td>Ajuste Direto Regime Geral</td>\n",
       "      <td>Aquisição de serviços</td>\n",
       "      <td>Adesão à rede de parceiros para o desenvolvime...</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>6583</td>\n",
       "      <td>986</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10152</th>\n",
       "      <td>subscricao acesso revista cientifico (varios l...</td>\n",
       "      <td>1</td>\n",
       "      <td>6307468</td>\n",
       "      <td>Concurso público</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>Subscrição do acesso a revistas científicas (v...</td>\n",
       "      <td>49</td>\n",
       "      <td>1</td>\n",
       "      <td>9</td>\n",
       "      <td>775</td>\n",
       "      <td>1130</td>\n",
       "      <td>117</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    lemm  ctr_ads       id  \\\n",
       "10082  servico mapeamento necessidade competencia are...        1  6407988   \n",
       "10128  subscricao catalogo atividade parar desenvolvi...        1  7373595   \n",
       "10130  adesao rede parceiro parar desenvolvimento loc...        1  7495110   \n",
       "10152  subscricao acesso revista cientifico (varios l...        1  6307468   \n",
       "\n",
       "         contractingProcedureType              contractType  \\\n",
       "10082  Ajuste Direto Regime Geral     Aquisição de serviços   \n",
       "10128  Ajuste Direto Regime Geral  Aquisição de bens móveis   \n",
       "10130  Ajuste Direto Regime Geral     Aquisição de serviços   \n",
       "10152            Concurso público  Aquisição de bens móveis   \n",
       "\n",
       "                                  objectBriefDescription  country  \\\n",
       "10082  serviços de mapeamento de necessidades de comp...       49   \n",
       "10128  Subscrição de catálogo de atividades para dese...       49   \n",
       "10130  Adesão à rede de parceiros para o desenvolvime...       49   \n",
       "10152  Subscrição do acesso a revistas científicas (v...       49   \n",
       "\n",
       "       parsedPriceBracket  executionDeadlineDays  cpvsSplit   nif  county  \n",
       "10082                   1                      3       6680  4375      -1  \n",
       "10128                   1                     11        786   121      70  \n",
       "10130                   1                     11       6583   986     117  \n",
       "10152                   1                      9        775  1130     117  "
      ]
     },
     "execution_count": 850,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#TC2\n",
    "samp_ctr[samp_ctr.index.isin([10128, 10082, 10130, 10152])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f94d12a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "samp_ctr[samp_ctr.index.isin([<>])]#Info removed to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cfd6f61f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC3\n",
    "c_no = <>\n",
    "c_list = [<>] #Info removed to comply with the NDA\n",
    "\n",
    "\n",
    "indx = final_ctr.index[final_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "indx = final_ctr.index[final_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(final_ctr[final_ctr['id'].isin(c_list)]['id'])\n",
    "lst = final_ctr[final_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 1000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "657a864d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC3\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors #Info removed to comply with the NDA\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e279f87d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC3\n",
    "samp_ctr[samp_ctr.index.isin([<>])]#Info removed to comply with the NDA\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "17b58c4b",
   "metadata": {},
   "source": [
    "# Testing the embeddings distiluse-base-multilingual-cased-v1: Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8a9f8bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "a_no = <>\n",
    "a_list = [<>]\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 4000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f284f70c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cbe9bdac",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_Type2s[samp_Type2s.index.isin([<>])]\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd8ed28d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "a_no = <>\n",
    "a_list = []\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 5000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6773388e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_txt_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors \n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6157baf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "samp_Type2s[samp_Type2s.index.isin([<>])] \n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4471af0d",
   "metadata": {},
   "source": [
    "# Trial 2 features - text, category1ssplit : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 853,
   "id": "e511ee92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_503</th>\n",
       "      <th>col_504</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>cpvsSplit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.008769</td>\n",
       "      <td>-0.035696</td>\n",
       "      <td>-0.081959</td>\n",
       "      <td>-0.065422</td>\n",
       "      <td>0.106423</td>\n",
       "      <td>-0.006806</td>\n",
       "      <td>0.014415</td>\n",
       "      <td>-0.019768</td>\n",
       "      <td>-0.007022</td>\n",
       "      <td>0.022455</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.004052</td>\n",
       "      <td>0.065656</td>\n",
       "      <td>-0.080810</td>\n",
       "      <td>0.016209</td>\n",
       "      <td>-0.011536</td>\n",
       "      <td>0.013828</td>\n",
       "      <td>0.040486</td>\n",
       "      <td>0.051302</td>\n",
       "      <td>0.014021</td>\n",
       "      <td>4690</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.041093</td>\n",
       "      <td>-0.032101</td>\n",
       "      <td>-0.062681</td>\n",
       "      <td>-0.027410</td>\n",
       "      <td>0.083845</td>\n",
       "      <td>-0.030770</td>\n",
       "      <td>-0.019008</td>\n",
       "      <td>-0.087988</td>\n",
       "      <td>0.053893</td>\n",
       "      <td>0.023575</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.008800</td>\n",
       "      <td>0.037727</td>\n",
       "      <td>-0.035784</td>\n",
       "      <td>0.050928</td>\n",
       "      <td>-0.014732</td>\n",
       "      <td>-0.041200</td>\n",
       "      <td>0.017684</td>\n",
       "      <td>-0.089092</td>\n",
       "      <td>0.001281</td>\n",
       "      <td>4698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.026384</td>\n",
       "      <td>-0.036660</td>\n",
       "      <td>0.010439</td>\n",
       "      <td>0.003870</td>\n",
       "      <td>-0.000859</td>\n",
       "      <td>0.011253</td>\n",
       "      <td>-0.004238</td>\n",
       "      <td>-0.006598</td>\n",
       "      <td>-0.041758</td>\n",
       "      <td>-0.015051</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.013433</td>\n",
       "      <td>-0.012822</td>\n",
       "      <td>-0.015036</td>\n",
       "      <td>-0.095395</td>\n",
       "      <td>-0.002170</td>\n",
       "      <td>-0.000507</td>\n",
       "      <td>-0.061892</td>\n",
       "      <td>0.020786</td>\n",
       "      <td>-0.019073</td>\n",
       "      <td>4708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.013121</td>\n",
       "      <td>-0.041674</td>\n",
       "      <td>-0.012382</td>\n",
       "      <td>-0.035365</td>\n",
       "      <td>0.101415</td>\n",
       "      <td>0.027916</td>\n",
       "      <td>0.009208</td>\n",
       "      <td>-0.016659</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>0.010975</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.024724</td>\n",
       "      <td>-0.014496</td>\n",
       "      <td>-0.037792</td>\n",
       "      <td>0.022144</td>\n",
       "      <td>-0.045335</td>\n",
       "      <td>-0.020217</td>\n",
       "      <td>-0.001707</td>\n",
       "      <td>0.073547</td>\n",
       "      <td>0.006090</td>\n",
       "      <td>4724</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.006715</td>\n",
       "      <td>-0.055337</td>\n",
       "      <td>0.003673</td>\n",
       "      <td>-0.004573</td>\n",
       "      <td>0.057083</td>\n",
       "      <td>0.051284</td>\n",
       "      <td>0.037291</td>\n",
       "      <td>-0.038526</td>\n",
       "      <td>-0.037250</td>\n",
       "      <td>0.031229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.038382</td>\n",
       "      <td>-0.078159</td>\n",
       "      <td>0.047688</td>\n",
       "      <td>0.032358</td>\n",
       "      <td>0.024627</td>\n",
       "      <td>-0.033376</td>\n",
       "      <td>0.013414</td>\n",
       "      <td>-0.022359</td>\n",
       "      <td>0.049040</td>\n",
       "      <td>4703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10198</th>\n",
       "      <td>-0.019831</td>\n",
       "      <td>0.004905</td>\n",
       "      <td>0.027139</td>\n",
       "      <td>-0.016492</td>\n",
       "      <td>-0.058190</td>\n",
       "      <td>0.028122</td>\n",
       "      <td>-0.017875</td>\n",
       "      <td>0.019658</td>\n",
       "      <td>-0.020580</td>\n",
       "      <td>-0.023857</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.041780</td>\n",
       "      <td>-0.004194</td>\n",
       "      <td>0.009265</td>\n",
       "      <td>-0.030309</td>\n",
       "      <td>0.020870</td>\n",
       "      <td>0.053502</td>\n",
       "      <td>0.048767</td>\n",
       "      <td>-0.002036</td>\n",
       "      <td>-0.012913</td>\n",
       "      <td>2583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10199</th>\n",
       "      <td>-0.025081</td>\n",
       "      <td>-0.052650</td>\n",
       "      <td>-0.045189</td>\n",
       "      <td>-0.000725</td>\n",
       "      <td>-0.001611</td>\n",
       "      <td>0.042363</td>\n",
       "      <td>-0.022679</td>\n",
       "      <td>0.031628</td>\n",
       "      <td>0.016473</td>\n",
       "      <td>-0.020218</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.007561</td>\n",
       "      <td>0.009151</td>\n",
       "      <td>-0.000184</td>\n",
       "      <td>-0.002363</td>\n",
       "      <td>0.003779</td>\n",
       "      <td>-0.002796</td>\n",
       "      <td>0.112329</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>-0.030824</td>\n",
       "      <td>4320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10200</th>\n",
       "      <td>0.007497</td>\n",
       "      <td>0.047767</td>\n",
       "      <td>-0.001658</td>\n",
       "      <td>-0.026412</td>\n",
       "      <td>0.055747</td>\n",
       "      <td>-0.010143</td>\n",
       "      <td>0.070942</td>\n",
       "      <td>0.019195</td>\n",
       "      <td>-0.034207</td>\n",
       "      <td>0.040163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001890</td>\n",
       "      <td>0.012175</td>\n",
       "      <td>0.001437</td>\n",
       "      <td>0.017530</td>\n",
       "      <td>0.021713</td>\n",
       "      <td>0.050491</td>\n",
       "      <td>-0.057415</td>\n",
       "      <td>-0.008700</td>\n",
       "      <td>-0.009758</td>\n",
       "      <td>597</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10201</th>\n",
       "      <td>0.007973</td>\n",
       "      <td>-0.006567</td>\n",
       "      <td>0.007972</td>\n",
       "      <td>-0.024185</td>\n",
       "      <td>0.037945</td>\n",
       "      <td>-0.024951</td>\n",
       "      <td>0.021616</td>\n",
       "      <td>0.010492</td>\n",
       "      <td>-0.025213</td>\n",
       "      <td>0.039273</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.075808</td>\n",
       "      <td>-0.054949</td>\n",
       "      <td>0.004109</td>\n",
       "      <td>0.009871</td>\n",
       "      <td>0.007268</td>\n",
       "      <td>0.058270</td>\n",
       "      <td>0.018326</td>\n",
       "      <td>0.012770</td>\n",
       "      <td>0.049915</td>\n",
       "      <td>3051</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10202</th>\n",
       "      <td>0.006300</td>\n",
       "      <td>-0.034266</td>\n",
       "      <td>-0.021577</td>\n",
       "      <td>-0.033345</td>\n",
       "      <td>0.005970</td>\n",
       "      <td>0.015349</td>\n",
       "      <td>-0.009623</td>\n",
       "      <td>-0.016358</td>\n",
       "      <td>-0.022686</td>\n",
       "      <td>0.004165</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.022496</td>\n",
       "      <td>-0.006637</td>\n",
       "      <td>-0.033977</td>\n",
       "      <td>0.061615</td>\n",
       "      <td>0.003057</td>\n",
       "      <td>0.069206</td>\n",
       "      <td>-0.045256</td>\n",
       "      <td>0.009401</td>\n",
       "      <td>0.048284</td>\n",
       "      <td>127</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10203 rows × 513 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.008769 -0.035696 -0.081959 -0.065422  0.106423 -0.006806  0.014415   \n",
       "1      0.041093 -0.032101 -0.062681 -0.027410  0.083845 -0.030770 -0.019008   \n",
       "2      0.026384 -0.036660  0.010439  0.003870 -0.000859  0.011253 -0.004238   \n",
       "3      0.013121 -0.041674 -0.012382 -0.035365  0.101415  0.027916  0.009208   \n",
       "4      0.006715 -0.055337  0.003673 -0.004573  0.057083  0.051284  0.037291   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10198 -0.019831  0.004905  0.027139 -0.016492 -0.058190  0.028122 -0.017875   \n",
       "10199 -0.025081 -0.052650 -0.045189 -0.000725 -0.001611  0.042363 -0.022679   \n",
       "10200  0.007497  0.047767 -0.001658 -0.026412  0.055747 -0.010143  0.070942   \n",
       "10201  0.007973 -0.006567  0.007972 -0.024185  0.037945 -0.024951  0.021616   \n",
       "10202  0.006300 -0.034266 -0.021577 -0.033345  0.005970  0.015349 -0.009623   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_503   col_504   col_505  \\\n",
       "0     -0.019768 -0.007022  0.022455  ... -0.004052  0.065656 -0.080810   \n",
       "1     -0.087988  0.053893  0.023575  ... -0.008800  0.037727 -0.035784   \n",
       "2     -0.006598 -0.041758 -0.015051  ... -0.013433 -0.012822 -0.015036   \n",
       "3     -0.016659  0.002207  0.010975  ... -0.024724 -0.014496 -0.037792   \n",
       "4     -0.038526 -0.037250  0.031229  ...  0.038382 -0.078159  0.047688   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10198  0.019658 -0.020580 -0.023857  ... -0.041780 -0.004194  0.009265   \n",
       "10199  0.031628  0.016473 -0.020218  ... -0.007561  0.009151 -0.000184   \n",
       "10200  0.019195 -0.034207  0.040163  ...  0.001890  0.012175  0.001437   \n",
       "10201  0.010492 -0.025213  0.039273  ... -0.075808 -0.054949  0.004109   \n",
       "10202 -0.016358 -0.022686  0.004165  ... -0.022496 -0.006637 -0.033977   \n",
       "\n",
       "        col_506   col_507   col_508   col_509   col_510   col_511  cpvsSplit  \n",
       "0      0.016209 -0.011536  0.013828  0.040486  0.051302  0.014021       4690  \n",
       "1      0.050928 -0.014732 -0.041200  0.017684 -0.089092  0.001281       4698  \n",
       "2     -0.095395 -0.002170 -0.000507 -0.061892  0.020786 -0.019073       4708  \n",
       "3      0.022144 -0.045335 -0.020217 -0.001707  0.073547  0.006090       4724  \n",
       "4      0.032358  0.024627 -0.033376  0.013414 -0.022359  0.049040       4703  \n",
       "...         ...       ...       ...       ...       ...       ...        ...  \n",
       "10198 -0.030309  0.020870  0.053502  0.048767 -0.002036 -0.012913       2583  \n",
       "10199 -0.002363  0.003779 -0.002796  0.112329  0.023464 -0.030824       4320  \n",
       "10200  0.017530  0.021713  0.050491 -0.057415 -0.008700 -0.009758        597  \n",
       "10201  0.009871  0.007268  0.058270  0.018326  0.012770  0.049915       3051  \n",
       "10202  0.061615  0.003057  0.069206 -0.045256  0.009401  0.048284        127  \n",
       "\n",
       "[10203 rows x 513 columns]"
      ]
     },
     "execution_count": 853,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type1s : multilingual\n",
    "final_ctr = pd.concat([ctr_embeds_df_2, samp_ctr['category1sSplit']], axis=1)\n",
    "#final_ctr = final_ctr.drop(columns = ['lemm'])\n",
    "final_ctr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25399f4e",
   "metadata": {},
   "source": [
    "# Build the ANNOY Index for 2 features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 854,
   "id": "4689eabb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 16.75 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 513\n",
    "\n",
    "t_ctr_ftr_2 = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_ctr.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_ftr_2.add_item(index, row)\n",
    "\n",
    "t_ctr_ftr_2.build(10) # 10 trees\n",
    "t_ctr_ftr_2.save('test_ctr_ftr_2.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c458e9d8",
   "metadata": {},
   "source": [
    "# Testing 2 features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cf337faf",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 7000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82f578aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e447ef7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[samp_ctr.index.isin([<>])] #Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7dca24cc",
   "metadata": {},
   "source": [
    "# Trial weighted features - text, category1ssplit, nif, parsedPriceBracket : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "id": "a774c8ee",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_504</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>nif</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.016792</td>\n",
       "      <td>-0.018643</td>\n",
       "      <td>-0.039004</td>\n",
       "      <td>-0.029895</td>\n",
       "      <td>0.054194</td>\n",
       "      <td>-0.019316</td>\n",
       "      <td>0.007323</td>\n",
       "      <td>-0.017478</td>\n",
       "      <td>0.001804</td>\n",
       "      <td>0.012764</td>\n",
       "      <td>...</td>\n",
       "      <td>0.036609</td>\n",
       "      <td>0.009088</td>\n",
       "      <td>-0.010209</td>\n",
       "      <td>-0.025550</td>\n",
       "      <td>0.011412</td>\n",
       "      <td>-0.001523</td>\n",
       "      <td>0.000394</td>\n",
       "      <td>0.012107</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.018126</td>\n",
       "      <td>-0.017007</td>\n",
       "      <td>-0.033536</td>\n",
       "      <td>-0.010536</td>\n",
       "      <td>0.041234</td>\n",
       "      <td>-0.041406</td>\n",
       "      <td>-0.014353</td>\n",
       "      <td>-0.061255</td>\n",
       "      <td>0.034637</td>\n",
       "      <td>0.015067</td>\n",
       "      <td>...</td>\n",
       "      <td>0.028621</td>\n",
       "      <td>-0.013683</td>\n",
       "      <td>-0.002110</td>\n",
       "      <td>-0.008999</td>\n",
       "      <td>-0.001672</td>\n",
       "      <td>0.011201</td>\n",
       "      <td>0.001808</td>\n",
       "      <td>0.015103</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.015830</td>\n",
       "      <td>-0.021996</td>\n",
       "      <td>0.006263</td>\n",
       "      <td>0.002322</td>\n",
       "      <td>-0.000516</td>\n",
       "      <td>0.006752</td>\n",
       "      <td>-0.002543</td>\n",
       "      <td>-0.003959</td>\n",
       "      <td>-0.025055</td>\n",
       "      <td>-0.009030</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031862</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>-0.023262</td>\n",
       "      <td>-0.007924</td>\n",
       "      <td>0.011927</td>\n",
       "      <td>-0.011813</td>\n",
       "      <td>0.003423</td>\n",
       "      <td>0.005073</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.008573</td>\n",
       "      <td>-0.027852</td>\n",
       "      <td>0.002949</td>\n",
       "      <td>-0.021954</td>\n",
       "      <td>0.046245</td>\n",
       "      <td>-0.009400</td>\n",
       "      <td>0.007797</td>\n",
       "      <td>-0.017694</td>\n",
       "      <td>-0.005431</td>\n",
       "      <td>0.001391</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035916</td>\n",
       "      <td>-0.000432</td>\n",
       "      <td>-0.021911</td>\n",
       "      <td>0.008485</td>\n",
       "      <td>0.018192</td>\n",
       "      <td>0.007023</td>\n",
       "      <td>0.000096</td>\n",
       "      <td>0.025782</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.004029</td>\n",
       "      <td>-0.033202</td>\n",
       "      <td>0.002204</td>\n",
       "      <td>-0.002744</td>\n",
       "      <td>0.034250</td>\n",
       "      <td>0.030770</td>\n",
       "      <td>0.022375</td>\n",
       "      <td>-0.023115</td>\n",
       "      <td>-0.022350</td>\n",
       "      <td>0.018738</td>\n",
       "      <td>...</td>\n",
       "      <td>0.020884</td>\n",
       "      <td>0.009681</td>\n",
       "      <td>0.002245</td>\n",
       "      <td>-0.035181</td>\n",
       "      <td>-0.002482</td>\n",
       "      <td>-0.004843</td>\n",
       "      <td>-0.011499</td>\n",
       "      <td>0.005506</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10163</th>\n",
       "      <td>-0.008608</td>\n",
       "      <td>-0.000429</td>\n",
       "      <td>0.020157</td>\n",
       "      <td>-0.012266</td>\n",
       "      <td>0.046668</td>\n",
       "      <td>-0.017598</td>\n",
       "      <td>-0.003401</td>\n",
       "      <td>0.010460</td>\n",
       "      <td>-0.023512</td>\n",
       "      <td>-0.005675</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.014046</td>\n",
       "      <td>0.015082</td>\n",
       "      <td>-0.007095</td>\n",
       "      <td>0.008758</td>\n",
       "      <td>0.012790</td>\n",
       "      <td>-0.007523</td>\n",
       "      <td>-0.016513</td>\n",
       "      <td>-0.012945</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10164</th>\n",
       "      <td>-0.011846</td>\n",
       "      <td>-0.022739</td>\n",
       "      <td>-0.022890</td>\n",
       "      <td>-0.013924</td>\n",
       "      <td>0.049583</td>\n",
       "      <td>0.007865</td>\n",
       "      <td>-0.008198</td>\n",
       "      <td>0.012502</td>\n",
       "      <td>0.017764</td>\n",
       "      <td>-0.000175</td>\n",
       "      <td>...</td>\n",
       "      <td>0.012753</td>\n",
       "      <td>0.010746</td>\n",
       "      <td>-0.024242</td>\n",
       "      <td>-0.042271</td>\n",
       "      <td>-0.010351</td>\n",
       "      <td>-0.016525</td>\n",
       "      <td>-0.010162</td>\n",
       "      <td>-0.004336</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10165</th>\n",
       "      <td>0.004498</td>\n",
       "      <td>0.028660</td>\n",
       "      <td>-0.000995</td>\n",
       "      <td>-0.015847</td>\n",
       "      <td>0.033448</td>\n",
       "      <td>-0.006086</td>\n",
       "      <td>0.042565</td>\n",
       "      <td>0.011517</td>\n",
       "      <td>-0.020524</td>\n",
       "      <td>0.024098</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.021228</td>\n",
       "      <td>0.011940</td>\n",
       "      <td>-0.004379</td>\n",
       "      <td>0.011598</td>\n",
       "      <td>0.023775</td>\n",
       "      <td>-0.008921</td>\n",
       "      <td>-0.001624</td>\n",
       "      <td>-0.030797</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10166</th>\n",
       "      <td>0.004784</td>\n",
       "      <td>-0.003940</td>\n",
       "      <td>0.004783</td>\n",
       "      <td>-0.014511</td>\n",
       "      <td>0.022767</td>\n",
       "      <td>-0.014971</td>\n",
       "      <td>0.012970</td>\n",
       "      <td>0.006295</td>\n",
       "      <td>-0.015128</td>\n",
       "      <td>0.023564</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.023020</td>\n",
       "      <td>0.003051</td>\n",
       "      <td>-0.007237</td>\n",
       "      <td>-0.001841</td>\n",
       "      <td>0.022719</td>\n",
       "      <td>-0.002490</td>\n",
       "      <td>0.008838</td>\n",
       "      <td>-0.008453</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10167</th>\n",
       "      <td>0.003780</td>\n",
       "      <td>-0.020559</td>\n",
       "      <td>-0.012946</td>\n",
       "      <td>-0.020007</td>\n",
       "      <td>0.003582</td>\n",
       "      <td>0.009210</td>\n",
       "      <td>-0.005774</td>\n",
       "      <td>-0.009815</td>\n",
       "      <td>-0.013612</td>\n",
       "      <td>0.002499</td>\n",
       "      <td>...</td>\n",
       "      <td>0.016069</td>\n",
       "      <td>-0.004513</td>\n",
       "      <td>0.005644</td>\n",
       "      <td>-0.014978</td>\n",
       "      <td>0.018789</td>\n",
       "      <td>-0.013017</td>\n",
       "      <td>-0.003252</td>\n",
       "      <td>0.019314</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10168 rows × 1026 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.016792 -0.018643 -0.039004 -0.029895  0.054194 -0.019316  0.007323   \n",
       "1      0.018126 -0.017007 -0.033536 -0.010536  0.041234 -0.041406 -0.014353   \n",
       "2      0.015830 -0.021996  0.006263  0.002322 -0.000516  0.006752 -0.002543   \n",
       "3     -0.008573 -0.027852  0.002949 -0.021954  0.046245 -0.009400  0.007797   \n",
       "4      0.004029 -0.033202  0.002204 -0.002744  0.034250  0.030770  0.022375   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10163 -0.008608 -0.000429  0.020157 -0.012266  0.046668 -0.017598 -0.003401   \n",
       "10164 -0.011846 -0.022739 -0.022890 -0.013924  0.049583  0.007865 -0.008198   \n",
       "10165  0.004498  0.028660 -0.000995 -0.015847  0.033448 -0.006086  0.042565   \n",
       "10166  0.004784 -0.003940  0.004783 -0.014511  0.022767 -0.014971  0.012970   \n",
       "10167  0.003780 -0.020559 -0.012946 -0.020007  0.003582  0.009210 -0.005774   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_504   col_505   col_506  \\\n",
       "0     -0.017478  0.001804  0.012764  ...  0.036609  0.009088 -0.010209   \n",
       "1     -0.061255  0.034637  0.015067  ...  0.028621 -0.013683 -0.002110   \n",
       "2     -0.003959 -0.025055 -0.009030  ...  0.031862  0.002847 -0.023262   \n",
       "3     -0.017694 -0.005431  0.001391  ...  0.035916 -0.000432 -0.021911   \n",
       "4     -0.023115 -0.022350  0.018738  ...  0.020884  0.009681  0.002245   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10163  0.010460 -0.023512 -0.005675  ... -0.014046  0.015082 -0.007095   \n",
       "10164  0.012502  0.017764 -0.000175  ...  0.012753  0.010746 -0.024242   \n",
       "10165  0.011517 -0.020524  0.024098  ... -0.021228  0.011940 -0.004379   \n",
       "10166  0.006295 -0.015128  0.023564  ... -0.023020  0.003051 -0.007237   \n",
       "10167 -0.009815 -0.013612  0.002499  ...  0.016069 -0.004513  0.005644   \n",
       "\n",
       "        col_507   col_508   col_509   col_510   col_511  parsedPriceBracket  \\\n",
       "0     -0.025550  0.011412 -0.001523  0.000394  0.012107                0.05   \n",
       "1     -0.008999 -0.001672  0.011201  0.001808  0.015103                0.05   \n",
       "2     -0.007924  0.011927 -0.011813  0.003423  0.005073                0.05   \n",
       "3      0.008485  0.018192  0.007023  0.000096  0.025782                0.05   \n",
       "4     -0.035181 -0.002482 -0.004843 -0.011499  0.005506                0.05   \n",
       "...         ...       ...       ...       ...       ...                 ...   \n",
       "10163  0.008758  0.012790 -0.007523 -0.016513 -0.012945                0.05   \n",
       "10164 -0.042271 -0.010351 -0.016525 -0.010162 -0.004336                0.05   \n",
       "10165  0.011598  0.023775 -0.008921 -0.001624 -0.030797                0.05   \n",
       "10166 -0.001841  0.022719 -0.002490  0.008838 -0.008453                0.05   \n",
       "10167 -0.014978  0.018789 -0.013017 -0.003252  0.019314                0.10   \n",
       "\n",
       "       nif  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  \n",
       "...    ...  \n",
       "10163    0  \n",
       "10164    0  \n",
       "10165    0  \n",
       "10166    0  \n",
       "10167    0  \n",
       "\n",
       "[10168 rows x 1026 columns]"
      ]
     },
     "execution_count": 454,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type1s : multilingual\n",
    "final_ctr = pd.concat([ctr_embeds_df_2 * 0.6, ctr_embeds_df_category1s * 0.35, samp_ctr['parsedPriceBracket'] * 0.05,  samp_ctr['nif'] * 0], axis=1)\n",
    "#final_ctr = final_ctr.drop(columns = ['lemm'])\n",
    "final_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 529,
   "id": "33e17af8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1026"
      ]
     },
     "execution_count": 529,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_ctr.shape[1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f84a7413",
   "metadata": {},
   "source": [
    "# Build the ANNOY index using weighted features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 455,
   "id": "cef888ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 21.42 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 1026\n",
    "\n",
    "t_ctr_ftr_2_wt = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_ctr.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_ftr_2_wt.add_item(index, row)\n",
    "\n",
    "t_ctr_ftr_2_wt.build(10) # 10 trees\n",
    "t_ctr_ftr_2_wt.save('test_ctr_ftr_2_wt.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57472fb4",
   "metadata": {},
   "source": [
    "# Testing weighted features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8c57c362",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 1000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5339098",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9b90319",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit','category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10019, 10026, 10130, 10133, 10041, 8071, 617, 10161, 10130, 9999, 10079, 688, 682])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8421a210",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit', 'category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10054, 10161, 10164, 2029, 2409, 3335, 3159, 2399, 1501, 3356, 2387])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e484a67",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#TC2\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00e83289",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a1eb6a3",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "#TC2\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit', 'category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10011, 10018, 10102, 10147, 8928, 9782, 7516, 10018, 5895, 5880, 7543, 1781, 8392])]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e65b563",
   "metadata": {},
   "source": [
    "# Trial weighted features - text, category1ssplit, nif, parsedPriceBracket : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 981,
   "id": "0906a1ca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.024220</td>\n",
       "      <td>-0.045912</td>\n",
       "      <td>-0.020986</td>\n",
       "      <td>-0.008549</td>\n",
       "      <td>0.054871</td>\n",
       "      <td>0.013257</td>\n",
       "      <td>-0.008021</td>\n",
       "      <td>-0.067579</td>\n",
       "      <td>0.003622</td>\n",
       "      <td>0.005795</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004481</td>\n",
       "      <td>0.029657</td>\n",
       "      <td>-0.005545</td>\n",
       "      <td>0.016505</td>\n",
       "      <td>0.022840</td>\n",
       "      <td>0.039504</td>\n",
       "      <td>0.036913</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.076048</td>\n",
       "      <td>0.037826</td>\n",
       "      <td>0.017801</td>\n",
       "      <td>-0.018255</td>\n",
       "      <td>0.086328</td>\n",
       "      <td>-0.018215</td>\n",
       "      <td>0.060745</td>\n",
       "      <td>-0.028341</td>\n",
       "      <td>0.031218</td>\n",
       "      <td>0.030642</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004697</td>\n",
       "      <td>-0.047096</td>\n",
       "      <td>-0.050770</td>\n",
       "      <td>0.023003</td>\n",
       "      <td>0.012869</td>\n",
       "      <td>-0.017902</td>\n",
       "      <td>-0.072109</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.026081</td>\n",
       "      <td>0.005757</td>\n",
       "      <td>-0.016483</td>\n",
       "      <td>-0.060839</td>\n",
       "      <td>0.064346</td>\n",
       "      <td>-0.034761</td>\n",
       "      <td>-0.029246</td>\n",
       "      <td>-0.007448</td>\n",
       "      <td>0.040436</td>\n",
       "      <td>0.019701</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000148</td>\n",
       "      <td>0.017771</td>\n",
       "      <td>-0.038433</td>\n",
       "      <td>-0.020825</td>\n",
       "      <td>0.042849</td>\n",
       "      <td>-0.019691</td>\n",
       "      <td>-0.043829</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.054735</td>\n",
       "      <td>-0.022269</td>\n",
       "      <td>0.010686</td>\n",
       "      <td>-0.035819</td>\n",
       "      <td>0.021901</td>\n",
       "      <td>0.076630</td>\n",
       "      <td>-0.012732</td>\n",
       "      <td>-0.019730</td>\n",
       "      <td>-0.029121</td>\n",
       "      <td>-0.002356</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.046156</td>\n",
       "      <td>0.020172</td>\n",
       "      <td>-0.011376</td>\n",
       "      <td>0.044809</td>\n",
       "      <td>0.029195</td>\n",
       "      <td>0.031821</td>\n",
       "      <td>0.048732</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001414</td>\n",
       "      <td>0.046565</td>\n",
       "      <td>0.073109</td>\n",
       "      <td>0.023697</td>\n",
       "      <td>0.014041</td>\n",
       "      <td>-0.046530</td>\n",
       "      <td>0.004635</td>\n",
       "      <td>0.046016</td>\n",
       "      <td>0.004140</td>\n",
       "      <td>0.000998</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.100692</td>\n",
       "      <td>0.013127</td>\n",
       "      <td>-0.025101</td>\n",
       "      <td>-0.064973</td>\n",
       "      <td>-0.006318</td>\n",
       "      <td>-0.028992</td>\n",
       "      <td>-0.024700</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10219</th>\n",
       "      <td>-0.029705</td>\n",
       "      <td>-0.024705</td>\n",
       "      <td>-0.025263</td>\n",
       "      <td>0.015487</td>\n",
       "      <td>0.093688</td>\n",
       "      <td>0.022982</td>\n",
       "      <td>-0.011590</td>\n",
       "      <td>-0.013941</td>\n",
       "      <td>-0.040275</td>\n",
       "      <td>0.045163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.063992</td>\n",
       "      <td>0.040280</td>\n",
       "      <td>-0.019289</td>\n",
       "      <td>0.035681</td>\n",
       "      <td>-0.022211</td>\n",
       "      <td>-0.026217</td>\n",
       "      <td>-0.002171</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10220</th>\n",
       "      <td>-0.041041</td>\n",
       "      <td>0.013317</td>\n",
       "      <td>0.035926</td>\n",
       "      <td>-0.053735</td>\n",
       "      <td>0.024570</td>\n",
       "      <td>0.011895</td>\n",
       "      <td>0.005424</td>\n",
       "      <td>0.001817</td>\n",
       "      <td>-0.012700</td>\n",
       "      <td>-0.039497</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014541</td>\n",
       "      <td>0.036006</td>\n",
       "      <td>0.020576</td>\n",
       "      <td>0.008361</td>\n",
       "      <td>0.016710</td>\n",
       "      <td>-0.008317</td>\n",
       "      <td>0.080487</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10221</th>\n",
       "      <td>-0.005121</td>\n",
       "      <td>0.031187</td>\n",
       "      <td>-0.039318</td>\n",
       "      <td>-0.066958</td>\n",
       "      <td>0.071893</td>\n",
       "      <td>0.008878</td>\n",
       "      <td>0.008889</td>\n",
       "      <td>-0.011997</td>\n",
       "      <td>0.028955</td>\n",
       "      <td>0.000701</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.015090</td>\n",
       "      <td>-0.017232</td>\n",
       "      <td>-0.037847</td>\n",
       "      <td>0.002226</td>\n",
       "      <td>0.034090</td>\n",
       "      <td>0.015655</td>\n",
       "      <td>-0.050307</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10222</th>\n",
       "      <td>-0.008070</td>\n",
       "      <td>0.043932</td>\n",
       "      <td>-0.028641</td>\n",
       "      <td>-0.008111</td>\n",
       "      <td>0.107252</td>\n",
       "      <td>-0.011243</td>\n",
       "      <td>0.012140</td>\n",
       "      <td>-0.034764</td>\n",
       "      <td>-0.049123</td>\n",
       "      <td>0.012485</td>\n",
       "      <td>...</td>\n",
       "      <td>0.078929</td>\n",
       "      <td>0.006864</td>\n",
       "      <td>-0.030277</td>\n",
       "      <td>0.021937</td>\n",
       "      <td>0.052737</td>\n",
       "      <td>0.056380</td>\n",
       "      <td>-0.001355</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10223</th>\n",
       "      <td>-0.002255</td>\n",
       "      <td>0.085448</td>\n",
       "      <td>0.039749</td>\n",
       "      <td>-0.067235</td>\n",
       "      <td>0.068298</td>\n",
       "      <td>0.053565</td>\n",
       "      <td>0.006981</td>\n",
       "      <td>-0.006713</td>\n",
       "      <td>-0.087817</td>\n",
       "      <td>0.022560</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001602</td>\n",
       "      <td>-0.041276</td>\n",
       "      <td>-0.011615</td>\n",
       "      <td>0.027237</td>\n",
       "      <td>0.028706</td>\n",
       "      <td>0.043702</td>\n",
       "      <td>0.048434</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10224 rows × 515 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.024220 -0.045912 -0.020986 -0.008549  0.054871  0.013257 -0.008021   \n",
       "1      0.076048  0.037826  0.017801 -0.018255  0.086328 -0.018215  0.060745   \n",
       "2      0.026081  0.005757 -0.016483 -0.060839  0.064346 -0.034761 -0.029246   \n",
       "3     -0.054735 -0.022269  0.010686 -0.035819  0.021901  0.076630 -0.012732   \n",
       "4      0.001414  0.046565  0.073109  0.023697  0.014041 -0.046530  0.004635   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10219 -0.029705 -0.024705 -0.025263  0.015487  0.093688  0.022982 -0.011590   \n",
       "10220 -0.041041  0.013317  0.035926 -0.053735  0.024570  0.011895  0.005424   \n",
       "10221 -0.005121  0.031187 -0.039318 -0.066958  0.071893  0.008878  0.008889   \n",
       "10222 -0.008070  0.043932 -0.028641 -0.008111  0.107252 -0.011243  0.012140   \n",
       "10223 -0.002255  0.085448  0.039749 -0.067235  0.068298  0.053565  0.006981   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_505   col_506   col_507  \\\n",
       "0     -0.067579  0.003622  0.005795  ...  0.004481  0.029657 -0.005545   \n",
       "1     -0.028341  0.031218  0.030642  ...  0.004697 -0.047096 -0.050770   \n",
       "2     -0.007448  0.040436  0.019701  ... -0.000148  0.017771 -0.038433   \n",
       "3     -0.019730 -0.029121 -0.002356  ... -0.046156  0.020172 -0.011376   \n",
       "4      0.046016  0.004140  0.000998  ... -0.100692  0.013127 -0.025101   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10219 -0.013941 -0.040275  0.045163  ...  0.063992  0.040280 -0.019289   \n",
       "10220  0.001817 -0.012700 -0.039497  ...  0.014541  0.036006  0.020576   \n",
       "10221 -0.011997  0.028955  0.000701  ... -0.015090 -0.017232 -0.037847   \n",
       "10222 -0.034764 -0.049123  0.012485  ...  0.078929  0.006864 -0.030277   \n",
       "10223 -0.006713 -0.087817  0.022560  ...  0.001602 -0.041276 -0.011615   \n",
       "\n",
       "        col_508   col_509   col_510   col_511  parsedPriceBracket  cpvsSplit  \\\n",
       "0      0.016505  0.022840  0.039504  0.036913                 0.2          0   \n",
       "1      0.023003  0.012869 -0.017902 -0.072109                 0.1          0   \n",
       "2     -0.020825  0.042849 -0.019691 -0.043829                 0.1          0   \n",
       "3      0.044809  0.029195  0.031821  0.048732                 0.1          0   \n",
       "4     -0.064973 -0.006318 -0.028992 -0.024700                 0.1          0   \n",
       "...         ...       ...       ...       ...                 ...        ...   \n",
       "10219  0.035681 -0.022211 -0.026217 -0.002171                 0.1          0   \n",
       "10220  0.008361  0.016710 -0.008317  0.080487                 0.2          0   \n",
       "10221  0.002226  0.034090  0.015655 -0.050307                 0.1          0   \n",
       "10222  0.021937  0.052737  0.056380 -0.001355                 0.3          0   \n",
       "10223  0.027237  0.028706  0.043702  0.048434                 0.2          0   \n",
       "\n",
       "       nif  \n",
       "0        0  \n",
       "1        0  \n",
       "2        0  \n",
       "3        0  \n",
       "4        0  \n",
       "...    ...  \n",
       "10219    0  \n",
       "10220    0  \n",
       "10221    0  \n",
       "10222    0  \n",
       "10223    0  \n",
       "\n",
       "[10224 rows x 515 columns]"
      ]
     },
     "execution_count": 981,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type2s\n",
    "final_Type2s = pd.concat([Type2s_embeds_df_2 * 0.9, samp_Type2s['parsedPriceBracket'] * 0.1, samp_Type2s['category1sSplit'] * 0, samp_Type2s['nif'] * 0], axis=1)\n",
    "#final_Type2s = final_Type2s.drop(columns = ['lemm'])\n",
    "final_Type2s"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e706a40",
   "metadata": {},
   "source": [
    "# Build the ANNOY index using weighted features : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 982,
   "id": "325ce368",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 15.97 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type2s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 515\n",
    "\n",
    "t_Type2s_ftr_2_wt = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_Type2s.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_Type2s_ftr_2_wt.add_item(index, row)\n",
    "\n",
    "t_Type2s_ftr_2_wt.build(10) # 10 trees\n",
    "t_Type2s_ftr_2_wt.save('test_Type2s_ftr_2_wt.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ec41e96",
   "metadata": {},
   "source": [
    "# Testing weighted features : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "488878ef",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "a_no = <>\n",
    "a_list = [<>]\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 2600 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24abb57a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "a_no = <>\n",
    "a_list = [<>]\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 1950 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "334a6c9a",
   "metadata": {},
   "source": [
    "# Trial all features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "398afda3",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "b59d4293",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "      <th>contractType</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.025188</td>\n",
       "      <td>-0.027965</td>\n",
       "      <td>-0.058506</td>\n",
       "      <td>-0.044843</td>\n",
       "      <td>0.081290</td>\n",
       "      <td>-0.028974</td>\n",
       "      <td>0.010984</td>\n",
       "      <td>-0.026217</td>\n",
       "      <td>0.002706</td>\n",
       "      <td>0.019146</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.003772</td>\n",
       "      <td>0.014956</td>\n",
       "      <td>0.043034</td>\n",
       "      <td>0.021761</td>\n",
       "      <td>0.018051</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.027189</td>\n",
       "      <td>-0.025510</td>\n",
       "      <td>-0.050304</td>\n",
       "      <td>-0.015804</td>\n",
       "      <td>0.061851</td>\n",
       "      <td>-0.062109</td>\n",
       "      <td>-0.021529</td>\n",
       "      <td>-0.091883</td>\n",
       "      <td>0.051956</td>\n",
       "      <td>0.022601</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.002431</td>\n",
       "      <td>-0.038368</td>\n",
       "      <td>0.018954</td>\n",
       "      <td>-0.119253</td>\n",
       "      <td>-0.006458</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.023746</td>\n",
       "      <td>-0.032994</td>\n",
       "      <td>0.009395</td>\n",
       "      <td>0.003483</td>\n",
       "      <td>-0.000773</td>\n",
       "      <td>0.010128</td>\n",
       "      <td>-0.003814</td>\n",
       "      <td>-0.005938</td>\n",
       "      <td>-0.037582</td>\n",
       "      <td>-0.013546</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001953</td>\n",
       "      <td>-0.000456</td>\n",
       "      <td>-0.055703</td>\n",
       "      <td>0.018707</td>\n",
       "      <td>-0.017165</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.012859</td>\n",
       "      <td>-0.041778</td>\n",
       "      <td>0.004424</td>\n",
       "      <td>-0.032930</td>\n",
       "      <td>0.069367</td>\n",
       "      <td>-0.014100</td>\n",
       "      <td>0.011696</td>\n",
       "      <td>-0.026541</td>\n",
       "      <td>-0.008146</td>\n",
       "      <td>0.002086</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.026605</td>\n",
       "      <td>-0.016850</td>\n",
       "      <td>0.009539</td>\n",
       "      <td>0.029777</td>\n",
       "      <td>-0.006781</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.006043</td>\n",
       "      <td>-0.049803</td>\n",
       "      <td>0.003306</td>\n",
       "      <td>-0.004116</td>\n",
       "      <td>0.051375</td>\n",
       "      <td>0.046155</td>\n",
       "      <td>0.033562</td>\n",
       "      <td>-0.034673</td>\n",
       "      <td>-0.033525</td>\n",
       "      <td>0.028106</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022164</td>\n",
       "      <td>-0.030039</td>\n",
       "      <td>0.012072</td>\n",
       "      <td>-0.020123</td>\n",
       "      <td>0.044136</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10198</th>\n",
       "      <td>-0.005465</td>\n",
       "      <td>-0.006654</td>\n",
       "      <td>0.038080</td>\n",
       "      <td>-0.026083</td>\n",
       "      <td>0.031696</td>\n",
       "      <td>-0.006419</td>\n",
       "      <td>-0.034606</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>-0.019114</td>\n",
       "      <td>-0.006628</td>\n",
       "      <td>...</td>\n",
       "      <td>0.024234</td>\n",
       "      <td>0.047100</td>\n",
       "      <td>0.038359</td>\n",
       "      <td>-0.001135</td>\n",
       "      <td>-0.013706</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10199</th>\n",
       "      <td>-0.011662</td>\n",
       "      <td>-0.039156</td>\n",
       "      <td>-0.026785</td>\n",
       "      <td>-0.021801</td>\n",
       "      <td>0.027409</td>\n",
       "      <td>0.009786</td>\n",
       "      <td>-0.011875</td>\n",
       "      <td>0.021311</td>\n",
       "      <td>0.031874</td>\n",
       "      <td>-0.009141</td>\n",
       "      <td>...</td>\n",
       "      <td>0.016894</td>\n",
       "      <td>0.001043</td>\n",
       "      <td>0.089262</td>\n",
       "      <td>0.011554</td>\n",
       "      <td>-0.038507</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Concessão de obras públicas</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10200</th>\n",
       "      <td>0.006748</td>\n",
       "      <td>0.042990</td>\n",
       "      <td>-0.001492</td>\n",
       "      <td>-0.023771</td>\n",
       "      <td>0.050172</td>\n",
       "      <td>-0.009129</td>\n",
       "      <td>0.063847</td>\n",
       "      <td>0.017276</td>\n",
       "      <td>-0.030787</td>\n",
       "      <td>0.036147</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>0.045442</td>\n",
       "      <td>-0.051673</td>\n",
       "      <td>-0.007830</td>\n",
       "      <td>-0.008782</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10201</th>\n",
       "      <td>0.007176</td>\n",
       "      <td>-0.005910</td>\n",
       "      <td>0.007175</td>\n",
       "      <td>-0.021767</td>\n",
       "      <td>0.034151</td>\n",
       "      <td>-0.022456</td>\n",
       "      <td>0.019454</td>\n",
       "      <td>0.009442</td>\n",
       "      <td>-0.022692</td>\n",
       "      <td>0.035346</td>\n",
       "      <td>...</td>\n",
       "      <td>0.006541</td>\n",
       "      <td>0.052443</td>\n",
       "      <td>0.016494</td>\n",
       "      <td>0.011493</td>\n",
       "      <td>0.044924</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10202</th>\n",
       "      <td>0.005670</td>\n",
       "      <td>-0.030839</td>\n",
       "      <td>-0.019419</td>\n",
       "      <td>-0.030010</td>\n",
       "      <td>0.005373</td>\n",
       "      <td>0.013814</td>\n",
       "      <td>-0.008660</td>\n",
       "      <td>-0.014722</td>\n",
       "      <td>-0.020418</td>\n",
       "      <td>0.003749</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002751</td>\n",
       "      <td>0.062285</td>\n",
       "      <td>-0.040730</td>\n",
       "      <td>0.008461</td>\n",
       "      <td>0.043456</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10203 rows × 517 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.025188 -0.027965 -0.058506 -0.044843  0.081290 -0.028974  0.010984   \n",
       "1      0.027189 -0.025510 -0.050304 -0.015804  0.061851 -0.062109 -0.021529   \n",
       "2      0.023746 -0.032994  0.009395  0.003483 -0.000773  0.010128 -0.003814   \n",
       "3     -0.012859 -0.041778  0.004424 -0.032930  0.069367 -0.014100  0.011696   \n",
       "4      0.006043 -0.049803  0.003306 -0.004116  0.051375  0.046155  0.033562   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10198 -0.005465 -0.006654  0.038080 -0.026083  0.031696 -0.006419 -0.034606   \n",
       "10199 -0.011662 -0.039156 -0.026785 -0.021801  0.027409  0.009786 -0.011875   \n",
       "10200  0.006748  0.042990 -0.001492 -0.023771  0.050172 -0.009129  0.063847   \n",
       "10201  0.007176 -0.005910  0.007175 -0.021767  0.034151 -0.022456  0.019454   \n",
       "10202  0.005670 -0.030839 -0.019419 -0.030010  0.005373  0.013814 -0.008660   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_507   col_508   col_509  \\\n",
       "0     -0.026217  0.002706  0.019146  ... -0.003772  0.014956  0.043034   \n",
       "1     -0.091883  0.051956  0.022601  ... -0.002431 -0.038368  0.018954   \n",
       "2     -0.005938 -0.037582 -0.013546  ... -0.001953 -0.000456 -0.055703   \n",
       "3     -0.026541 -0.008146  0.002086  ... -0.026605 -0.016850  0.009539   \n",
       "4     -0.034673 -0.033525  0.028106  ...  0.022164 -0.030039  0.012072   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10198  0.019542 -0.019114 -0.006628  ...  0.024234  0.047100  0.038359   \n",
       "10199  0.021311  0.031874 -0.009141  ...  0.016894  0.001043  0.089262   \n",
       "10200  0.017276 -0.030787  0.036147  ...  0.019542  0.045442 -0.051673   \n",
       "10201  0.009442 -0.022692  0.035346  ...  0.006541  0.052443  0.016494   \n",
       "10202 -0.014722 -0.020418  0.003749  ...  0.002751  0.062285 -0.040730   \n",
       "\n",
       "        col_510   col_511  parsedPriceBracket  cpvsSplit  nif  \\\n",
       "0      0.021761  0.018051                 0.1          0    0   \n",
       "1     -0.119253 -0.006458                 0.1          0    0   \n",
       "2      0.018707 -0.017165                 0.1          0    0   \n",
       "3      0.029777 -0.006781                 0.1          0    0   \n",
       "4     -0.020123  0.044136                 0.1          0    0   \n",
       "...         ...       ...                 ...        ...  ...   \n",
       "10198 -0.001135 -0.013706                 0.1          0    0   \n",
       "10199  0.011554 -0.038507                 0.1          0    0   \n",
       "10200 -0.007830 -0.008782                 0.1          0    0   \n",
       "10201  0.011493  0.044924                 0.1          0    0   \n",
       "10202  0.008461  0.043456                 0.2          0    0   \n",
       "\n",
       "                        contractType  country  \n",
       "0      Empreitadas de obras públicas        0  \n",
       "1      Empreitadas de obras públicas        0  \n",
       "2      Empreitadas de obras públicas        0  \n",
       "3      Empreitadas de obras públicas        0  \n",
       "4      Empreitadas de obras públicas        0  \n",
       "...                              ...      ...  \n",
       "10198       Aquisição de bens móveis        0  \n",
       "10199    Concessão de obras públicas        0  \n",
       "10200       Aquisição de bens móveis        0  \n",
       "10201       Aquisição de bens móveis        0  \n",
       "10202       Aquisição de bens móveis        0  \n",
       "\n",
       "[10203 rows x 517 columns]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type1s : multilingual\n",
    "final_ctr = pd.concat([ctr_embeds_df_2*.9, samp_ctr['parsedPriceBracket']*.1, samp_ctr['category1sSplit']*0, samp_ctr['nif']*0,  samp_ctr['Type1Type'], samp_ctr['country']*0 ], axis=1)\n",
    "#samp_ctr['Type1ingProcedureType']*0,\n",
    "#final_ctr = final_ctr.drop(columns = ['lemm'])\n",
    "final_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "ccda20a9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>country</th>\n",
       "      <th>contractType_Aquisição de bens móveis&lt;br/&gt;Aquisição de serviços</th>\n",
       "      <th>contractType_Aquisição de bens móveis&lt;br/&gt;Empreitadas de obras públicas</th>\n",
       "      <th>contractType_Aquisição de serviços</th>\n",
       "      <th>contractType_Aquisição de serviços&lt;br/&gt;Aquisição de bens móveis</th>\n",
       "      <th>contractType_Aquisição de serviços&lt;br/&gt;Locação de bens móveis</th>\n",
       "      <th>contractType_Concessão de obras públicas</th>\n",
       "      <th>contractType_Empreitadas de obras públicas</th>\n",
       "      <th>contractType_Locação de bens móveis</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de serviços</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.025188</td>\n",
       "      <td>-0.027965</td>\n",
       "      <td>-0.058506</td>\n",
       "      <td>-0.044843</td>\n",
       "      <td>0.081290</td>\n",
       "      <td>-0.028974</td>\n",
       "      <td>0.010984</td>\n",
       "      <td>-0.026217</td>\n",
       "      <td>0.002706</td>\n",
       "      <td>0.019146</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.027189</td>\n",
       "      <td>-0.025510</td>\n",
       "      <td>-0.050304</td>\n",
       "      <td>-0.015804</td>\n",
       "      <td>0.061851</td>\n",
       "      <td>-0.062109</td>\n",
       "      <td>-0.021529</td>\n",
       "      <td>-0.091883</td>\n",
       "      <td>0.051956</td>\n",
       "      <td>0.022601</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.023746</td>\n",
       "      <td>-0.032994</td>\n",
       "      <td>0.009395</td>\n",
       "      <td>0.003483</td>\n",
       "      <td>-0.000773</td>\n",
       "      <td>0.010128</td>\n",
       "      <td>-0.003814</td>\n",
       "      <td>-0.005938</td>\n",
       "      <td>-0.037582</td>\n",
       "      <td>-0.013546</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.012859</td>\n",
       "      <td>-0.041778</td>\n",
       "      <td>0.004424</td>\n",
       "      <td>-0.032930</td>\n",
       "      <td>0.069367</td>\n",
       "      <td>-0.014100</td>\n",
       "      <td>0.011696</td>\n",
       "      <td>-0.026541</td>\n",
       "      <td>-0.008146</td>\n",
       "      <td>0.002086</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.006043</td>\n",
       "      <td>-0.049803</td>\n",
       "      <td>0.003306</td>\n",
       "      <td>-0.004116</td>\n",
       "      <td>0.051375</td>\n",
       "      <td>0.046155</td>\n",
       "      <td>0.033562</td>\n",
       "      <td>-0.034673</td>\n",
       "      <td>-0.033525</td>\n",
       "      <td>0.028106</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10198</th>\n",
       "      <td>-0.005465</td>\n",
       "      <td>-0.006654</td>\n",
       "      <td>0.038080</td>\n",
       "      <td>-0.026083</td>\n",
       "      <td>0.031696</td>\n",
       "      <td>-0.006419</td>\n",
       "      <td>-0.034606</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>-0.019114</td>\n",
       "      <td>-0.006628</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10199</th>\n",
       "      <td>-0.011662</td>\n",
       "      <td>-0.039156</td>\n",
       "      <td>-0.026785</td>\n",
       "      <td>-0.021801</td>\n",
       "      <td>0.027409</td>\n",
       "      <td>0.009786</td>\n",
       "      <td>-0.011875</td>\n",
       "      <td>0.021311</td>\n",
       "      <td>0.031874</td>\n",
       "      <td>-0.009141</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10200</th>\n",
       "      <td>0.006748</td>\n",
       "      <td>0.042990</td>\n",
       "      <td>-0.001492</td>\n",
       "      <td>-0.023771</td>\n",
       "      <td>0.050172</td>\n",
       "      <td>-0.009129</td>\n",
       "      <td>0.063847</td>\n",
       "      <td>0.017276</td>\n",
       "      <td>-0.030787</td>\n",
       "      <td>0.036147</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10201</th>\n",
       "      <td>0.007176</td>\n",
       "      <td>-0.005910</td>\n",
       "      <td>0.007175</td>\n",
       "      <td>-0.021767</td>\n",
       "      <td>0.034151</td>\n",
       "      <td>-0.022456</td>\n",
       "      <td>0.019454</td>\n",
       "      <td>0.009442</td>\n",
       "      <td>-0.022692</td>\n",
       "      <td>0.035346</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10202</th>\n",
       "      <td>0.005670</td>\n",
       "      <td>-0.030839</td>\n",
       "      <td>-0.019419</td>\n",
       "      <td>-0.030010</td>\n",
       "      <td>0.005373</td>\n",
       "      <td>0.013814</td>\n",
       "      <td>-0.008660</td>\n",
       "      <td>-0.014722</td>\n",
       "      <td>-0.020418</td>\n",
       "      <td>0.003749</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10203 rows × 525 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.025188 -0.027965 -0.058506 -0.044843  0.081290 -0.028974  0.010984   \n",
       "1      0.027189 -0.025510 -0.050304 -0.015804  0.061851 -0.062109 -0.021529   \n",
       "2      0.023746 -0.032994  0.009395  0.003483 -0.000773  0.010128 -0.003814   \n",
       "3     -0.012859 -0.041778  0.004424 -0.032930  0.069367 -0.014100  0.011696   \n",
       "4      0.006043 -0.049803  0.003306 -0.004116  0.051375  0.046155  0.033562   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10198 -0.005465 -0.006654  0.038080 -0.026083  0.031696 -0.006419 -0.034606   \n",
       "10199 -0.011662 -0.039156 -0.026785 -0.021801  0.027409  0.009786 -0.011875   \n",
       "10200  0.006748  0.042990 -0.001492 -0.023771  0.050172 -0.009129  0.063847   \n",
       "10201  0.007176 -0.005910  0.007175 -0.021767  0.034151 -0.022456  0.019454   \n",
       "10202  0.005670 -0.030839 -0.019419 -0.030010  0.005373  0.013814 -0.008660   \n",
       "\n",
       "          col_7     col_8     col_9  ...  country  \\\n",
       "0     -0.026217  0.002706  0.019146  ...        0   \n",
       "1     -0.091883  0.051956  0.022601  ...        0   \n",
       "2     -0.005938 -0.037582 -0.013546  ...        0   \n",
       "3     -0.026541 -0.008146  0.002086  ...        0   \n",
       "4     -0.034673 -0.033525  0.028106  ...        0   \n",
       "...         ...       ...       ...  ...      ...   \n",
       "10198  0.019542 -0.019114 -0.006628  ...        0   \n",
       "10199  0.021311  0.031874 -0.009141  ...        0   \n",
       "10200  0.017276 -0.030787  0.036147  ...        0   \n",
       "10201  0.009442 -0.022692  0.035346  ...        0   \n",
       "10202 -0.014722 -0.020418  0.003749  ...        0   \n",
       "\n",
       "       contractType_Aquisição de bens móveis<br/>Aquisição de serviços  \\\n",
       "0                                                      0                 \n",
       "1                                                      0                 \n",
       "2                                                      0                 \n",
       "3                                                      0                 \n",
       "4                                                      0                 \n",
       "...                                                  ...                 \n",
       "10198                                                  0                 \n",
       "10199                                                  0                 \n",
       "10200                                                  0                 \n",
       "10201                                                  0                 \n",
       "10202                                                  0                 \n",
       "\n",
       "       contractType_Aquisição de bens móveis<br/>Empreitadas de obras públicas  \\\n",
       "0                                                      0                         \n",
       "1                                                      0                         \n",
       "2                                                      0                         \n",
       "3                                                      0                         \n",
       "4                                                      0                         \n",
       "...                                                  ...                         \n",
       "10198                                                  0                         \n",
       "10199                                                  0                         \n",
       "10200                                                  0                         \n",
       "10201                                                  0                         \n",
       "10202                                                  0                         \n",
       "\n",
       "       contractType_Aquisição de serviços  \\\n",
       "0                                       0   \n",
       "1                                       0   \n",
       "2                                       0   \n",
       "3                                       0   \n",
       "4                                       0   \n",
       "...                                   ...   \n",
       "10198                                   0   \n",
       "10199                                   0   \n",
       "10200                                   0   \n",
       "10201                                   0   \n",
       "10202                                   0   \n",
       "\n",
       "       contractType_Aquisição de serviços<br/>Aquisição de bens móveis  \\\n",
       "0                                                      0                 \n",
       "1                                                      0                 \n",
       "2                                                      0                 \n",
       "3                                                      0                 \n",
       "4                                                      0                 \n",
       "...                                                  ...                 \n",
       "10198                                                  0                 \n",
       "10199                                                  0                 \n",
       "10200                                                  0                 \n",
       "10201                                                  0                 \n",
       "10202                                                  0                 \n",
       "\n",
       "       contractType_Aquisição de serviços<br/>Locação de bens móveis  \\\n",
       "0                                                      0               \n",
       "1                                                      0               \n",
       "2                                                      0               \n",
       "3                                                      0               \n",
       "4                                                      0               \n",
       "...                                                  ...               \n",
       "10198                                                  0               \n",
       "10199                                                  0               \n",
       "10200                                                  0               \n",
       "10201                                                  0               \n",
       "10202                                                  0               \n",
       "\n",
       "       contractType_Concessão de obras públicas  \\\n",
       "0                                             0   \n",
       "1                                             0   \n",
       "2                                             0   \n",
       "3                                             0   \n",
       "4                                             0   \n",
       "...                                         ...   \n",
       "10198                                         0   \n",
       "10199                                         1   \n",
       "10200                                         0   \n",
       "10201                                         0   \n",
       "10202                                         0   \n",
       "\n",
       "       contractType_Empreitadas de obras públicas  \\\n",
       "0                                               1   \n",
       "1                                               1   \n",
       "2                                               1   \n",
       "3                                               1   \n",
       "4                                               1   \n",
       "...                                           ...   \n",
       "10198                                           0   \n",
       "10199                                           0   \n",
       "10200                                           0   \n",
       "10201                                           0   \n",
       "10202                                           0   \n",
       "\n",
       "       contractType_Locação de bens móveis  \\\n",
       "0                                        0   \n",
       "1                                        0   \n",
       "2                                        0   \n",
       "3                                        0   \n",
       "4                                        0   \n",
       "...                                    ...   \n",
       "10198                                    0   \n",
       "10199                                    0   \n",
       "10200                                    0   \n",
       "10201                                    0   \n",
       "10202                                    0   \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de serviços  \n",
       "0                                                     0  \n",
       "1                                                     0  \n",
       "2                                                     0  \n",
       "3                                                     0  \n",
       "4                                                     0  \n",
       "...                                                 ...  \n",
       "10198                                                 0  \n",
       "10199                                                 0  \n",
       "10200                                                 0  \n",
       "10201                                                 0  \n",
       "10202                                                 0  \n",
       "\n",
       "[10203 rows x 525 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Converting categorical data into dummy or indicator variables for Type1s\n",
    "final_ctr = pd.get_dummies(final_ctr, prefix={#'Type1ingProcedureType':'Type1ingProcedureType', \n",
    "                                              'Type1Type':'Type1Type'}, \n",
    "                                              columns = [#'Type1ingProcedureType', \n",
    "                                                         'Type1Type'],\n",
    "                          drop_first=True)\n",
    "final_ctr"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ee8da14",
   "metadata": {},
   "source": [
    "# Build the ANNOY index using all features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "05b205ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 10.55 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 525\n",
    "\n",
    "t_ctr_all_ftrs = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_ctr.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_all_ftrs.add_item(index, row)\n",
    "\n",
    "t_ctr_all_ftrs.build(10) # 10 trees\n",
    "t_ctr_all_ftrs.save('test_ctr_all_ftrs.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83ad15f5",
   "metadata": {},
   "source": [
    "# Testing with all the features : Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5637f5d6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_all_ftrs.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 1000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "42955160",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_all_ftrs.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee560061",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit', 'nif', 'parsedPriceBracket', 'Type1Type']][samp_ctr.index.isin([10168, 620, 10164, 693, 687, 3241, 680, 8098, 9009, 2029])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fb8e4496",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_all_ftrs.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "115eaee9",
   "metadata": {},
   "source": [
    "# Trial all features : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbbace4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "samp_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1016,
   "id": "ee2ac76e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "      <th>contractType</th>\n",
       "      <th>country</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.026911</td>\n",
       "      <td>-0.051014</td>\n",
       "      <td>-0.023318</td>\n",
       "      <td>-0.009499</td>\n",
       "      <td>0.060968</td>\n",
       "      <td>0.014730</td>\n",
       "      <td>-0.008912</td>\n",
       "      <td>-0.075087</td>\n",
       "      <td>0.004025</td>\n",
       "      <td>0.006439</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.006161</td>\n",
       "      <td>0.018339</td>\n",
       "      <td>0.025377</td>\n",
       "      <td>0.043894</td>\n",
       "      <td>0.041015</td>\n",
       "      <td>2</td>\n",
       "      <td>5625</td>\n",
       "      <td>50</td>\n",
       "      <td>NA</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.084498</td>\n",
       "      <td>0.042029</td>\n",
       "      <td>0.019779</td>\n",
       "      <td>-0.020284</td>\n",
       "      <td>0.095920</td>\n",
       "      <td>-0.020238</td>\n",
       "      <td>0.067495</td>\n",
       "      <td>-0.031489</td>\n",
       "      <td>0.034687</td>\n",
       "      <td>0.034047</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.056411</td>\n",
       "      <td>0.025558</td>\n",
       "      <td>0.014299</td>\n",
       "      <td>-0.019891</td>\n",
       "      <td>-0.080121</td>\n",
       "      <td>1</td>\n",
       "      <td>6574</td>\n",
       "      <td>2886</td>\n",
       "      <td>Aquisição de serviços</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.028979</td>\n",
       "      <td>0.006396</td>\n",
       "      <td>-0.018315</td>\n",
       "      <td>-0.067599</td>\n",
       "      <td>0.071495</td>\n",
       "      <td>-0.038623</td>\n",
       "      <td>-0.032496</td>\n",
       "      <td>-0.008276</td>\n",
       "      <td>0.044929</td>\n",
       "      <td>0.021890</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.042704</td>\n",
       "      <td>-0.023139</td>\n",
       "      <td>0.047610</td>\n",
       "      <td>-0.021879</td>\n",
       "      <td>-0.048699</td>\n",
       "      <td>1</td>\n",
       "      <td>4379</td>\n",
       "      <td>1047</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.060817</td>\n",
       "      <td>-0.024744</td>\n",
       "      <td>0.011874</td>\n",
       "      <td>-0.039798</td>\n",
       "      <td>0.024335</td>\n",
       "      <td>0.085144</td>\n",
       "      <td>-0.014147</td>\n",
       "      <td>-0.021923</td>\n",
       "      <td>-0.032357</td>\n",
       "      <td>-0.002618</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.012641</td>\n",
       "      <td>0.049787</td>\n",
       "      <td>0.032439</td>\n",
       "      <td>0.035357</td>\n",
       "      <td>0.054146</td>\n",
       "      <td>1</td>\n",
       "      <td>4706</td>\n",
       "      <td>3277</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001571</td>\n",
       "      <td>0.051739</td>\n",
       "      <td>0.081232</td>\n",
       "      <td>0.026330</td>\n",
       "      <td>0.015601</td>\n",
       "      <td>-0.051699</td>\n",
       "      <td>0.005150</td>\n",
       "      <td>0.051129</td>\n",
       "      <td>0.004600</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.027890</td>\n",
       "      <td>-0.072192</td>\n",
       "      <td>-0.007020</td>\n",
       "      <td>-0.032213</td>\n",
       "      <td>-0.027444</td>\n",
       "      <td>1</td>\n",
       "      <td>6862</td>\n",
       "      <td>3045</td>\n",
       "      <td>Locação de bens móveis</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10219</th>\n",
       "      <td>-0.033006</td>\n",
       "      <td>-0.027450</td>\n",
       "      <td>-0.028070</td>\n",
       "      <td>0.017208</td>\n",
       "      <td>0.104097</td>\n",
       "      <td>0.025535</td>\n",
       "      <td>-0.012878</td>\n",
       "      <td>-0.015490</td>\n",
       "      <td>-0.044750</td>\n",
       "      <td>0.050181</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.021432</td>\n",
       "      <td>0.039645</td>\n",
       "      <td>-0.024679</td>\n",
       "      <td>-0.029130</td>\n",
       "      <td>-0.002412</td>\n",
       "      <td>1</td>\n",
       "      <td>5824</td>\n",
       "      <td>4389</td>\n",
       "      <td>Aquisição de serviços</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10220</th>\n",
       "      <td>-0.045601</td>\n",
       "      <td>0.014797</td>\n",
       "      <td>0.039917</td>\n",
       "      <td>-0.059706</td>\n",
       "      <td>0.027300</td>\n",
       "      <td>0.013217</td>\n",
       "      <td>0.006027</td>\n",
       "      <td>0.002019</td>\n",
       "      <td>-0.014111</td>\n",
       "      <td>-0.043886</td>\n",
       "      <td>...</td>\n",
       "      <td>0.022862</td>\n",
       "      <td>0.009290</td>\n",
       "      <td>0.018566</td>\n",
       "      <td>-0.009241</td>\n",
       "      <td>0.089430</td>\n",
       "      <td>2</td>\n",
       "      <td>6767</td>\n",
       "      <td>4510</td>\n",
       "      <td>Aquisição de serviços</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10221</th>\n",
       "      <td>-0.005691</td>\n",
       "      <td>0.034652</td>\n",
       "      <td>-0.043687</td>\n",
       "      <td>-0.074397</td>\n",
       "      <td>0.079882</td>\n",
       "      <td>0.009865</td>\n",
       "      <td>0.009877</td>\n",
       "      <td>-0.013330</td>\n",
       "      <td>0.032172</td>\n",
       "      <td>0.000779</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.042052</td>\n",
       "      <td>0.002473</td>\n",
       "      <td>0.037878</td>\n",
       "      <td>0.017394</td>\n",
       "      <td>-0.055897</td>\n",
       "      <td>1</td>\n",
       "      <td>4703</td>\n",
       "      <td>12</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10222</th>\n",
       "      <td>-0.008967</td>\n",
       "      <td>0.048813</td>\n",
       "      <td>-0.031824</td>\n",
       "      <td>-0.009013</td>\n",
       "      <td>0.119169</td>\n",
       "      <td>-0.012493</td>\n",
       "      <td>0.013489</td>\n",
       "      <td>-0.038627</td>\n",
       "      <td>-0.054581</td>\n",
       "      <td>0.013872</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.033642</td>\n",
       "      <td>0.024375</td>\n",
       "      <td>0.058597</td>\n",
       "      <td>0.062644</td>\n",
       "      <td>-0.001506</td>\n",
       "      <td>3</td>\n",
       "      <td>4670</td>\n",
       "      <td>7083</td>\n",
       "      <td>Empreitadas de obras públicas</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10223</th>\n",
       "      <td>-0.002505</td>\n",
       "      <td>0.094942</td>\n",
       "      <td>0.044165</td>\n",
       "      <td>-0.074706</td>\n",
       "      <td>0.075887</td>\n",
       "      <td>0.059517</td>\n",
       "      <td>0.007757</td>\n",
       "      <td>-0.007459</td>\n",
       "      <td>-0.097575</td>\n",
       "      <td>0.025067</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.012905</td>\n",
       "      <td>0.030264</td>\n",
       "      <td>0.031896</td>\n",
       "      <td>0.048558</td>\n",
       "      <td>0.053816</td>\n",
       "      <td>2</td>\n",
       "      <td>1777</td>\n",
       "      <td>3098</td>\n",
       "      <td>Aquisição de bens móveis</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10224 rows × 517 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.026911 -0.051014 -0.023318 -0.009499  0.060968  0.014730 -0.008912   \n",
       "1      0.084498  0.042029  0.019779 -0.020284  0.095920 -0.020238  0.067495   \n",
       "2      0.028979  0.006396 -0.018315 -0.067599  0.071495 -0.038623 -0.032496   \n",
       "3     -0.060817 -0.024744  0.011874 -0.039798  0.024335  0.085144 -0.014147   \n",
       "4      0.001571  0.051739  0.081232  0.026330  0.015601 -0.051699  0.005150   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10219 -0.033006 -0.027450 -0.028070  0.017208  0.104097  0.025535 -0.012878   \n",
       "10220 -0.045601  0.014797  0.039917 -0.059706  0.027300  0.013217  0.006027   \n",
       "10221 -0.005691  0.034652 -0.043687 -0.074397  0.079882  0.009865  0.009877   \n",
       "10222 -0.008967  0.048813 -0.031824 -0.009013  0.119169 -0.012493  0.013489   \n",
       "10223 -0.002505  0.094942  0.044165 -0.074706  0.075887  0.059517  0.007757   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_507   col_508   col_509  \\\n",
       "0     -0.075087  0.004025  0.006439  ... -0.006161  0.018339  0.025377   \n",
       "1     -0.031489  0.034687  0.034047  ... -0.056411  0.025558  0.014299   \n",
       "2     -0.008276  0.044929  0.021890  ... -0.042704 -0.023139  0.047610   \n",
       "3     -0.021923 -0.032357 -0.002618  ... -0.012641  0.049787  0.032439   \n",
       "4      0.051129  0.004600  0.001109  ... -0.027890 -0.072192 -0.007020   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10219 -0.015490 -0.044750  0.050181  ... -0.021432  0.039645 -0.024679   \n",
       "10220  0.002019 -0.014111 -0.043886  ...  0.022862  0.009290  0.018566   \n",
       "10221 -0.013330  0.032172  0.000779  ... -0.042052  0.002473  0.037878   \n",
       "10222 -0.038627 -0.054581  0.013872  ... -0.033642  0.024375  0.058597   \n",
       "10223 -0.007459 -0.097575  0.025067  ... -0.012905  0.030264  0.031896   \n",
       "\n",
       "        col_510   col_511  parsedPriceBracket  cpvsSplit   nif  \\\n",
       "0      0.043894  0.041015                   2       5625    50   \n",
       "1     -0.019891 -0.080121                   1       6574  2886   \n",
       "2     -0.021879 -0.048699                   1       4379  1047   \n",
       "3      0.035357  0.054146                   1       4706  3277   \n",
       "4     -0.032213 -0.027444                   1       6862  3045   \n",
       "...         ...       ...                 ...        ...   ...   \n",
       "10219 -0.029130 -0.002412                   1       5824  4389   \n",
       "10220 -0.009241  0.089430                   2       6767  4510   \n",
       "10221  0.017394 -0.055897                   1       4703    12   \n",
       "10222  0.062644 -0.001506                   3       4670  7083   \n",
       "10223  0.048558  0.053816                   2       1777  3098   \n",
       "\n",
       "                        contractType  country  \n",
       "0                                 NA       43  \n",
       "1              Aquisição de serviços       43  \n",
       "2      Empreitadas de obras públicas       43  \n",
       "3      Empreitadas de obras públicas       43  \n",
       "4             Locação de bens móveis       43  \n",
       "...                              ...      ...  \n",
       "10219          Aquisição de serviços       43  \n",
       "10220          Aquisição de serviços       43  \n",
       "10221  Empreitadas de obras públicas       43  \n",
       "10222  Empreitadas de obras públicas       43  \n",
       "10223       Aquisição de bens móveis       43  \n",
       "\n",
       "[10224 rows x 517 columns]"
      ]
     },
     "execution_count": 1016,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type2s\n",
    "final_Type2s = pd.concat([Type2s_embeds_df_2, samp_Type2s['parsedPriceBracket'], samp_Type2s['category1sSplit'], samp_Type2s['nif'], samp_Type2s['Type1Type'], samp_Type2s['country']], axis=1)\n",
    "#final_Type2s = final_Type2s.drop(columns = ['lemm'])\n",
    "final_Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1017,
   "id": "0466ef53",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>contractType_Sociedade</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de bens móveis</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de bens móveis&lt;br/&gt;Aquisição de serviços</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de bens móveis&lt;br/&gt;Concessão de obras públicas</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de serviços</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de serviços&lt;br/&gt;Aquisição de bens móveis</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Aquisição de serviços&lt;br/&gt;Locação de bens móveis</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Empreitadas de obras públicas</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Locação de bens móveis</th>\n",
       "      <th>contractType_Sociedade&lt;br/&gt;Locação de bens móveis&lt;br/&gt;Aquisição de bens móveis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.026911</td>\n",
       "      <td>-0.051014</td>\n",
       "      <td>-0.023318</td>\n",
       "      <td>-0.009499</td>\n",
       "      <td>0.060968</td>\n",
       "      <td>0.014730</td>\n",
       "      <td>-0.008912</td>\n",
       "      <td>-0.075087</td>\n",
       "      <td>0.004025</td>\n",
       "      <td>0.006439</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.084498</td>\n",
       "      <td>0.042029</td>\n",
       "      <td>0.019779</td>\n",
       "      <td>-0.020284</td>\n",
       "      <td>0.095920</td>\n",
       "      <td>-0.020238</td>\n",
       "      <td>0.067495</td>\n",
       "      <td>-0.031489</td>\n",
       "      <td>0.034687</td>\n",
       "      <td>0.034047</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.028979</td>\n",
       "      <td>0.006396</td>\n",
       "      <td>-0.018315</td>\n",
       "      <td>-0.067599</td>\n",
       "      <td>0.071495</td>\n",
       "      <td>-0.038623</td>\n",
       "      <td>-0.032496</td>\n",
       "      <td>-0.008276</td>\n",
       "      <td>0.044929</td>\n",
       "      <td>0.021890</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.060817</td>\n",
       "      <td>-0.024744</td>\n",
       "      <td>0.011874</td>\n",
       "      <td>-0.039798</td>\n",
       "      <td>0.024335</td>\n",
       "      <td>0.085144</td>\n",
       "      <td>-0.014147</td>\n",
       "      <td>-0.021923</td>\n",
       "      <td>-0.032357</td>\n",
       "      <td>-0.002618</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.001571</td>\n",
       "      <td>0.051739</td>\n",
       "      <td>0.081232</td>\n",
       "      <td>0.026330</td>\n",
       "      <td>0.015601</td>\n",
       "      <td>-0.051699</td>\n",
       "      <td>0.005150</td>\n",
       "      <td>0.051129</td>\n",
       "      <td>0.004600</td>\n",
       "      <td>0.001109</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10219</th>\n",
       "      <td>-0.033006</td>\n",
       "      <td>-0.027450</td>\n",
       "      <td>-0.028070</td>\n",
       "      <td>0.017208</td>\n",
       "      <td>0.104097</td>\n",
       "      <td>0.025535</td>\n",
       "      <td>-0.012878</td>\n",
       "      <td>-0.015490</td>\n",
       "      <td>-0.044750</td>\n",
       "      <td>0.050181</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10220</th>\n",
       "      <td>-0.045601</td>\n",
       "      <td>0.014797</td>\n",
       "      <td>0.039917</td>\n",
       "      <td>-0.059706</td>\n",
       "      <td>0.027300</td>\n",
       "      <td>0.013217</td>\n",
       "      <td>0.006027</td>\n",
       "      <td>0.002019</td>\n",
       "      <td>-0.014111</td>\n",
       "      <td>-0.043886</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10221</th>\n",
       "      <td>-0.005691</td>\n",
       "      <td>0.034652</td>\n",
       "      <td>-0.043687</td>\n",
       "      <td>-0.074397</td>\n",
       "      <td>0.079882</td>\n",
       "      <td>0.009865</td>\n",
       "      <td>0.009877</td>\n",
       "      <td>-0.013330</td>\n",
       "      <td>0.032172</td>\n",
       "      <td>0.000779</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10222</th>\n",
       "      <td>-0.008967</td>\n",
       "      <td>0.048813</td>\n",
       "      <td>-0.031824</td>\n",
       "      <td>-0.009013</td>\n",
       "      <td>0.119169</td>\n",
       "      <td>-0.012493</td>\n",
       "      <td>0.013489</td>\n",
       "      <td>-0.038627</td>\n",
       "      <td>-0.054581</td>\n",
       "      <td>0.013872</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10223</th>\n",
       "      <td>-0.002505</td>\n",
       "      <td>0.094942</td>\n",
       "      <td>0.044165</td>\n",
       "      <td>-0.074706</td>\n",
       "      <td>0.075887</td>\n",
       "      <td>0.059517</td>\n",
       "      <td>0.007757</td>\n",
       "      <td>-0.007459</td>\n",
       "      <td>-0.097575</td>\n",
       "      <td>0.025067</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10224 rows × 762 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.026911 -0.051014 -0.023318 -0.009499  0.060968  0.014730 -0.008912   \n",
       "1      0.084498  0.042029  0.019779 -0.020284  0.095920 -0.020238  0.067495   \n",
       "2      0.028979  0.006396 -0.018315 -0.067599  0.071495 -0.038623 -0.032496   \n",
       "3     -0.060817 -0.024744  0.011874 -0.039798  0.024335  0.085144 -0.014147   \n",
       "4      0.001571  0.051739  0.081232  0.026330  0.015601 -0.051699  0.005150   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10219 -0.033006 -0.027450 -0.028070  0.017208  0.104097  0.025535 -0.012878   \n",
       "10220 -0.045601  0.014797  0.039917 -0.059706  0.027300  0.013217  0.006027   \n",
       "10221 -0.005691  0.034652 -0.043687 -0.074397  0.079882  0.009865  0.009877   \n",
       "10222 -0.008967  0.048813 -0.031824 -0.009013  0.119169 -0.012493  0.013489   \n",
       "10223 -0.002505  0.094942  0.044165 -0.074706  0.075887  0.059517  0.007757   \n",
       "\n",
       "          col_7     col_8     col_9  ...  contractType_Sociedade  \\\n",
       "0     -0.075087  0.004025  0.006439  ...                       0   \n",
       "1     -0.031489  0.034687  0.034047  ...                       0   \n",
       "2     -0.008276  0.044929  0.021890  ...                       0   \n",
       "3     -0.021923 -0.032357 -0.002618  ...                       0   \n",
       "4      0.051129  0.004600  0.001109  ...                       0   \n",
       "...         ...       ...       ...  ...                     ...   \n",
       "10219 -0.015490 -0.044750  0.050181  ...                       0   \n",
       "10220  0.002019 -0.014111 -0.043886  ...                       0   \n",
       "10221 -0.013330  0.032172  0.000779  ...                       0   \n",
       "10222 -0.038627 -0.054581  0.013872  ...                       0   \n",
       "10223 -0.007459 -0.097575  0.025067  ...                       0   \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de bens móveis  \\\n",
       "0                                                      0     \n",
       "1                                                      0     \n",
       "2                                                      0     \n",
       "3                                                      0     \n",
       "4                                                      0     \n",
       "...                                                  ...     \n",
       "10219                                                  0     \n",
       "10220                                                  0     \n",
       "10221                                                  0     \n",
       "10222                                                  0     \n",
       "10223                                                  0     \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de bens móveis<br/>Aquisição de serviços  \\\n",
       "0                                                      0                               \n",
       "1                                                      0                               \n",
       "2                                                      0                               \n",
       "3                                                      0                               \n",
       "4                                                      0                               \n",
       "...                                                  ...                               \n",
       "10219                                                  0                               \n",
       "10220                                                  0                               \n",
       "10221                                                  0                               \n",
       "10222                                                  0                               \n",
       "10223                                                  0                               \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de bens móveis<br/>Concessão de obras públicas  \\\n",
       "0                                                      0                                     \n",
       "1                                                      0                                     \n",
       "2                                                      0                                     \n",
       "3                                                      0                                     \n",
       "4                                                      0                                     \n",
       "...                                                  ...                                     \n",
       "10219                                                  0                                     \n",
       "10220                                                  0                                     \n",
       "10221                                                  0                                     \n",
       "10222                                                  0                                     \n",
       "10223                                                  0                                     \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de serviços  \\\n",
       "0                                                     0   \n",
       "1                                                     0   \n",
       "2                                                     0   \n",
       "3                                                     0   \n",
       "4                                                     0   \n",
       "...                                                 ...   \n",
       "10219                                                 0   \n",
       "10220                                                 0   \n",
       "10221                                                 0   \n",
       "10222                                                 0   \n",
       "10223                                                 0   \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de serviços<br/>Aquisição de bens móveis  \\\n",
       "0                                                      0                               \n",
       "1                                                      0                               \n",
       "2                                                      0                               \n",
       "3                                                      0                               \n",
       "4                                                      0                               \n",
       "...                                                  ...                               \n",
       "10219                                                  0                               \n",
       "10220                                                  0                               \n",
       "10221                                                  0                               \n",
       "10222                                                  0                               \n",
       "10223                                                  0                               \n",
       "\n",
       "       contractType_Sociedade<br/>Aquisição de serviços<br/>Locação de bens móveis  \\\n",
       "0                                                      0                             \n",
       "1                                                      0                             \n",
       "2                                                      0                             \n",
       "3                                                      0                             \n",
       "4                                                      0                             \n",
       "...                                                  ...                             \n",
       "10219                                                  0                             \n",
       "10220                                                  0                             \n",
       "10221                                                  0                             \n",
       "10222                                                  0                             \n",
       "10223                                                  0                             \n",
       "\n",
       "       contractType_Sociedade<br/>Empreitadas de obras públicas  \\\n",
       "0                                                      0          \n",
       "1                                                      0          \n",
       "2                                                      0          \n",
       "3                                                      0          \n",
       "4                                                      0          \n",
       "...                                                  ...          \n",
       "10219                                                  0          \n",
       "10220                                                  0          \n",
       "10221                                                  0          \n",
       "10222                                                  0          \n",
       "10223                                                  0          \n",
       "\n",
       "       contractType_Sociedade<br/>Locação de bens móveis  \\\n",
       "0                                                      0   \n",
       "1                                                      0   \n",
       "2                                                      0   \n",
       "3                                                      0   \n",
       "4                                                      0   \n",
       "...                                                  ...   \n",
       "10219                                                  0   \n",
       "10220                                                  0   \n",
       "10221                                                  0   \n",
       "10222                                                  0   \n",
       "10223                                                  0   \n",
       "\n",
       "       contractType_Sociedade<br/>Locação de bens móveis<br/>Aquisição de bens móveis  \n",
       "0                                                      0                               \n",
       "1                                                      0                               \n",
       "2                                                      0                               \n",
       "3                                                      0                               \n",
       "4                                                      0                               \n",
       "...                                                  ...                               \n",
       "10219                                                  0                               \n",
       "10220                                                  0                               \n",
       "10221                                                  0                               \n",
       "10222                                                  0                               \n",
       "10223                                                  0                               \n",
       "\n",
       "[10224 rows x 762 columns]"
      ]
     },
     "execution_count": 1017,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Converting categorical data into dummy or indicator variables for Type2s\n",
    "final_Type2s = pd.get_dummies(final_Type2s, prefix={'Type1Type':'Type1Type'}, \n",
    "                                              columns = ['Type1Type'],\n",
    "                          drop_first=True)\n",
    "final_Type2s"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "23ee7e21",
   "metadata": {},
   "source": [
    "# Build the ANNOY index using all features : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1018,
   "id": "f2ef6166",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 16.48 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type2s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 762\n",
    "\n",
    "\n",
    "t_Type2s_all_ftrs = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_Type2s.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_Type2s_all_ftrs.add_item(index, row)\n",
    "\n",
    "t_Type2s_all_ftrs.build(10) # 10 trees\n",
    "t_Type2s_all_ftrs.save('test_Type2s_all_ftrs.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))   "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c165b92",
   "metadata": {},
   "source": [
    "# Testing with all features : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8254eabc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "a_no = <>\n",
    "a_list = [<>>]\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_all_ftrs.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 2200 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f8133c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "a_no = <>\n",
    "a_list = [<>]\n",
    "indx = samp_Type2s.index[samp_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(samp_Type2s[samp_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = samp_Type2s[samp_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s_all_ftrs.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 5000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "69c3dd30",
   "metadata": {},
   "source": [
    "# Sampling a smaller set only from the given list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1facc78b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#using Type1 nos from the test set\n",
    "ctr_list = [<>] #Info removed to comply with the NDA\n",
    "\n",
    "#plus 10K Type1s\n",
    "#samp_ctr = data_df[(data_df.ctr_Type2s == 1)].head(10000)\n",
    "#samp_ctr = samp_ctr.append(data_df[data_df['id'].isin(ctr_list)]).reset_index(drop=True) \n",
    "samp_ctr_s = data_df[data_df['id'].isin(ctr_list)].reset_index(drop=True) \n",
    "samp_ctr_s \n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1032,
   "id": "a1bee4e5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 203 records to process today\n",
      "203 records are added\n",
      "\n",
      "This function took 10.90 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Text Embedding for Type1 Titles\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "model = SentenceTransformer('distiluse-base-multilingual-cased-v1')\n",
    "\n",
    "start = time.time()\n",
    "embeddings_ctr_s = []\n",
    "\n",
    "n = len(samp_ctr_s['lemm'])\n",
    "\n",
    "print(f\"There are {n} records to process today\")\n",
    "embeddings_ctr_s = model.encode(samp_ctr['lemm'])\n",
    "print(f\"{n} records are added\")\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1033,
   "id": "65098f92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_502</th>\n",
       "      <th>col_503</th>\n",
       "      <th>col_504</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.012597</td>\n",
       "      <td>-0.021169</td>\n",
       "      <td>0.004364</td>\n",
       "      <td>-0.034760</td>\n",
       "      <td>0.011707</td>\n",
       "      <td>-0.031097</td>\n",
       "      <td>-0.009160</td>\n",
       "      <td>0.011405</td>\n",
       "      <td>0.027637</td>\n",
       "      <td>-0.011639</td>\n",
       "      <td>...</td>\n",
       "      <td>0.015424</td>\n",
       "      <td>0.001912</td>\n",
       "      <td>0.017126</td>\n",
       "      <td>-0.045687</td>\n",
       "      <td>0.080389</td>\n",
       "      <td>-0.012189</td>\n",
       "      <td>0.080207</td>\n",
       "      <td>-0.003466</td>\n",
       "      <td>-0.027553</td>\n",
       "      <td>0.053678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.058579</td>\n",
       "      <td>0.001575</td>\n",
       "      <td>-0.023590</td>\n",
       "      <td>-0.004432</td>\n",
       "      <td>0.068550</td>\n",
       "      <td>0.020469</td>\n",
       "      <td>-0.021316</td>\n",
       "      <td>0.035644</td>\n",
       "      <td>0.040441</td>\n",
       "      <td>0.011761</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021265</td>\n",
       "      <td>-0.053317</td>\n",
       "      <td>-0.028228</td>\n",
       "      <td>-0.050260</td>\n",
       "      <td>0.036425</td>\n",
       "      <td>-0.063614</td>\n",
       "      <td>0.001759</td>\n",
       "      <td>-0.003221</td>\n",
       "      <td>-0.011604</td>\n",
       "      <td>0.059579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.039020</td>\n",
       "      <td>-0.079821</td>\n",
       "      <td>-0.015701</td>\n",
       "      <td>-0.017194</td>\n",
       "      <td>0.064387</td>\n",
       "      <td>-0.022447</td>\n",
       "      <td>0.009039</td>\n",
       "      <td>0.041438</td>\n",
       "      <td>0.020767</td>\n",
       "      <td>-0.007066</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.011862</td>\n",
       "      <td>0.010413</td>\n",
       "      <td>0.044966</td>\n",
       "      <td>-0.001938</td>\n",
       "      <td>0.058422</td>\n",
       "      <td>0.008124</td>\n",
       "      <td>0.054385</td>\n",
       "      <td>0.036261</td>\n",
       "      <td>0.031451</td>\n",
       "      <td>0.026654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.059088</td>\n",
       "      <td>0.050582</td>\n",
       "      <td>0.032243</td>\n",
       "      <td>-0.064278</td>\n",
       "      <td>0.104531</td>\n",
       "      <td>-0.102314</td>\n",
       "      <td>0.021116</td>\n",
       "      <td>0.002731</td>\n",
       "      <td>-0.096195</td>\n",
       "      <td>0.028712</td>\n",
       "      <td>...</td>\n",
       "      <td>0.033884</td>\n",
       "      <td>-0.052595</td>\n",
       "      <td>0.076570</td>\n",
       "      <td>-0.007741</td>\n",
       "      <td>-0.008991</td>\n",
       "      <td>-0.060037</td>\n",
       "      <td>-0.008681</td>\n",
       "      <td>0.022304</td>\n",
       "      <td>-0.025945</td>\n",
       "      <td>-0.045215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.053640</td>\n",
       "      <td>0.086881</td>\n",
       "      <td>-0.069122</td>\n",
       "      <td>0.024354</td>\n",
       "      <td>-0.068090</td>\n",
       "      <td>0.072996</td>\n",
       "      <td>0.007695</td>\n",
       "      <td>0.011354</td>\n",
       "      <td>0.017544</td>\n",
       "      <td>0.006993</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018511</td>\n",
       "      <td>-0.031353</td>\n",
       "      <td>0.002072</td>\n",
       "      <td>0.021107</td>\n",
       "      <td>-0.004926</td>\n",
       "      <td>0.006753</td>\n",
       "      <td>-0.008968</td>\n",
       "      <td>-0.036318</td>\n",
       "      <td>-0.004508</td>\n",
       "      <td>0.010584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>-0.019831</td>\n",
       "      <td>0.004905</td>\n",
       "      <td>0.027139</td>\n",
       "      <td>-0.016492</td>\n",
       "      <td>-0.058190</td>\n",
       "      <td>0.028122</td>\n",
       "      <td>-0.017875</td>\n",
       "      <td>0.019658</td>\n",
       "      <td>-0.020580</td>\n",
       "      <td>-0.023857</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.003145</td>\n",
       "      <td>-0.041780</td>\n",
       "      <td>-0.004194</td>\n",
       "      <td>0.009265</td>\n",
       "      <td>-0.030309</td>\n",
       "      <td>0.020869</td>\n",
       "      <td>0.053502</td>\n",
       "      <td>0.048767</td>\n",
       "      <td>-0.002036</td>\n",
       "      <td>-0.012913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>-0.025081</td>\n",
       "      <td>-0.052650</td>\n",
       "      <td>-0.045189</td>\n",
       "      <td>-0.000725</td>\n",
       "      <td>-0.001611</td>\n",
       "      <td>0.042363</td>\n",
       "      <td>-0.022679</td>\n",
       "      <td>0.031628</td>\n",
       "      <td>0.016473</td>\n",
       "      <td>-0.020218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035204</td>\n",
       "      <td>-0.007561</td>\n",
       "      <td>0.009151</td>\n",
       "      <td>-0.000184</td>\n",
       "      <td>-0.002363</td>\n",
       "      <td>0.003779</td>\n",
       "      <td>-0.002796</td>\n",
       "      <td>0.112329</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>-0.030824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>0.007497</td>\n",
       "      <td>0.047767</td>\n",
       "      <td>-0.001658</td>\n",
       "      <td>-0.026412</td>\n",
       "      <td>0.055747</td>\n",
       "      <td>-0.010143</td>\n",
       "      <td>0.070942</td>\n",
       "      <td>0.019195</td>\n",
       "      <td>-0.034207</td>\n",
       "      <td>0.040163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031154</td>\n",
       "      <td>0.001890</td>\n",
       "      <td>0.012175</td>\n",
       "      <td>0.001437</td>\n",
       "      <td>0.017530</td>\n",
       "      <td>0.021713</td>\n",
       "      <td>0.050491</td>\n",
       "      <td>-0.057415</td>\n",
       "      <td>-0.008700</td>\n",
       "      <td>-0.009758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>0.007973</td>\n",
       "      <td>-0.006567</td>\n",
       "      <td>0.007972</td>\n",
       "      <td>-0.024185</td>\n",
       "      <td>0.037945</td>\n",
       "      <td>-0.024951</td>\n",
       "      <td>0.021616</td>\n",
       "      <td>0.010492</td>\n",
       "      <td>-0.025213</td>\n",
       "      <td>0.039273</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042061</td>\n",
       "      <td>-0.075808</td>\n",
       "      <td>-0.054949</td>\n",
       "      <td>0.004109</td>\n",
       "      <td>0.009871</td>\n",
       "      <td>0.007268</td>\n",
       "      <td>0.058270</td>\n",
       "      <td>0.018326</td>\n",
       "      <td>0.012770</td>\n",
       "      <td>0.049915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>0.006300</td>\n",
       "      <td>-0.034266</td>\n",
       "      <td>-0.021577</td>\n",
       "      <td>-0.033345</td>\n",
       "      <td>0.005970</td>\n",
       "      <td>0.015349</td>\n",
       "      <td>-0.009623</td>\n",
       "      <td>-0.016358</td>\n",
       "      <td>-0.022686</td>\n",
       "      <td>0.004165</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060385</td>\n",
       "      <td>-0.022496</td>\n",
       "      <td>-0.006637</td>\n",
       "      <td>-0.033977</td>\n",
       "      <td>0.061615</td>\n",
       "      <td>0.003057</td>\n",
       "      <td>0.069206</td>\n",
       "      <td>-0.045256</td>\n",
       "      <td>0.009401</td>\n",
       "      <td>0.048284</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>203 rows × 512 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0   -0.012597 -0.021169  0.004364 -0.034760  0.011707 -0.031097 -0.009160   \n",
       "1    0.058579  0.001575 -0.023590 -0.004432  0.068550  0.020469 -0.021316   \n",
       "2   -0.039020 -0.079821 -0.015701 -0.017194  0.064387 -0.022447  0.009039   \n",
       "3    0.059088  0.050582  0.032243 -0.064278  0.104531 -0.102314  0.021116   \n",
       "4    0.053640  0.086881 -0.069122  0.024354 -0.068090  0.072996  0.007695   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "198 -0.019831  0.004905  0.027139 -0.016492 -0.058190  0.028122 -0.017875   \n",
       "199 -0.025081 -0.052650 -0.045189 -0.000725 -0.001611  0.042363 -0.022679   \n",
       "200  0.007497  0.047767 -0.001658 -0.026412  0.055747 -0.010143  0.070942   \n",
       "201  0.007973 -0.006567  0.007972 -0.024185  0.037945 -0.024951  0.021616   \n",
       "202  0.006300 -0.034266 -0.021577 -0.033345  0.005970  0.015349 -0.009623   \n",
       "\n",
       "        col_7     col_8     col_9  ...   col_502   col_503   col_504  \\\n",
       "0    0.011405  0.027637 -0.011639  ...  0.015424  0.001912  0.017126   \n",
       "1    0.035644  0.040441  0.011761  ...  0.021265 -0.053317 -0.028228   \n",
       "2    0.041438  0.020767 -0.007066  ... -0.011862  0.010413  0.044966   \n",
       "3    0.002731 -0.096195  0.028712  ...  0.033884 -0.052595  0.076570   \n",
       "4    0.011354  0.017544  0.006993  ...  0.018511 -0.031353  0.002072   \n",
       "..        ...       ...       ...  ...       ...       ...       ...   \n",
       "198  0.019658 -0.020580 -0.023857  ... -0.003145 -0.041780 -0.004194   \n",
       "199  0.031628  0.016473 -0.020218  ...  0.035204 -0.007561  0.009151   \n",
       "200  0.019195 -0.034207  0.040163  ...  0.031154  0.001890  0.012175   \n",
       "201  0.010492 -0.025213  0.039273  ...  0.042061 -0.075808 -0.054949   \n",
       "202 -0.016358 -0.022686  0.004165  ...  0.060385 -0.022496 -0.006637   \n",
       "\n",
       "      col_505   col_506   col_507   col_508   col_509   col_510   col_511  \n",
       "0   -0.045687  0.080389 -0.012189  0.080207 -0.003466 -0.027553  0.053678  \n",
       "1   -0.050260  0.036425 -0.063614  0.001759 -0.003221 -0.011604  0.059579  \n",
       "2   -0.001938  0.058422  0.008124  0.054385  0.036261  0.031451  0.026654  \n",
       "3   -0.007741 -0.008991 -0.060037 -0.008681  0.022304 -0.025945 -0.045215  \n",
       "4    0.021107 -0.004926  0.006753 -0.008968 -0.036318 -0.004508  0.010584  \n",
       "..        ...       ...       ...       ...       ...       ...       ...  \n",
       "198  0.009265 -0.030309  0.020869  0.053502  0.048767 -0.002036 -0.012913  \n",
       "199 -0.000184 -0.002363  0.003779 -0.002796  0.112329  0.023464 -0.030824  \n",
       "200  0.001437  0.017530  0.021713  0.050491 -0.057415 -0.008700 -0.009758  \n",
       "201  0.004109  0.009871  0.007268  0.058270  0.018326  0.012770  0.049915  \n",
       "202 -0.033977  0.061615  0.003057  0.069206 -0.045256  0.009401  0.048284  \n",
       "\n",
       "[203 rows x 512 columns]"
      ]
     },
     "execution_count": 1033,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ctr_embeds_df_s = pd.DataFrame(embeddings_ctr_s)\n",
    "ctr_embeds_df_s.columns = ['col_'+str(num) for num in range(embeddings_ctr_s.shape[1])]\n",
    "ctr_embeds_df_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1034,
   "id": "a9a0f663",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "      <th>parsedPriceBracket</th>\n",
       "      <th>cpvsSplit</th>\n",
       "      <th>nif</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.011337</td>\n",
       "      <td>-0.019052</td>\n",
       "      <td>0.003927</td>\n",
       "      <td>-0.031284</td>\n",
       "      <td>0.010536</td>\n",
       "      <td>-0.027988</td>\n",
       "      <td>-0.008244</td>\n",
       "      <td>0.010264</td>\n",
       "      <td>0.024874</td>\n",
       "      <td>-0.010475</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.041119</td>\n",
       "      <td>0.072350</td>\n",
       "      <td>-0.010970</td>\n",
       "      <td>0.072187</td>\n",
       "      <td>-0.003120</td>\n",
       "      <td>-0.024798</td>\n",
       "      <td>0.048310</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.052721</td>\n",
       "      <td>0.001417</td>\n",
       "      <td>-0.021231</td>\n",
       "      <td>-0.003988</td>\n",
       "      <td>0.061695</td>\n",
       "      <td>0.018422</td>\n",
       "      <td>-0.019184</td>\n",
       "      <td>0.032079</td>\n",
       "      <td>0.036397</td>\n",
       "      <td>0.010585</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.045234</td>\n",
       "      <td>0.032783</td>\n",
       "      <td>-0.057253</td>\n",
       "      <td>0.001583</td>\n",
       "      <td>-0.002899</td>\n",
       "      <td>-0.010444</td>\n",
       "      <td>0.053621</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.035118</td>\n",
       "      <td>-0.071839</td>\n",
       "      <td>-0.014131</td>\n",
       "      <td>-0.015474</td>\n",
       "      <td>0.057949</td>\n",
       "      <td>-0.020202</td>\n",
       "      <td>0.008135</td>\n",
       "      <td>0.037294</td>\n",
       "      <td>0.018690</td>\n",
       "      <td>-0.006359</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001745</td>\n",
       "      <td>0.052580</td>\n",
       "      <td>0.007311</td>\n",
       "      <td>0.048947</td>\n",
       "      <td>0.032635</td>\n",
       "      <td>0.028306</td>\n",
       "      <td>0.023988</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.053179</td>\n",
       "      <td>0.045524</td>\n",
       "      <td>0.029019</td>\n",
       "      <td>-0.057850</td>\n",
       "      <td>0.094078</td>\n",
       "      <td>-0.092082</td>\n",
       "      <td>0.019005</td>\n",
       "      <td>0.002458</td>\n",
       "      <td>-0.086575</td>\n",
       "      <td>0.025841</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.006967</td>\n",
       "      <td>-0.008092</td>\n",
       "      <td>-0.054033</td>\n",
       "      <td>-0.007813</td>\n",
       "      <td>0.020074</td>\n",
       "      <td>-0.023350</td>\n",
       "      <td>-0.040694</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.048276</td>\n",
       "      <td>0.078193</td>\n",
       "      <td>-0.062210</td>\n",
       "      <td>0.021919</td>\n",
       "      <td>-0.061281</td>\n",
       "      <td>0.065697</td>\n",
       "      <td>0.006926</td>\n",
       "      <td>0.010218</td>\n",
       "      <td>0.015790</td>\n",
       "      <td>0.006293</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018997</td>\n",
       "      <td>-0.004433</td>\n",
       "      <td>0.006077</td>\n",
       "      <td>-0.008072</td>\n",
       "      <td>-0.032686</td>\n",
       "      <td>-0.004057</td>\n",
       "      <td>0.009525</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>-0.017847</td>\n",
       "      <td>0.004415</td>\n",
       "      <td>0.024425</td>\n",
       "      <td>-0.014843</td>\n",
       "      <td>-0.052371</td>\n",
       "      <td>0.025310</td>\n",
       "      <td>-0.016088</td>\n",
       "      <td>0.017692</td>\n",
       "      <td>-0.018522</td>\n",
       "      <td>-0.021471</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008338</td>\n",
       "      <td>-0.027278</td>\n",
       "      <td>0.018783</td>\n",
       "      <td>0.048152</td>\n",
       "      <td>0.043890</td>\n",
       "      <td>-0.001833</td>\n",
       "      <td>-0.011621</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>-0.022573</td>\n",
       "      <td>-0.047385</td>\n",
       "      <td>-0.040670</td>\n",
       "      <td>-0.000652</td>\n",
       "      <td>-0.001450</td>\n",
       "      <td>0.038126</td>\n",
       "      <td>-0.020411</td>\n",
       "      <td>0.028465</td>\n",
       "      <td>0.014826</td>\n",
       "      <td>-0.018196</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.000166</td>\n",
       "      <td>-0.002127</td>\n",
       "      <td>0.003401</td>\n",
       "      <td>-0.002516</td>\n",
       "      <td>0.101096</td>\n",
       "      <td>0.021118</td>\n",
       "      <td>-0.027741</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>200</th>\n",
       "      <td>0.006748</td>\n",
       "      <td>0.042990</td>\n",
       "      <td>-0.001492</td>\n",
       "      <td>-0.023771</td>\n",
       "      <td>0.050172</td>\n",
       "      <td>-0.009129</td>\n",
       "      <td>0.063847</td>\n",
       "      <td>0.017276</td>\n",
       "      <td>-0.030787</td>\n",
       "      <td>0.036147</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001293</td>\n",
       "      <td>0.015777</td>\n",
       "      <td>0.019542</td>\n",
       "      <td>0.045442</td>\n",
       "      <td>-0.051673</td>\n",
       "      <td>-0.007830</td>\n",
       "      <td>-0.008782</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>201</th>\n",
       "      <td>0.007176</td>\n",
       "      <td>-0.005910</td>\n",
       "      <td>0.007175</td>\n",
       "      <td>-0.021767</td>\n",
       "      <td>0.034151</td>\n",
       "      <td>-0.022456</td>\n",
       "      <td>0.019454</td>\n",
       "      <td>0.009442</td>\n",
       "      <td>-0.022692</td>\n",
       "      <td>0.035346</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003698</td>\n",
       "      <td>0.008884</td>\n",
       "      <td>0.006541</td>\n",
       "      <td>0.052443</td>\n",
       "      <td>0.016494</td>\n",
       "      <td>0.011493</td>\n",
       "      <td>0.044924</td>\n",
       "      <td>0.1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>202</th>\n",
       "      <td>0.005670</td>\n",
       "      <td>-0.030839</td>\n",
       "      <td>-0.019419</td>\n",
       "      <td>-0.030010</td>\n",
       "      <td>0.005373</td>\n",
       "      <td>0.013814</td>\n",
       "      <td>-0.008660</td>\n",
       "      <td>-0.014722</td>\n",
       "      <td>-0.020418</td>\n",
       "      <td>0.003749</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.030580</td>\n",
       "      <td>0.055454</td>\n",
       "      <td>0.002751</td>\n",
       "      <td>0.062285</td>\n",
       "      <td>-0.040730</td>\n",
       "      <td>0.008461</td>\n",
       "      <td>0.043456</td>\n",
       "      <td>0.2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>203 rows × 515 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0   -0.011337 -0.019052  0.003927 -0.031284  0.010536 -0.027988 -0.008244   \n",
       "1    0.052721  0.001417 -0.021231 -0.003988  0.061695  0.018422 -0.019184   \n",
       "2   -0.035118 -0.071839 -0.014131 -0.015474  0.057949 -0.020202  0.008135   \n",
       "3    0.053179  0.045524  0.029019 -0.057850  0.094078 -0.092082  0.019005   \n",
       "4    0.048276  0.078193 -0.062210  0.021919 -0.061281  0.065697  0.006926   \n",
       "..        ...       ...       ...       ...       ...       ...       ...   \n",
       "198 -0.017847  0.004415  0.024425 -0.014843 -0.052371  0.025310 -0.016088   \n",
       "199 -0.022573 -0.047385 -0.040670 -0.000652 -0.001450  0.038126 -0.020411   \n",
       "200  0.006748  0.042990 -0.001492 -0.023771  0.050172 -0.009129  0.063847   \n",
       "201  0.007176 -0.005910  0.007175 -0.021767  0.034151 -0.022456  0.019454   \n",
       "202  0.005670 -0.030839 -0.019419 -0.030010  0.005373  0.013814 -0.008660   \n",
       "\n",
       "        col_7     col_8     col_9  ...   col_505   col_506   col_507  \\\n",
       "0    0.010264  0.024874 -0.010475  ... -0.041119  0.072350 -0.010970   \n",
       "1    0.032079  0.036397  0.010585  ... -0.045234  0.032783 -0.057253   \n",
       "2    0.037294  0.018690 -0.006359  ... -0.001745  0.052580  0.007311   \n",
       "3    0.002458 -0.086575  0.025841  ... -0.006967 -0.008092 -0.054033   \n",
       "4    0.010218  0.015790  0.006293  ...  0.018997 -0.004433  0.006077   \n",
       "..        ...       ...       ...  ...       ...       ...       ...   \n",
       "198  0.017692 -0.018522 -0.021471  ...  0.008338 -0.027278  0.018783   \n",
       "199  0.028465  0.014826 -0.018196  ... -0.000166 -0.002127  0.003401   \n",
       "200  0.017276 -0.030787  0.036147  ...  0.001293  0.015777  0.019542   \n",
       "201  0.009442 -0.022692  0.035346  ...  0.003698  0.008884  0.006541   \n",
       "202 -0.014722 -0.020418  0.003749  ... -0.030580  0.055454  0.002751   \n",
       "\n",
       "      col_508   col_509   col_510   col_511  parsedPriceBracket  cpvsSplit  \\\n",
       "0    0.072187 -0.003120 -0.024798  0.048310                 0.1          0   \n",
       "1    0.001583 -0.002899 -0.010444  0.053621                 0.1          0   \n",
       "2    0.048947  0.032635  0.028306  0.023988                 0.1          0   \n",
       "3   -0.007813  0.020074 -0.023350 -0.040694                 0.1          0   \n",
       "4   -0.008072 -0.032686 -0.004057  0.009525                 0.1          0   \n",
       "..        ...       ...       ...       ...                 ...        ...   \n",
       "198  0.048152  0.043890 -0.001833 -0.011621                 0.1          0   \n",
       "199 -0.002516  0.101096  0.021118 -0.027741                 0.1          0   \n",
       "200  0.045442 -0.051673 -0.007830 -0.008782                 0.1          0   \n",
       "201  0.052443  0.016494  0.011493  0.044924                 0.1          0   \n",
       "202  0.062285 -0.040730  0.008461  0.043456                 0.2          0   \n",
       "\n",
       "     nif  \n",
       "0      0  \n",
       "1      0  \n",
       "2      0  \n",
       "3      0  \n",
       "4      0  \n",
       "..   ...  \n",
       "198    0  \n",
       "199    0  \n",
       "200    0  \n",
       "201    0  \n",
       "202    0  \n",
       "\n",
       "[203 rows x 515 columns]"
      ]
     },
     "execution_count": 1034,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replacing text with the embeddings for Type1s : multilingual\n",
    "final_ctr_s = pd.concat([ctr_embeds_df_s * 0.9, samp_ctr['parsedPriceBracket'] * 0.1, samp_ctr['category1sSplit'] * 0, samp_ctr['nif'] * 0], axis=1)\n",
    "#final_ctr = final_ctr.drop(columns = ['lemm'])\n",
    "final_ctr_s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1035,
   "id": "c754b4d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "This function took 0.23 s to compute.\n"
     ]
    }
   ],
   "source": [
    "#Using ANNOY: Approximate Nearest Neighbours Oh yeah! by Spotify for Type1s\n",
    "from annoy import AnnoyIndex\n",
    "\n",
    "import time\n",
    "start = time.time()\n",
    "\n",
    "f = 515\n",
    "\n",
    "t_ctr_ftr_s = AnnoyIndex(f, 'angular')  #Length of item vector that will be indexed\n",
    "#for index, row in dummy_df.head(1000):\n",
    "   # t.add_item(index, row)\n",
    "#adf = final_df.iloc[:500000, :]\n",
    "for index, row in final_ctr_s.iterrows():\n",
    "    #print(f\"index: {index}, row: {row.nif}\")\n",
    "    t_ctr_ftr_s.add_item(index, row)\n",
    "\n",
    "t_ctr_ftr_s.build(10) # 10 trees\n",
    "t_ctr_ftr_s.save('test_ctr_ftr_s.ann')\n",
    "\n",
    "end = time.time()\n",
    "\n",
    "# Total time to compute\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "977b4f41",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 36 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "055e9848",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3bd7cf18",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([168, 164, 37, 67, 165, 52, 65, 66, 76, 196])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4e6ccd2a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 10 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99fc96b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acea4559",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#TC2\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([182, 53, 29, 46, 70, 189, 150, 92, 48, 137])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 296,
   "id": "58a829a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "#Load the data for testing\n",
    "samp_ctr = pd.read_csv('./samp_ctr.csv',lineterminator='\\n', index_col=0)  \n",
    "samp_Type2s = pd.read_csv('./samp_Type2s.csv',lineterminator='\\n', index_col=0)  "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1175c012",
   "metadata": {},
   "source": [
    "# Extra Test Cases: Type1s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3807d4a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#\n",
    "c_no = <>\n",
    "c_list = [<>>]\n",
    "\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 863 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aab4118a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "12e14f97",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit','category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10106, 2964, 386, 4725, 9963, 2841, 405, 1266, 10159, 5890])]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 610,
   "id": "cc802a4c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ctr_ads                                                                                 1.0\n",
       "id                                                                                4990474.0\n",
       "country                                                                                49.0\n",
       "parsedPriceBracket                                                                      1.0\n",
       "executionDeadlineDays                                                                  11.0\n",
       "                                                                                    ...    \n",
       "contractType_Sociedade<br/>Aquisição de serviços<br/>Aquisição de bens móveis           0.0\n",
       "contractType_Sociedade<br/>Aquisição de serviços<br/>Locação de bens móveis             0.0\n",
       "contractType_Sociedade<br/>Empreitadas de obras públicas                                0.0\n",
       "contractType_Sociedade<br/>Locação de bens móveis                                       0.0\n",
       "contractType_Sociedade<br/>Locação de bens móveis<br/>Aquisição de bens móveis          0.0\n",
       "Name: 7479, Length: 781, dtype: float64"
      ]
     },
     "execution_count": 610,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_ctr.iloc[7479]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "201cc44b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 3000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9102f0d7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "add20f70",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit','category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10133, 10041, 8071, 617, 10161, 10130, 9999, 10079, 688, 682])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d47a57ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC2\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d291e16c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fdd9e4a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit','category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10089, 10059, 10062, 10157, 1465, 3572, 750, 975, 10089, 2870, 563, 10071, 1285])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41081e62",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC3\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7682f3ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a331e7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC3\n",
    "samp_ctr[['id', 'objectBriefDescription', 'lemm', 'category1sSplit','category1s_text', 'nif', 'parsedPriceBracket']][samp_ctr.index.isin([10147, 8928, 9782, 7516, 5880, 7543, 5895, 1340, 8392, 7510])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35cd9293",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#TC4\n",
    "c_no = <>\n",
    "c_list = [<>]\n",
    "\n",
    "indx = samp_ctr.index[samp_ctr['id'] == c_no].tolist()\n",
    "print(f\"Type1 No.: {c_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type1s to be matched: \")\n",
    "print(samp_ctr[samp_ctr['id'].isin(c_list)]['id'])\n",
    "lst = samp_ctr[samp_ctr['id'].isin(c_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 3000 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f317fbc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "08576661",
   "metadata": {},
   "outputs": [],
   "source": [
    "final_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4586c368",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 500 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "#result.sort()\n",
    "print(result)\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb7280ed",
   "metadata": {},
   "source": [
    "# Extra Test Cases : Type2s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2c5d2249",
   "metadata": {},
   "outputs": [],
   "source": [
    "#TC1\n",
    "a_no = <>\n",
    "a_list = [<>]\n",
    "indx = final_Type2s.index[final_Type2s['id'] == a_no].tolist()\n",
    "print(f\"Ad No.: {a_no} index: {indx}\")\n",
    "print(\"\\nIndexes of Type2s to be matched: \")\n",
    "print(final_Type2s[final_Type2s['id'].isin(a_list)]['id'])\n",
    "lst = final_Type2s[final_Type2s['id'].isin(a_list)].index\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result['Col1'] = pd.DataFrame(u.get_nns_by_item(indx[0], 500 )) # will find the 1000 nearest neighbors\n",
    "print(\"\\nMatch Results\")\n",
    "res = result[result['Col1'].isin(lst)].Col1\n",
    "print(f\"{len(res)} records match\")\n",
    "res\n",
    "\n",
    "#Cleared Output to comply with the NDA\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34c7fa0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_Type2s.ann') # super fast, will just mmap the file\n",
    "result = pd.DataFrame()\n",
    "result = u.get_nns_by_item(<>, 10 ) # will find the 1000 nearest neighbors\n",
    "\n",
    "result.sort()\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dc39af4",
   "metadata": {},
   "outputs": [],
   "source": [
    "#To view the full list of result\n",
    "c_no_list = [<>]\n",
    "\n",
    "u = AnnoyIndex(f, 'angular')\n",
    "u.load('test_ctr_ftr_2_wt.ann') # super fast, will just mmap the file\n",
    "build_csv_df = pd.DataFrame()\n",
    "\n",
    "for c_no in c_no_list:\n",
    "    result_df  = pd.DataFrame()\n",
    "    indx = samp_ctr.index[samp_ctr['id'] == c_no].to_list()\n",
    "    result_df = pd.DataFrame(u.get_nns_by_item(indx[0], 10 )) # will find the 10 nearest neighbors\n",
    "    result_df['rank']= result_df.index+1\n",
    "    result_df['ref_Type1_no'] = c_no\n",
    "    result_df.rename(columns = {0:'index'}, inplace = True)\n",
    "\n",
    "    abc = samp_ctr[samp_ctr.index.isin(result_df['index'])]\n",
    "    abc['index']= abc.index\n",
    "    result_df = result_df.merge(abc, how='inner',on='index')\n",
    "    build_csv_df = build_csv_df.append(result_df)\n",
    "    \n",
    "print(build_csv_df)\n",
    "build_csv_df.to_csv(\"./top_10_matches_Type1s.csv\")\n",
    "\n",
    "#Cleared Output to comply with the NDA"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f09ead6",
   "metadata": {},
   "source": [
    "# Trying FAISS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1061,
   "id": "acceefd3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Collecting faiss-cpu\n",
      "  Downloading faiss_cpu-1.7.3-cp38-cp38-macosx_10_9_x86_64.whl (5.6 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m5.6/5.6 MB\u001b[0m \u001b[31m1.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: faiss-cpu\n",
      "Successfully installed faiss-cpu-1.7.3\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "#pip install faiss-cpu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "8d646179",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>col_0</th>\n",
       "      <th>col_1</th>\n",
       "      <th>col_2</th>\n",
       "      <th>col_3</th>\n",
       "      <th>col_4</th>\n",
       "      <th>col_5</th>\n",
       "      <th>col_6</th>\n",
       "      <th>col_7</th>\n",
       "      <th>col_8</th>\n",
       "      <th>col_9</th>\n",
       "      <th>...</th>\n",
       "      <th>col_502</th>\n",
       "      <th>col_503</th>\n",
       "      <th>col_504</th>\n",
       "      <th>col_505</th>\n",
       "      <th>col_506</th>\n",
       "      <th>col_507</th>\n",
       "      <th>col_508</th>\n",
       "      <th>col_509</th>\n",
       "      <th>col_510</th>\n",
       "      <th>col_511</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.008769</td>\n",
       "      <td>-0.035696</td>\n",
       "      <td>-0.081959</td>\n",
       "      <td>-0.065422</td>\n",
       "      <td>0.106423</td>\n",
       "      <td>-0.006806</td>\n",
       "      <td>0.014415</td>\n",
       "      <td>-0.019768</td>\n",
       "      <td>-0.007022</td>\n",
       "      <td>0.022455</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.053410</td>\n",
       "      <td>-0.004052</td>\n",
       "      <td>0.065656</td>\n",
       "      <td>-0.080810</td>\n",
       "      <td>0.016209</td>\n",
       "      <td>-0.011536</td>\n",
       "      <td>0.013828</td>\n",
       "      <td>0.040486</td>\n",
       "      <td>0.051302</td>\n",
       "      <td>0.014021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.041093</td>\n",
       "      <td>-0.032101</td>\n",
       "      <td>-0.062681</td>\n",
       "      <td>-0.027410</td>\n",
       "      <td>0.083845</td>\n",
       "      <td>-0.030770</td>\n",
       "      <td>-0.019008</td>\n",
       "      <td>-0.087988</td>\n",
       "      <td>0.053893</td>\n",
       "      <td>0.023575</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.024192</td>\n",
       "      <td>-0.008800</td>\n",
       "      <td>0.037727</td>\n",
       "      <td>-0.035784</td>\n",
       "      <td>0.050928</td>\n",
       "      <td>-0.014732</td>\n",
       "      <td>-0.041200</td>\n",
       "      <td>0.017684</td>\n",
       "      <td>-0.089092</td>\n",
       "      <td>0.001281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.026384</td>\n",
       "      <td>-0.036660</td>\n",
       "      <td>0.010439</td>\n",
       "      <td>0.003870</td>\n",
       "      <td>-0.000859</td>\n",
       "      <td>0.011253</td>\n",
       "      <td>-0.004238</td>\n",
       "      <td>-0.006598</td>\n",
       "      <td>-0.041758</td>\n",
       "      <td>-0.015051</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031650</td>\n",
       "      <td>-0.013433</td>\n",
       "      <td>-0.012822</td>\n",
       "      <td>-0.015036</td>\n",
       "      <td>-0.095395</td>\n",
       "      <td>-0.002170</td>\n",
       "      <td>-0.000507</td>\n",
       "      <td>-0.061892</td>\n",
       "      <td>0.020786</td>\n",
       "      <td>-0.019073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.013121</td>\n",
       "      <td>-0.041674</td>\n",
       "      <td>-0.012382</td>\n",
       "      <td>-0.035365</td>\n",
       "      <td>0.101415</td>\n",
       "      <td>0.027916</td>\n",
       "      <td>0.009208</td>\n",
       "      <td>-0.016659</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>0.010975</td>\n",
       "      <td>...</td>\n",
       "      <td>0.018381</td>\n",
       "      <td>-0.024724</td>\n",
       "      <td>-0.014496</td>\n",
       "      <td>-0.037792</td>\n",
       "      <td>0.022144</td>\n",
       "      <td>-0.045335</td>\n",
       "      <td>-0.020217</td>\n",
       "      <td>-0.001707</td>\n",
       "      <td>0.073547</td>\n",
       "      <td>0.006090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.006715</td>\n",
       "      <td>-0.055337</td>\n",
       "      <td>0.003673</td>\n",
       "      <td>-0.004573</td>\n",
       "      <td>0.057083</td>\n",
       "      <td>0.051284</td>\n",
       "      <td>0.037291</td>\n",
       "      <td>-0.038526</td>\n",
       "      <td>-0.037250</td>\n",
       "      <td>0.031229</td>\n",
       "      <td>...</td>\n",
       "      <td>0.064072</td>\n",
       "      <td>0.038382</td>\n",
       "      <td>-0.078159</td>\n",
       "      <td>0.047688</td>\n",
       "      <td>0.032358</td>\n",
       "      <td>0.024627</td>\n",
       "      <td>-0.033376</td>\n",
       "      <td>0.013414</td>\n",
       "      <td>-0.022359</td>\n",
       "      <td>0.049040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10198</th>\n",
       "      <td>-0.019831</td>\n",
       "      <td>0.004905</td>\n",
       "      <td>0.027139</td>\n",
       "      <td>-0.016492</td>\n",
       "      <td>-0.058190</td>\n",
       "      <td>0.028122</td>\n",
       "      <td>-0.017875</td>\n",
       "      <td>0.019658</td>\n",
       "      <td>-0.020580</td>\n",
       "      <td>-0.023857</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.003145</td>\n",
       "      <td>-0.041780</td>\n",
       "      <td>-0.004194</td>\n",
       "      <td>0.009265</td>\n",
       "      <td>-0.030309</td>\n",
       "      <td>0.020870</td>\n",
       "      <td>0.053502</td>\n",
       "      <td>0.048767</td>\n",
       "      <td>-0.002036</td>\n",
       "      <td>-0.012913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10199</th>\n",
       "      <td>-0.025081</td>\n",
       "      <td>-0.052650</td>\n",
       "      <td>-0.045189</td>\n",
       "      <td>-0.000725</td>\n",
       "      <td>-0.001611</td>\n",
       "      <td>0.042363</td>\n",
       "      <td>-0.022679</td>\n",
       "      <td>0.031628</td>\n",
       "      <td>0.016473</td>\n",
       "      <td>-0.020218</td>\n",
       "      <td>...</td>\n",
       "      <td>0.035204</td>\n",
       "      <td>-0.007561</td>\n",
       "      <td>0.009151</td>\n",
       "      <td>-0.000184</td>\n",
       "      <td>-0.002363</td>\n",
       "      <td>0.003779</td>\n",
       "      <td>-0.002796</td>\n",
       "      <td>0.112329</td>\n",
       "      <td>0.023464</td>\n",
       "      <td>-0.030824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10200</th>\n",
       "      <td>0.007497</td>\n",
       "      <td>0.047767</td>\n",
       "      <td>-0.001658</td>\n",
       "      <td>-0.026412</td>\n",
       "      <td>0.055747</td>\n",
       "      <td>-0.010143</td>\n",
       "      <td>0.070942</td>\n",
       "      <td>0.019195</td>\n",
       "      <td>-0.034207</td>\n",
       "      <td>0.040163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.031154</td>\n",
       "      <td>0.001890</td>\n",
       "      <td>0.012175</td>\n",
       "      <td>0.001437</td>\n",
       "      <td>0.017530</td>\n",
       "      <td>0.021713</td>\n",
       "      <td>0.050491</td>\n",
       "      <td>-0.057415</td>\n",
       "      <td>-0.008700</td>\n",
       "      <td>-0.009758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10201</th>\n",
       "      <td>0.007973</td>\n",
       "      <td>-0.006567</td>\n",
       "      <td>0.007972</td>\n",
       "      <td>-0.024185</td>\n",
       "      <td>0.037945</td>\n",
       "      <td>-0.024951</td>\n",
       "      <td>0.021616</td>\n",
       "      <td>0.010492</td>\n",
       "      <td>-0.025213</td>\n",
       "      <td>0.039273</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042061</td>\n",
       "      <td>-0.075808</td>\n",
       "      <td>-0.054949</td>\n",
       "      <td>0.004109</td>\n",
       "      <td>0.009871</td>\n",
       "      <td>0.007268</td>\n",
       "      <td>0.058270</td>\n",
       "      <td>0.018326</td>\n",
       "      <td>0.012770</td>\n",
       "      <td>0.049915</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10202</th>\n",
       "      <td>0.006300</td>\n",
       "      <td>-0.034266</td>\n",
       "      <td>-0.021577</td>\n",
       "      <td>-0.033345</td>\n",
       "      <td>0.005970</td>\n",
       "      <td>0.015349</td>\n",
       "      <td>-0.009623</td>\n",
       "      <td>-0.016358</td>\n",
       "      <td>-0.022686</td>\n",
       "      <td>0.004165</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060385</td>\n",
       "      <td>-0.022496</td>\n",
       "      <td>-0.006637</td>\n",
       "      <td>-0.033977</td>\n",
       "      <td>0.061615</td>\n",
       "      <td>0.003057</td>\n",
       "      <td>0.069206</td>\n",
       "      <td>-0.045256</td>\n",
       "      <td>0.009401</td>\n",
       "      <td>0.048284</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10203 rows × 512 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          col_0     col_1     col_2     col_3     col_4     col_5     col_6  \\\n",
       "0     -0.008769 -0.035696 -0.081959 -0.065422  0.106423 -0.006806  0.014415   \n",
       "1      0.041093 -0.032101 -0.062681 -0.027410  0.083845 -0.030770 -0.019008   \n",
       "2      0.026384 -0.036660  0.010439  0.003870 -0.000859  0.011253 -0.004238   \n",
       "3      0.013121 -0.041674 -0.012382 -0.035365  0.101415  0.027916  0.009208   \n",
       "4      0.006715 -0.055337  0.003673 -0.004573  0.057083  0.051284  0.037291   \n",
       "...         ...       ...       ...       ...       ...       ...       ...   \n",
       "10198 -0.019831  0.004905  0.027139 -0.016492 -0.058190  0.028122 -0.017875   \n",
       "10199 -0.025081 -0.052650 -0.045189 -0.000725 -0.001611  0.042363 -0.022679   \n",
       "10200  0.007497  0.047767 -0.001658 -0.026412  0.055747 -0.010143  0.070942   \n",
       "10201  0.007973 -0.006567  0.007972 -0.024185  0.037945 -0.024951  0.021616   \n",
       "10202  0.006300 -0.034266 -0.021577 -0.033345  0.005970  0.015349 -0.009623   \n",
       "\n",
       "          col_7     col_8     col_9  ...   col_502   col_503   col_504  \\\n",
       "0     -0.019768 -0.007022  0.022455  ... -0.053410 -0.004052  0.065656   \n",
       "1     -0.087988  0.053893  0.023575  ... -0.024192 -0.008800  0.037727   \n",
       "2     -0.006598 -0.041758 -0.015051  ...  0.031650 -0.013433 -0.012822   \n",
       "3     -0.016659  0.002207  0.010975  ...  0.018381 -0.024724 -0.014496   \n",
       "4     -0.038526 -0.037250  0.031229  ...  0.064072  0.038382 -0.078159   \n",
       "...         ...       ...       ...  ...       ...       ...       ...   \n",
       "10198  0.019658 -0.020580 -0.023857  ... -0.003145 -0.041780 -0.004194   \n",
       "10199  0.031628  0.016473 -0.020218  ...  0.035204 -0.007561  0.009151   \n",
       "10200  0.019195 -0.034207  0.040163  ...  0.031154  0.001890  0.012175   \n",
       "10201  0.010492 -0.025213  0.039273  ...  0.042061 -0.075808 -0.054949   \n",
       "10202 -0.016358 -0.022686  0.004165  ...  0.060385 -0.022496 -0.006637   \n",
       "\n",
       "        col_505   col_506   col_507   col_508   col_509   col_510   col_511  \n",
       "0     -0.080810  0.016209 -0.011536  0.013828  0.040486  0.051302  0.014021  \n",
       "1     -0.035784  0.050928 -0.014732 -0.041200  0.017684 -0.089092  0.001281  \n",
       "2     -0.015036 -0.095395 -0.002170 -0.000507 -0.061892  0.020786 -0.019073  \n",
       "3     -0.037792  0.022144 -0.045335 -0.020217 -0.001707  0.073547  0.006090  \n",
       "4      0.047688  0.032358  0.024627 -0.033376  0.013414 -0.022359  0.049040  \n",
       "...         ...       ...       ...       ...       ...       ...       ...  \n",
       "10198  0.009265 -0.030309  0.020870  0.053502  0.048767 -0.002036 -0.012913  \n",
       "10199 -0.000184 -0.002363  0.003779 -0.002796  0.112329  0.023464 -0.030824  \n",
       "10200  0.001437  0.017530  0.021713  0.050491 -0.057415 -0.008700 -0.009758  \n",
       "10201  0.004109  0.009871  0.007268  0.058270  0.018326  0.012770  0.049915  \n",
       "10202 -0.033977  0.061615  0.003057  0.069206 -0.045256  0.009401  0.048284  \n",
       "\n",
       "[10203 rows x 512 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ctr_embeds_df_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5921f0c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss \n",
    "\n",
    "start = time.time()\n",
    "\n",
    "vectors = embeddings_ctr_2\n",
    "vec_dim = vectors.shape[1]\n",
    "\n",
    "index = faiss.IndexFlatL2(vec_dim)\n",
    "faiss.normalize_L2(vectors)\n",
    "index.add(vectors)\n",
    "\n",
    "end = time.time()\n",
    "# Total time to compute\n",
    "\n",
    "print('\\nThis function took {:.2f} s to compute.'.format(end - start))"
   ]
  },
  {
   "cell_type": "raw",
   "id": "b56b86f1",
   "metadata": {},
   "source": [
    "# FAISS ran too slow - it killed my kernel twice \n",
    "# Readings off the internet suggest that supporting M1 GPU is neither currently supported nor \n",
    "# planned  and need to add a lot of implementations even if M1 GPU would be supported, because faiss uses CUDA \n",
    "# to implement GPGPU codes and CUDA can't work on M1 GPU."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}