Fb_Friend_ecommendation_Featurization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "168QPRizVFFg"
   },
   "source": [
    "# Facebook Friend Recommendation - Featurization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Q8lS7fVyVFFl"
   },
   "outputs": [],
   "source": [
    "#Importing Libraries\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import csv\n",
    "import pandas as pd\n",
    "import datetime \n",
    "import time \n",
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "from matplotlib import rcParams\n",
    "from sklearn.cluster import MiniBatchKMeans, KMeans\n",
    "import math\n",
    "import pickle\n",
    "import os\n",
    "import xgboost as xgb\n",
    "import warnings\n",
    "import networkx as nx\n",
    "import pdb\n",
    "import pickle\n",
    "from pandas import HDFStore,DataFrame\n",
    "from pandas import read_hdf\n",
    "from scipy.sparse.linalg import svds, eigs\n",
    "import gc\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1znHayNeVFFt"
   },
   "source": [
    "#  Reading Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Uq9HbHwEVFFv",
    "outputId": "b2aa525a-93d3-47c3-8216-416a811bc812"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: \n",
      "Type: DiGraph\n",
      "Number of nodes: 1780722\n",
      "Number of edges: 7550015\n",
      "Average in degree:   4.2399\n",
      "Average out degree:   4.2399\n"
     ]
    }
   ],
   "source": [
    "if os.path.isfile('train_pos_after_eda.csv'):\n",
    "    train_graph=nx.read_edgelist('train_pos_after_eda.csv',delimiter=',',create_using=nx.DiGraph(),nodetype=int)\n",
    "    print(nx.info(train_graph))\n",
    "else:\n",
    "    print(\"please run the FB_EDA.ipynb or download the files from drive\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HmlUa64tVFF7"
   },
   "source": [
    "#  Similarity measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ivVMUMiWVFF9"
   },
   "source": [
    "##  Jaccard Distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NoWCYuRBVFF_"
   },
   "source": [
    "\\begin{equation}\n",
    "j = \\frac{|X\\cap Y|}{|X \\cup Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Seo4z5SnVFGB"
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "def jaccard_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (len(set(train_graph.successors(a)).union(set(train_graph.successors(b)))))\n",
    "    except:\n",
    "        return 0\n",
    "    return sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oa9FMlS8VFGF",
    "outputId": "426a6833-1631-4024-c24a-d21ae7686472"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "#one test case\n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Gf8njOv6VFGK",
    "outputId": "8ba07727-a0ab-498e-819f-0d310876191c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LO-a5ZkKVFGO"
   },
   "outputs": [],
   "source": [
    "#for followers\n",
    "def jaccard_for_followers(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(g.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                 (len(set(train_graph.predecessors(a)).union(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DlbX2t0jVFGQ",
    "outputId": "7e4b4536-442a-4b0c-ae02-fb442c1955db"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(jaccard_for_followers(273084,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OgeBW2LMVFGU",
    "outputId": "1e12fabe-d990-4506-bb6b-c86b01d1b0af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "#node 1635354 not in graph \n",
    "print(jaccard_for_followees(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MnH2my2UVFGX"
   },
   "source": [
    "##  Cosine distance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XNvdBGS2VFGY"
   },
   "source": [
    "\\begin{equation}\n",
    "CosineDistance = \\frac{|X\\cap Y|}{|X|\\cdot|Y|} \n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Iznz67EdVFGZ"
   },
   "outputs": [],
   "source": [
    "#for followees\n",
    "def cosine_for_followees(a,b):\n",
    "    try:\n",
    "        if len(set(train_graph.successors(a))) == 0  | len(set(train_graph.successors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.successors(a)).intersection(set(train_graph.successors(b)))))/\\\n",
    "                                    (math.sqrt(len(set(train_graph.successors(a)))*len((set(train_graph.successors(b))))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "H55ALjkMVFGc",
    "outputId": "531fceba-60f4-4e6b-97f4-f37733dc468f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1505602))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "q0RGKgJFVFGf",
    "outputId": "41202fc6-f4aa-4a1d-d8f6-84f960a3fbba"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followees(273084,1635354))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KJ_yGxA0VFGj"
   },
   "outputs": [],
   "source": [
    "def cosine_for_followers(a,b):\n",
    "    try:\n",
    "        \n",
    "        if len(set(train_graph.predecessors(a))) == 0  | len(set(train_graph.predecessors(b))) == 0:\n",
    "            return 0\n",
    "        sim = (len(set(train_graph.predecessors(a)).intersection(set(train_graph.predecessors(b)))))/\\\n",
    "                                     (math.sqrt(len(set(train_graph.predecessors(a))))*(len(set(train_graph.predecessors(b)))))\n",
    "        return sim\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "75QrFJb6VFGm",
    "outputId": "f01e0558-f1e3-465f-ab14-0e4ca764f4aa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.02886751345948129\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(2,470294))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-ut4k_F0VFGq",
    "outputId": "8bc9607a-7262-43e2-9de8-f71d276762fc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n"
     ]
    }
   ],
   "source": [
    "print(cosine_for_followers(669354,1635354))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DaIHhWh6VFGv"
   },
   "source": [
    "##  Ranking Measures"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "GkkfYYZ6VFGy"
   },
   "source": [
    "##  Page Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AtvqwZ34VFGy"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('page_rank.p'):\n",
    "    pr = nx.pagerank(train_graph, alpha=0.85)\n",
    "    pickle.dump(pr,open('page_rank.p','wb'))\n",
    "else:\n",
    "    pr = pickle.load(open('page_rank.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lXGKYYf6VFG2",
    "outputId": "bb3d1b7a-81f9-44ab-dbe7-3214ccd47179"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 1.6556497245737814e-07\n",
      "max 2.7098251341935827e-05\n",
      "mean 5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "print('min',pr[min(pr, key=pr.get)])\n",
    "print('max',pr[max(pr, key=pr.get)])\n",
    "print('mean',float(sum(pr.values())) / len(pr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5xwlah4oVFG4",
    "outputId": "992fdfad-7ff6-4626-c9ee-d9bce220a680"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5.615699699389075e-07\n"
     ]
    }
   ],
   "source": [
    "#for imputing to nodes which are not there in Train data\n",
    "mean_pr = float(sum(pr.values())) / len(pr)\n",
    "print(mean_pr)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HhPbSL1tVFG7"
   },
   "source": [
    "#  Other Graph Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AgsorCl7VFG8"
   },
   "source": [
    "##  Shortest path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RA076ovzVFG9"
   },
   "outputs": [],
   "source": [
    "#if has direct edge then deleting that edge and calculating shortest path\n",
    "def compute_shortest_path_length(a,b):\n",
    "    p=-1\n",
    "    try:\n",
    "        if train_graph.has_edge(a,b):\n",
    "            train_graph.remove_edge(a,b)\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "            train_graph.add_edge(a,b)\n",
    "        else:\n",
    "            p= nx.shortest_path_length(train_graph,source=a,target=b)\n",
    "        return p\n",
    "    except:\n",
    "        return -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AxnKId11VFG_",
    "outputId": "15ca223a-6a04-4549-d010-54619b472a9e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(77697, 826021)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0huWCNtRVFHC",
    "outputId": "6debfa4f-2067-48bc-84b3-ab86e2d9dea6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-1"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#testing\n",
    "compute_shortest_path_length(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "baE_95bzVFHF"
   },
   "source": [
    "##  Checking for same community"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "15CIQqAbVFHG"
   },
   "outputs": [],
   "source": [
    "#getting weekly connected edges from graph \n",
    "wcc=list(nx.weakly_connected_components(train_graph))\n",
    "def belongs_to_same_wcc(a,b):\n",
    "    index = []\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    if train_graph.has_edge(a,b):\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if (b in index):\n",
    "                train_graph.remove_edge(a,b)\n",
    "                if compute_shortest_path_length(a,b)==-1:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 0\n",
    "                else:\n",
    "                    train_graph.add_edge(a,b)\n",
    "                    return 1\n",
    "            else:\n",
    "                return 0\n",
    "    else:\n",
    "            for i in wcc:\n",
    "                if a in i:\n",
    "                    index= i\n",
    "                    break\n",
    "            if(b in index):\n",
    "                return 1\n",
    "            else:\n",
    "                return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fAzOHtCFVFHI",
    "outputId": "2b043a87-b460-42bf-f37e-4c04bbed6586"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(861, 1659750)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "HMdYpPuGVFHK",
    "outputId": "2005e22c-b60f-48d7-839b-650bf97cae35"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "belongs_to_same_wcc(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "q74nth0OVFHN"
   },
   "source": [
    "##  Adamic/Adar Index:\n",
    "$$A(x,y)=\\sum_{u \\in N(x) \\cap N(y)}\\frac{1}{log(|N(u)|)}$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CeS98LI5VFHO"
   },
   "outputs": [],
   "source": [
    "#adar index\n",
    "def calc_adar_in(a,b):\n",
    "    sum=0\n",
    "    try:\n",
    "        n=list(set(train_graph.successors(a)).intersection(set(train_graph.successors(b))))\n",
    "        if len(n)!=0:\n",
    "            for i in n:\n",
    "                sum=sum+(1/np.log10(len(list(train_graph.predecessors(i)))))\n",
    "            return sum\n",
    "        else:\n",
    "            return 0\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KezFeRmyVFHQ",
    "outputId": "2f9c0e11-02d9-4f28-d67a-65e3d4943e99"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vj_m89bBVFHV",
    "outputId": "68a0a099-2954-402f-c80f-6d436ffa1aba"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_adar_in(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "pBUudhFAVFHY"
   },
   "source": [
    "##  Is persion was following back:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j_mwmopLVFHZ"
   },
   "outputs": [],
   "source": [
    "def follows_back(a,b):\n",
    "    if train_graph.has_edge(b,a):\n",
    "        return 1\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LdjUXIfbVFHb",
    "outputId": "ed3d8640-9834-4a95-e712-804292da70e9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(1,189226)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PmZtL65YVFHf",
    "outputId": "18ea6fe2-3f96-42c0-d116-ecb76ddba4b5"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "follows_back(669354,1635354)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "29Vrq2EXVFHi"
   },
   "source": [
    "##  Katz Centrality\n",
    "\n",
    " \n",
    "$$x_i = \\alpha \\sum_{j} A_{ij} x_j + \\beta,$$\n",
    "where `A` is the adjacency matrix of the graph G \n",
    "with eigenvalues $$\\lambda$$.\n",
    "The parameter $$\\beta$$ controls the initial centrality and \n",
    "$$\\alpha < \\frac{1}{\\lambda_{max}}.$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CN5OSqrkVFHj"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('katz.p'):\n",
    "    katz = nx.katz.katz_centrality(train_graph,alpha=0.005,beta=1)\n",
    "    pickle.dump(katz,open('katz.p','wb'))\n",
    "else:\n",
    "    katz = pickle.load(open('katz.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gcU83vw7VFHm",
    "outputId": "05f49ad4-46fe-4cf6-f32a-2fe4846b0714"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0007313532484065916\n",
      "max 0.003394554981699122\n",
      "mean 0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "print('min',katz[min(katz, key=katz.get)])\n",
    "print('max',katz[max(katz, key=katz.get)])\n",
    "print('mean',float(sum(katz.values())) / len(katz))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qcboIksiVFHt",
    "outputId": "99f52422-9edb-479a-d5d9-e33401160da7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0007483800935562018\n"
     ]
    }
   ],
   "source": [
    "mean_katz = float(sum(katz.values())) / len(katz)\n",
    "print(mean_katz)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SRZqGFgYVFHx"
   },
   "source": [
    "##  Hits Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WXNHRdzUVFHz"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('hits.p'):\n",
    "    hits = nx.hits(train_graph, max_iter=100, tol=1e-08, nstart=None, normalized=True)\n",
    "    pickle.dump(hits,open('hits.p','wb'))\n",
    "else:\n",
    "    hits = pickle.load(open('hits.p','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PSUwSZBVVFH3",
    "outputId": "77448253-5409-4229-f0be-b8dbc14d7f46"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "min 0.0\n",
      "max 0.004868653378780953\n",
      "mean 5.615699699344123e-07\n"
     ]
    }
   ],
   "source": [
    "print('min',hits[0][min(hits[0], key=hits[0].get)])\n",
    "print('max',hits[0][max(hits[0], key=hits[0].get)])\n",
    "print('mean',float(sum(hits[0].values())) / len(hits[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculating Preferential Attachment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def follower_pref_att(a,b):\n",
    "    try:\n",
    "        return len(set(train_graph.predecessors(a))) * len(set(train_graph.predecessors(b)))\n",
    "    except:\n",
    "        return 0\n",
    "    \n",
    "    \n",
    "def followee_pref_att(a,b):\n",
    "    try:\n",
    "        return len(set(train_graph.successors(a))) * len(set(train_graph.successors(b)))\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculating svd_dot "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference - https://github.com/shashimanyam/FACEBOOK-FRIEND-RECOMMENDATION/blob/master/fb_frnd_recommendation.ipynb\n",
    "\n",
    "def svd_dot_u(node):\n",
    "    try:\n",
    "        s_node = node[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']]\n",
    "        d_node = node[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']]\n",
    "        return np.dot(s_node, d_node)\n",
    "    except:\n",
    "        return 0\n",
    "    \n",
    "    \n",
    "def svd_dot_v(node):\n",
    "    try:\n",
    "        s_node = node[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]]\n",
    "        d_node = node[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']]\n",
    "        return np.dot(s_node, d_node)\n",
    "    except:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZZtowOLZVFH6"
   },
   "source": [
    "#  Featurization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "o6NnRWmLVFH6"
   },
   "source": [
    "##  Reading a sample of Data from both train and test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wgHje1UVVFH8"
   },
   "outputs": [],
   "source": [
    "import random\n",
    "if os.path.isfile('train_after_eda.csv'):\n",
    "    filename = \"train_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 15100030\n",
    "    # n_train = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_train =  15100028\n",
    "    s = 100000 #desired sample size\n",
    "    skip_train = sorted(random.sample(range(1,n_train+1),n_train-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zOzuRFFlVFH-"
   },
   "outputs": [],
   "source": [
    "if os.path.isfile('train_after_eda.csv'):\n",
    "    filename = \"test_after_eda.csv\"\n",
    "    # you uncomment this line, if you dont know the lentgh of the file name\n",
    "    # here we have hardcoded the number of lines as 3775008\n",
    "    # n_test = sum(1 for line in open(filename)) #number of records in file (excludes header)\n",
    "    n_test = 3775006\n",
    "    s = 50000 #desired sample size\n",
    "    skip_test = sorted(random.sample(range(1,n_test+1),n_test-s))\n",
    "    #https://stackoverflow.com/a/22259008/4084039"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3D_SeUCOVFH_",
    "outputId": "322902a4-0420-4b99-8606-5fd0de4bbea4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of rows in the train data file: 15100028\n",
      "Number of rows we are going to elimiate in train data are 15000028\n",
      "Number of rows in the test data file: 3775006\n",
      "Number of rows we are going to elimiate in test data are 3725006\n"
     ]
    }
   ],
   "source": [
    "print(\"Number of rows in the train data file:\", n_train)\n",
    "print(\"Number of rows we are going to elimiate in train data are\",len(skip_train))\n",
    "print(\"Number of rows in the test data file:\", n_test)\n",
    "print(\"Number of rows we are going to elimiate in test data are\",len(skip_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pCisf6PpVFID",
    "outputId": "daf2af43-3f98-4466-ad99-03bc54464714"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our train matrix size  (100002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>273084</td>\n",
       "      <td>1505602</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1580187</td>\n",
       "      <td>612293</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       273084           1505602               1\n",
       "1      1580187            612293               1"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_train = pd.read_csv('train_after_eda.csv', skiprows=skip_train, names=['source_node', 'destination_node'])\n",
    "df_final_train['indicator_link'] = pd.read_csv('train_y.csv', skiprows=skip_train, names=['indicator_link'])\n",
    "print(\"Our train matrix size \",df_final_train.shape)\n",
    "df_final_train.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tFn1RkdyVFIH",
    "outputId": "1ca99e70-6d2a-45f2-f51c-fd3b1211ad20"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our test matrix size  (50002, 3)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source_node</th>\n",
       "      <th>destination_node</th>\n",
       "      <th>indicator_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>848424</td>\n",
       "      <td>784690</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>539301</td>\n",
       "      <td>164255</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   source_node  destination_node  indicator_link\n",
       "0       848424            784690               1\n",
       "1       539301            164255               1"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_final_test = pd.read_csv('test_after_eda.csv', skiprows=skip_test, names=['source_node', 'destination_node'])\n",
    "df_final_test['indicator_link'] = pd.read_csv('test_y.csv', skiprows=skip_test, names=['indicator_link'])\n",
    "print(\"Our test matrix size \",df_final_test.shape)\n",
    "df_final_test.head(2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gIaOWDaDVFIJ"
   },
   "source": [
    "##  Adding a set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>jaccard_followers</li>\n",
    "<li>jaccard_followees</li>\n",
    "<li>cosine_followers</li>\n",
    "<li>cosine_followees</li>\n",
    "<li>num_followers_s</li>\n",
    "<li>num_followees_s</li>\n",
    "<li>num_followers_d</li>\n",
    "<li>num_followees_d</li>\n",
    "<li>inter_followers</li>\n",
    "<li>inter_followees</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2qTkOiBcVFIJ"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage1.h5'):\n",
    "    #mapping jaccrd followers to train and test data\n",
    "    df_final_train['jaccard_followers'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followers'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['jaccard_followees'] = df_final_train.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['jaccard_followees'] = df_final_test.apply(lambda row:\n",
    "                                            jaccard_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "\n",
    "        #mapping jaccrd followers to train and test data\n",
    "    df_final_train['cosine_followers'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followers'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followers(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping jaccrd followees to train and test data\n",
    "    df_final_train['cosine_followees'] = df_final_train.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)\n",
    "    df_final_test['cosine_followees'] = df_final_test.apply(lambda row:\n",
    "                                            cosine_for_followees(row['source_node'],row['destination_node']),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fz2eZpSnVFIL"
   },
   "outputs": [],
   "source": [
    "def compute_features_stage1(df_final):\n",
    "    #calculating no of followers followees for source and destination\n",
    "    #calculating intersection of followers and followees for source and destination\n",
    "    num_followers_s=[]\n",
    "    num_followees_s=[]\n",
    "    num_followers_d=[]\n",
    "    num_followees_d=[]\n",
    "    inter_followers=[]\n",
    "    inter_followees=[]\n",
    "    for i,row in df_final.iterrows():\n",
    "        try:\n",
    "            s1=set(train_graph.predecessors(row['source_node']))\n",
    "            s2=set(train_graph.successors(row['source_node']))\n",
    "        except:\n",
    "            s1 = set()\n",
    "            s2 = set()\n",
    "        try:\n",
    "            d1=set(train_graph.predecessors(row['destination_node']))\n",
    "            d2=set(train_graph.successors(row['destination_node']))\n",
    "        except:\n",
    "            d1 = set()\n",
    "            d2 = set()\n",
    "        num_followers_s.append(len(s1))\n",
    "        num_followees_s.append(len(s2))\n",
    "\n",
    "        num_followers_d.append(len(d1))\n",
    "        num_followees_d.append(len(d2))\n",
    "\n",
    "        inter_followers.append(len(s1.intersection(d1)))\n",
    "        inter_followees.append(len(s2.intersection(d2)))\n",
    "    \n",
    "    return num_followers_s, num_followers_d, num_followees_s, num_followees_d, inter_followers, inter_followees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VFc60kcRVFIN"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage1.h5'):\n",
    "    df_final_train['num_followers_s'], df_final_train['num_followers_d'], \\\n",
    "    df_final_train['num_followees_s'], df_final_train['num_followees_d'], \\\n",
    "    df_final_train['inter_followers'], df_final_train['inter_followees']= compute_features_stage1(df_final_train)\n",
    "    \n",
    "    df_final_test['num_followers_s'], df_final_test['num_followers_d'], \\\n",
    "    df_final_test['num_followees_s'], df_final_test['num_followees_d'], \\\n",
    "    df_final_test['inter_followers'], df_final_test['inter_followees']= compute_features_stage1(df_final_test)\n",
    "    \n",
    "    hdf = HDFStore('storage_sample_stage1.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('storage_sample_stage1.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('storage_sample_stage1.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "go_e8hxxVFIO"
   },
   "source": [
    "## 5.3 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>adar index</li>\n",
    "<li>is following back</li>\n",
    "<li>belongs to same weakly connect components</li>\n",
    "<li>shortest path between source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LqB0Peg0VFIP"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage2.h5'):\n",
    "    #mapping adar index on train\n",
    "    df_final_train['adar_index'] = df_final_train.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping adar index on test\n",
    "    df_final_test['adar_index'] = df_final_test.apply(lambda row: calc_adar_in(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping followback or not on train\n",
    "    df_final_train['follows_back'] = df_final_train.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #mapping followback or not on test\n",
    "    df_final_test['follows_back'] = df_final_test.apply(lambda row: follows_back(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping same component of wcc or not on train\n",
    "    df_final_train['same_comp'] = df_final_train.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    ##mapping same component of wcc or not on train\n",
    "    df_final_test['same_comp'] = df_final_test.apply(lambda row: belongs_to_same_wcc(row['source_node'],row['destination_node']),axis=1)\n",
    "    \n",
    "    #--------------------------------------------------------------------------------------------------------\n",
    "    #mapping shortest path on train \n",
    "    df_final_train['shortest_path'] = df_final_train.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "    #mapping shortest path on test\n",
    "    df_final_test['shortest_path'] = df_final_test.apply(lambda row: compute_shortest_path_length(row['source_node'],row['destination_node']),axis=1)\n",
    "\n",
    "    hdf = HDFStore('storage_sample_stage2.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('storage_sample_stage2.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('storage_sample_stage2.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HJ8Dbma_VFIR"
   },
   "source": [
    "## 5.4 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>Weight Features\n",
    "    <ul>\n",
    "        <li>weight of incoming edges</li>\n",
    "        <li>weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges * weight of outgoing edges</li>\n",
    "        <li>2*weight of incoming edges + weight of outgoing edges</li>\n",
    "        <li>weight of incoming edges + 2*weight of outgoing edges</li>\n",
    "    </ul>\n",
    "</li>\n",
    "<li>Page Ranking of source</li>\n",
    "<li>Page Ranking of dest</li>\n",
    "<li>katz of source</li>\n",
    "<li>katz of dest</li>\n",
    "<li>hubs of source</li>\n",
    "<li>hubs of dest</li>\n",
    "<li>authorities_s of source</li>\n",
    "<li>authorities_s of dest</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iVHI2jtNVFIS"
   },
   "source": [
    "#### Weight Features"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rXmUYF9FVFIT"
   },
   "source": [
    "In order to determine the similarity of nodes, an edge weight value was calculated between nodes. Edge weight decreases as the neighbor count goes up. Intuitively, consider one million people following a celebrity on a social network then chances are most of them never met each other or the celebrity. On the other hand, if a user has 30 contacts in his/her social network, the chances are higher that many of them know each other. \n",
    "`credit` - Graph-based Features for Supervised Link Prediction\n",
    "William Cukierski, Benjamin Hamner, Bo Yang"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Qzbs2no7VFIV"
   },
   "source": [
    "\\begin{equation}\n",
    "W = \\frac{1}{\\sqrt{1+|X|}}\n",
    "\\end{equation}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kkzUPrWaVFIV"
   },
   "source": [
    "it is directed graph so calculated Weighted in and Weighted out differently"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "FgNMzzTbVFIW",
    "outputId": "7e8e6d88-8bd6-45f6-f80e-82b093c18974"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████| 1780722/1780722 [00:15<00:00, 111989.96it/s]\n"
     ]
    }
   ],
   "source": [
    "#weight for source and destination of each link\n",
    "Weight_in = {}\n",
    "Weight_out = {}\n",
    "for i in  tqdm(train_graph.nodes()):\n",
    "    s1=set(train_graph.predecessors(i))\n",
    "    w_in = 1.0/(np.sqrt(1+len(s1)))\n",
    "    Weight_in[i]=w_in\n",
    "    \n",
    "    s2=set(train_graph.successors(i))\n",
    "    w_out = 1.0/(np.sqrt(1+len(s2)))\n",
    "    Weight_out[i]=w_out\n",
    "    \n",
    "#for imputing with mean\n",
    "mean_weight_in = np.mean(list(Weight_in.values()))\n",
    "mean_weight_out = np.mean(list(Weight_out.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AF4yPhIOVFIY"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage3.h5'):\n",
    "    #mapping to pandas train\n",
    "    df_final_train['weight_in'] = df_final_train.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_train['weight_out'] = df_final_train.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "    #mapping to pandas test\n",
    "    df_final_test['weight_in'] = df_final_test.destination_node.apply(lambda x: Weight_in.get(x,mean_weight_in))\n",
    "    df_final_test['weight_out'] = df_final_test.source_node.apply(lambda x: Weight_out.get(x,mean_weight_out))\n",
    "\n",
    "\n",
    "    #some features engineerings on the in and out weights\n",
    "    df_final_train['weight_f1'] = df_final_train.weight_in + df_final_train.weight_out\n",
    "    df_final_train['weight_f2'] = df_final_train.weight_in * df_final_train.weight_out\n",
    "    df_final_train['weight_f3'] = (2*df_final_train.weight_in + 1*df_final_train.weight_out)\n",
    "    df_final_train['weight_f4'] = (1*df_final_train.weight_in + 2*df_final_train.weight_out)\n",
    "\n",
    "    #some features engineerings on the in and out weights\n",
    "    df_final_test['weight_f1'] = df_final_test.weight_in + df_final_test.weight_out\n",
    "    df_final_test['weight_f2'] = df_final_test.weight_in * df_final_test.weight_out\n",
    "    df_final_test['weight_f3'] = (2*df_final_test.weight_in + 1*df_final_test.weight_out)\n",
    "    df_final_test['weight_f4'] = (1*df_final_test.weight_in + 2*df_final_test.weight_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uhxzhQ9aVFIa"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage3.h5'):\n",
    "    \n",
    "    #page rank for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding mean page rank \n",
    "    df_final_train['page_rank_s'] = df_final_train.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_train['page_rank_d'] = df_final_train.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "\n",
    "    df_final_test['page_rank_s'] = df_final_test.source_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    df_final_test['page_rank_d'] = df_final_test.destination_node.apply(lambda x:pr.get(x,mean_pr))\n",
    "    #================================================================================\n",
    "\n",
    "    #Katz centrality score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding mean katz score\n",
    "    df_final_train['katz_s'] = df_final_train.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_train['katz_d'] = df_final_train.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "\n",
    "    df_final_test['katz_s'] = df_final_test.source_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    df_final_test['katz_d'] = df_final_test.destination_node.apply(lambda x: katz.get(x,mean_katz))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['hubs_s'] = df_final_train.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_train['hubs_d'] = df_final_train.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "\n",
    "    df_final_test['hubs_s'] = df_final_test.source_node.apply(lambda x: hits[0].get(x,0))\n",
    "    df_final_test['hubs_d'] = df_final_test.destination_node.apply(lambda x: hits[0].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    #Hits algorithm score for source and destination in Train and Test\n",
    "    #if anything not there in train graph then adding 0\n",
    "    df_final_train['authorities_s'] = df_final_train.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_train['authorities_d'] = df_final_train.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "\n",
    "    df_final_test['authorities_s'] = df_final_test.source_node.apply(lambda x: hits[1].get(x,0))\n",
    "    df_final_test['authorities_d'] = df_final_test.destination_node.apply(lambda x: hits[1].get(x,0))\n",
    "    #================================================================================\n",
    "\n",
    "    hdf = HDFStore('storage_sample_stage3.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()\n",
    "else:\n",
    "    df_final_train = read_hdf('storage_sample_stage3.h5', 'train_df',mode='r')\n",
    "    df_final_test = read_hdf('storage_sample_stage3.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "p6xkDfD-VFIb"
   },
   "source": [
    "## 5.5 Adding new set of features\n",
    "\n",
    "__we will create these each of these features for both train and test data points__\n",
    "<ol>\n",
    "<li>SVD features for both source and destination</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WQO6E65eVFIc"
   },
   "outputs": [],
   "source": [
    "def svd(x, S):\n",
    "    try:\n",
    "        z = sadj_dict[x]\n",
    "        return S[z]\n",
    "    except:\n",
    "        return [0,0,0,0,0,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9sOyLwvNVFId"
   },
   "outputs": [],
   "source": [
    "#for svd features to get feature vector creating a dict node val and inedx in svd vector\n",
    "sadj_col = sorted(train_graph.nodes())\n",
    "sadj_dict = { val:idx for idx,val in enumerate(sadj_col)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zLSt8fGVVFIg"
   },
   "outputs": [],
   "source": [
    "Adj = nx.adjacency_matrix(train_graph,nodelist=sorted(train_graph.nodes())).asfptype()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "soq-VAHlVFIh",
    "outputId": "3f9bfb32-004f-4698-e415-469243250130"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Adjacency matrix Shape (1780722, 1780722)\n",
      "U Shape (1780722, 6)\n",
      "V Shape (6, 1780722)\n",
      "s Shape (6,)\n"
     ]
    }
   ],
   "source": [
    "U, s, V = svds(Adj, k = 6)\n",
    "print('Adjacency matrix Shape',Adj.shape)\n",
    "print('U Shape',U.shape)\n",
    "print('V Shape',V.shape)\n",
    "print('s Shape',s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ls5fqLFhVFIm"
   },
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage4.h5'):\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_train[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_train.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_train[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_train.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_u_s_1', 'svd_u_s_2','svd_u_s_3', 'svd_u_s_4', 'svd_u_s_5', 'svd_u_s_6']] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "    \n",
    "    df_final_test[['svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3', 'svd_u_d_4', 'svd_u_d_5','svd_u_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, U)).apply(pd.Series)\n",
    "\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_test[['svd_v_s_1','svd_v_s_2', 'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6',]] = \\\n",
    "    df_final_test.source_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "\n",
    "    df_final_test[['svd_v_d_1', 'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5','svd_v_d_6']] = \\\n",
    "    df_final_test.destination_node.apply(lambda x: svd(x, V.T)).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "\n",
    "    hdf = HDFStore('storage_sample_stage4.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_train = read_hdf('storage_sample_stage4.h5', 'train_df',mode='r')\n",
    "df_final_test = read_hdf('storage_sample_stage4.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.isfile('storage_sample_stage5.h5'):\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train['follower_pref'] = df_final_train.apply(lambda row: follower_pref_att(row['source_node'],row['destination_node']),axis=1).apply(pd.Series)\n",
    "    df_final_test['follower_pref'] = df_final_test.apply(lambda row: follower_pref_att(row['source_node'],row['destination_node']),axis=1).apply(pd.Series)\n",
    "\n",
    "\n",
    "    df_final_train['followee_pref'] = df_final_train.apply(lambda row: followee_pref_att(row['source_node'],row['destination_node']),axis=1).apply(pd.Series)\n",
    "    df_final_test['followee_pref'] = df_final_test.apply(lambda row: followee_pref_att(row['source_node'],row['destination_node']),axis=1).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    df_final_train['svd_dot_u'] = df_final_train.apply(lambda row:svd_dot_u(row),axis=1).apply(pd.Series)\n",
    "    df_final_train['svd_dot_v'] = df_final_train.apply(lambda row:svd_dot_v(row),axis=1).apply(pd.Series)\n",
    "\n",
    "\n",
    "    df_final_test['svd_dot_u'] = df_final_test.apply(lambda row:svd_dot_u(row),axis=1).apply(pd.Series)\n",
    "    df_final_test['svd_dot_v'] = df_final_test.apply(lambda row:svd_dot_v(row),axis=1).apply(pd.Series)\n",
    "    #===================================================================================================\n",
    "    \n",
    "    hdf = HDFStore('storage_sample_stage5.h5')\n",
    "    hdf.put('train_df',df_final_train, format='table', data_columns=True)\n",
    "    hdf.put('test_df',df_final_test, format='table', data_columns=True)\n",
    "    hdf.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_final_train = read_hdf('storage_sample_stage5.h5', 'train_df',mode='r')\n",
    "df_final_test = read_hdf('storage_sample_stage5.h5', 'test_df',mode='r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['source_node', 'destination_node', 'indicator_link',\n",
      "       'jaccard_followers', 'jaccard_followees', 'cosine_followers',\n",
      "       'cosine_followees', 'num_followers_s', 'num_followees_s',\n",
      "       'num_followees_d', 'inter_followers', 'inter_followees', 'adar_index',\n",
      "       'follows_back', 'same_comp', 'shortest_path', 'weight_in', 'weight_out',\n",
      "       'weight_f1', 'weight_f2', 'weight_f3', 'weight_f4', 'page_rank_s',\n",
      "       'page_rank_d', 'katz_s', 'katz_d', 'hubs_s', 'hubs_d', 'authorities_s',\n",
      "       'authorities_d', 'svd_u_s_1', 'svd_u_s_2', 'svd_u_s_3', 'svd_u_s_4',\n",
      "       'svd_u_s_5', 'svd_u_s_6', 'svd_u_d_1', 'svd_u_d_2', 'svd_u_d_3',\n",
      "       'svd_u_d_4', 'svd_u_d_5', 'svd_u_d_6', 'svd_v_s_1', 'svd_v_s_2',\n",
      "       'svd_v_s_3', 'svd_v_s_4', 'svd_v_s_5', 'svd_v_s_6', 'svd_v_d_1',\n",
      "       'svd_v_d_2', 'svd_v_d_3', 'svd_v_d_4', 'svd_v_d_5', 'svd_v_d_6',\n",
      "       'follower_pref', 'followee_pref', 'svd_dot_u', 'svd_dot_v'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "# Printing Features names\n",
    "\n",
    "print(df_final_train.columns)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [
    "DaIHhWh6VFGv",
    "GkkfYYZ6VFGy",
    "AgsorCl7VFG8",
    "baE_95bzVFHF",
    "pBUudhFAVFHY",
    "29Vrq2EXVFHi",
    "SRZqGFgYVFHx"
   ],
   "name": "FB_featurization.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}