07_pure_kgrec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "ml_path = './learning_data/'\n",
    "sedkgraph_path = './sedkgraph_data/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(ml_path+'y_test.pkl','rb') as f:\n",
    "    y_test = pickle.load(f)\n",
    "with open(ml_path+'topics.pkl','rb') as f:\n",
    "    topics = pickle.load(f)\n",
    "    \n",
    "sedkgraph = pd.read_csv(sedkgraph_path+'sedkgraph.csv')\n",
    "topics_numeric = pd.read_csv(sedkgraph_path+'topics_numerical_data.csv')\n",
    "test_samples_count = y_test.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Calculate simplet rate =\n",
    "### 0.5*(popularity_softened+rel_count_softened)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "topics_numeric['rate'] = 0.5*(topics_numeric['popularity_softened']+topics_numeric['rel_count_softened'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper function to get the previously assigned topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_assigned_topic_names(assigned, topics=topics):\n",
    "    indexes = list(np.where(assigned > 0)[0])\n",
    "    result = sorted([(topics[x],assigned[x]) for x in indexes], key=lambda e: e[1], reverse=True)\n",
    "    return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_kgrec = 5\n",
    "k_saved_files = 5"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper functions to get KGRec recommendations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude_from_recom = ['programming-language']\n",
    "\n",
    "flatten = lambda x: [item for sublist in x for item in sublist]\n",
    "\n",
    "def get_recom_scores(already_assigned,candidates, topics_numeric=topics_numeric, exclude_from_recom=exclude_from_recom):\n",
    "    candidates_list = list(set(flatten(candidates.values())).difference(set(exclude_from_recom)))\n",
    "    result = []\n",
    "    \n",
    "    for candidate in candidates_list:\n",
    "        score = 0\n",
    "        for recom,candidate_sublist in candidates.items():\n",
    "            if candidate in candidate_sublist:\n",
    "                score += already_assigned[recom]\n",
    "        score *= float(topics_numeric[topics_numeric.topic==candidate]['rate'])\n",
    "        \n",
    "        result.append((candidate,float(score)))\n",
    "    \n",
    "    result = sorted(result, key=lambda x: x[1], reverse=True)\n",
    "    try:\n",
    "        max_score = result[0][1]\n",
    "        result = list(map(lambda x: (x[0],x[1]/max_score), result))\n",
    "    except IndexError as e:\n",
    "        pass\n",
    "    \n",
    "    return result\n",
    "\n",
    "def get_kgrec_recoms(already_assigned, k, sedkgraph=sedkgraph, topics_numeric=topics_numeric):\n",
    "    already_assigned = dict(map(lambda x: (x[0],float(x[1])), already_assigned))\n",
    "    already_assigned_list = list(already_assigned.keys())\n",
    "    \n",
    "    candidates = {}\n",
    "    for recom in already_assigned_list:\n",
    "        if already_assigned[recom] == 0:\n",
    "            break\n",
    "        tmp_list = list(sedkgraph[sedkgraph.rhs==recom].lhs.values)+list(sedkgraph[sedkgraph.lhs==recom].rhs.values)\n",
    "        candidates[recom] = list(set(tmp_list).difference(set(already_assigned_list)))\n",
    "    \n",
    "    recom_scores = get_recom_scores(already_assigned,candidates)\n",
    "    \n",
    "    if len(recom_scores)<k:\n",
    "        recom_scores = recom_scores + [('no-more-recoms',0)]*(k-len(recom_scores))\n",
    "    \n",
    "    return np.array(recom_scores)[:k]\n",
    "\n",
    "def prep_already_assigned(already_assigned_array):\n",
    "    max_rec_len = max(list(map(lambda x: len(x), already_assigned_array)))\n",
    "    result = list(map(lambda x: x+[('no-more-recoms',0)]*(max_rec_len-len(x)),already_assigned_array))\n",
    "    return np.array(result)\n",
    "\n",
    "def vectorized_get_kgrec_recoms(already_assigned_array, k, sedkgraph=sedkgraph, topics_numeric=topics_numeric):\n",
    "    if not isinstance(already_assigned_array,np.ndarray):\n",
    "        already_assigned_array = prep_already_assigned(already_assigned_array)\n",
    "    shape = already_assigned_array.shape\n",
    "    results = np.apply_along_axis(lambda x: get_kgrec_recoms(x.reshape(shape[1],shape[2]), k),\n",
    "                                  1, already_assigned_array.reshape(shape[0],-1))\n",
    "    return np.array(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Results from KGRec (Top-k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "assigned_topics = list(map(lambda x: get_assigned_topic_names(x), y_test))\n",
    "kgrec_recoms = vectorized_get_kgrec_recoms(assigned_topics, k_kgrec)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Report on Failures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "KGRec failed to make 5 recommendations for 7.54% of the 29050 test repositoried.\n",
      "\n",
      "KGRec made NO recommendations for 2.06% of the 29050 test repositoried.\n"
     ]
    }
   ],
   "source": [
    "failed_k = []\n",
    "failed_0 = []\n",
    "\n",
    "for i in range(kgrec_recoms.shape[0]):\n",
    "    recoms = list(set(map(lambda x: x[0] ,kgrec_recoms[i])))\n",
    "    if 'no-more-recoms' in recoms:\n",
    "        failed_k.append(i)\n",
    "        if len(recoms)==1:\n",
    "            failed_0.append(i)\n",
    "\n",
    "print(f'\\nKGRec failed to make {k_kgrec} recommendations for {np.round(len(failed_k)*100/test_samples_count, 2)}% of the {test_samples_count} test repositoried.')\n",
    "print(f'\\nKGRec made NO recommendations for {np.round(len(failed_0)*100/test_samples_count, 2)}% of the {test_samples_count} test repositoried.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "    os.makedirs('./results/')\n",
    "except:\n",
    "    pass\n",
    "\n",
    "with open(f'./results/pure_kgrec_top{k_saved_files}_test_results.pkl','wb') as f:\n",
    "    pickle.dump(kgrec_recoms, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}