Automated Relevancy Tuning using Black Box Optimization Algorithms.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#solrpy library: http://pythonhosted.org/solrpy/overview.html\n",
    "import solr #to install: pip install solrpy\n",
    "#pandas library for data processing - only needed to index the solr core, can be removed otherwise\n",
    "import pandas as pd #to install: pip install pandas\n",
    "#scikit-optimize library: https://github.com/scikit-optimize\n",
    "import skopt #to install: pip install scikit-optimize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Settings\n",
    "\n",
    "# The files below are in the root folder of this GitHub repo. Launch jupyter notebook from that folder\n",
    "# in order to read these files: 'jupyter notebook'\n",
    "\n",
    "# Note: this is an artificial set of jobs, these are not real jobs, but are representative of our data\n",
    "# Job descriptions are omitted, but usually we search that field also\n",
    "jobs_data_file = \"jobs.csv\"\n",
    "\n",
    "# File of relevancy judgements - these are highly subjective judgements, please don't take them too seriously\n",
    "relevancy_file = \"relevancy_judegements.csv\"\n",
    "\n",
    "#solr url and core (Jobs)\n",
    "solr_url = \"http://localhost:8983/solr/Jobs\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Jobs Data, Index in Solr Jobs Core"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>jobTitle</th>\n",
       "      <th>jobSkills</th>\n",
       "      <th>employer</th>\n",
       "      <th>city</th>\n",
       "      <th>state</th>\n",
       "      <th>geoCode</th>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Lead Developer</td>\n",
       "      <td>[Project management, Java, Programming, QA]</td>\n",
       "      <td>IT Services and Networking Corp.</td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "      <td>40.7127837,-74.0059413</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Cloud Developer</td>\n",
       "      <td>[QA, Software engineering, Compiler, Network, ...</td>\n",
       "      <td>Large Search Giant Llc.</td>\n",
       "      <td>Des Moines</td>\n",
       "      <td>IA</td>\n",
       "      <td>41.6005448,-93.6091064</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Application Developer</td>\n",
       "      <td>[J2EE, Oracle, XML, QA, jQuery, JDBC, BIND, IBM]</td>\n",
       "      <td>Acme Inc</td>\n",
       "      <td>Des Moines</td>\n",
       "      <td>IA</td>\n",
       "      <td>41.6005448,-93.6091064</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Application Developer</td>\n",
       "      <td>[Programming, Lifecycle management, Network]</td>\n",
       "      <td>IT Services and Networking Corp.</td>\n",
       "      <td>Chicago</td>\n",
       "      <td>IL</td>\n",
       "      <td>41.8781136,-87.6297982</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pega Developer</td>\n",
       "      <td>[QA, Agile, Architecture]</td>\n",
       "      <td>Scientists and Quants Inc</td>\n",
       "      <td>New York</td>\n",
       "      <td>NY</td>\n",
       "      <td>40.7127837,-74.0059413</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                jobTitle                                          jobSkills  \\\n",
       "0         Lead Developer        [Project management, Java, Programming, QA]   \n",
       "1        Cloud Developer  [QA, Software engineering, Compiler, Network, ...   \n",
       "2  Application Developer   [J2EE, Oracle, XML, QA, jQuery, JDBC, BIND, IBM]   \n",
       "3  Application Developer       [Programming, Lifecycle management, Network]   \n",
       "4         Pega Developer                          [QA, Agile, Architecture]   \n",
       "\n",
       "                           employer        city state                 geoCode  \\\n",
       "0  IT Services and Networking Corp.    New York    NY  40.7127837,-74.0059413   \n",
       "1           Large Search Giant Llc.  Des Moines    IA  41.6005448,-93.6091064   \n",
       "2                          Acme Inc  Des Moines    IA  41.6005448,-93.6091064   \n",
       "3  IT Services and Networking Corp.     Chicago    IL  41.8781136,-87.6297982   \n",
       "4         Scientists and Quants Inc    New York    NY  40.7127837,-74.0059413   \n",
       "\n",
       "   id  \n",
       "0   0  \n",
       "1   1  \n",
       "2   2  \n",
       "3   3  \n",
       "4   4  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note: You can skip this section if you were able to load the Solr Jobs Core along with the data directory from the \n",
    "# './Solr Core and Config' sub-folder. Older versions of Solr won't read this data, so here's some code to populate \n",
    "# the index from the jobs.csv file\n",
    "\n",
    "jobs_df = pd.read_csv(jobs_data_file, sep=\",\")\n",
    "jobs_df[\"jobSkills\"] = jobs_df[\"jobSkills\"].apply(lambda sk: sk.split(\"|\"))\n",
    "# assign a unique doc id to each row\n",
    "jobs_df[\"id\"] = range(len(jobs_df))\n",
    "jobs_df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<?xml version=\"1.0\" encoding=\"UTF-8\"?>\\n<response>\\n<lst name=\"responseHeader\"><int name=\"status\">0</int><int name=\"QTime\">0</int></lst>\\n</response>\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "solr_connection = solr.Solr(solr_url, persistent=True, timeout=360, max_retries=5)\n",
    "\n",
    "# convert dataframe to a list of dictionaries (required solr client library document format)\n",
    "docs = jobs_df.T.to_dict().values()\n",
    "\n",
    "#wipe out any existing documents if present\n",
    "solr_connection.delete_query(\"*:*\")\n",
    "\n",
    "# send documents\n",
    "solr_connection.add_many(docs)\n",
    "\n",
    "# hard commit and optimize\n",
    "solr_connection.commit()\n",
    "solr_connection.optimize()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Relevancy Judgements File"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>query</th>\n",
       "      <th>fq</th>\n",
       "      <th>location</th>\n",
       "      <th>relevant</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>java developer</td>\n",
       "      <td>{!geofilt}&amp;sfield=geoCode&amp;pt=41.884251,-87.632...</td>\n",
       "      <td>Chicago, IL</td>\n",
       "      <td>8,20,27,52,127,159,194,354,364,414,485,499,677...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>data warehouse</td>\n",
       "      <td>{!geofilt}&amp;sfield=geoCode&amp;pt=41.884251,-87.632...</td>\n",
       "      <td>Chicago, IL</td>\n",
       "      <td>1078,1996,254,254,870,1968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>web services</td>\n",
       "      <td>{!geofilt}&amp;sfield=geoCode&amp;pt=40.7127837, -74.0...</td>\n",
       "      <td>New York, NY</td>\n",
       "      <td>1342,1449,395,1272,1512,54,608,1528,38,84,150,...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            query                                                 fq  \\\n",
       "0  java developer  {!geofilt}&sfield=geoCode&pt=41.884251,-87.632...   \n",
       "1  data warehouse  {!geofilt}&sfield=geoCode&pt=41.884251,-87.632...   \n",
       "2    web services  {!geofilt}&sfield=geoCode&pt=40.7127837, -74.0...   \n",
       "\n",
       "       location                                           relevant  \n",
       "0   Chicago, IL  8,20,27,52,127,159,194,354,364,414,485,499,677...  \n",
       "1   Chicago, IL                         1078,1996,254,254,870,1968  \n",
       "2  New York, NY  1342,1449,395,1272,1512,54,608,1528,38,84,150,...  "
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# The 'relevant' column is a list of document id's (the id field from the schema) that were both in the set of the top\n",
    "# 20 returned documents, and were subjectively judged as relevant to the original\n",
    "# query. We can subsequently use these to derive a MAP score for a given query\n",
    "\n",
    "rel_df = pd.read_csv(relevancy_file, sep=\"|\", converters={\"fq\": str, \"location\": str})\n",
    "searches = rel_df.T.to_dict()\n",
    "rel_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Takes a search id and a qf setting, and returns the list of doc ids, \n",
    "def get_results_for_search(sid, qf_value, rows):\n",
    "    search = searches[sid]\n",
    "    fq = \"\"\n",
    "    pt = \"0,0\"\n",
    "    \n",
    "    if not search[\"location\"].strip() == \"\" :\n",
    "        splt = filter(lambda s: \"pt=\" in s, search[\"fq\"].split(\"&\"))\n",
    "        if splt:\n",
    "            pt = splt[0].replace(\"pt=\",\"\")\n",
    "            fq = \"{!geofilt}\"\n",
    "\n",
    "    resp = solr_connection.select(\n",
    "       q=search[\"query\"], \n",
    "       fields=\"id\",\n",
    "       start=0, rows=rows, \n",
    "       qf=qf_value, # comes from get_solr_params\n",
    "       fq=fq,\n",
    "       sfield=\"geoCode\",\n",
    "       pt=pt,\n",
    "       score=False,\n",
    "       d=\"48.00\", wt=\"json\")\n",
    "    predicted = list(map(lambda res: res[\"id\"], resp.results))\n",
    "    # return predicted doc ids, along with relevent ones (for IR metric)\n",
    "    return predicted, list(map(int, search[\"relevant\"].split(\",\")))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IR Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def apk(actual, predicted, k=10):\n",
    "    \"\"\"\n",
    "    Computes the average precision at k.\n",
    "    This function computes the average prescision at k between two lists of\n",
    "    items.\n",
    "    Parameters\n",
    "    ----------\n",
    "    actual : set\n",
    "             A set of elements that are to be predicted (order doesn't matter)\n",
    "    predicted : list\n",
    "                A list of predicted elements (order does matter)\n",
    "    k : int, optional\n",
    "        The maximum number of predicted elements\n",
    "    Returns\n",
    "    -------\n",
    "    score : double\n",
    "            The average precision at k over the input lists\n",
    "    \"\"\"\n",
    "    if len(predicted)>k:\n",
    "        predicted = predicted[:k]\n",
    "\n",
    "    score = 0.0\n",
    "    num_hits = 0.0\n",
    "\n",
    "    for i,p in enumerate(predicted):\n",
    "        if p in actual and p not in predicted[:i]:\n",
    "            num_hits += 1.0\n",
    "            score += num_hits / (i+1.0)\n",
    "\n",
    "    if not actual:\n",
    "        return 0.0\n",
    "\n",
    "    return score / min(len(actual), k)\n",
    "\n",
    "def mean_average_precision_at_k(actual, predicted, k=10):\n",
    "    \"\"\"\n",
    "    Computes the mean average precision at k.\n",
    "    This function computes the mean average prescision at k between two lists\n",
    "    of lists of items.\n",
    "    Parameters\n",
    "    ----------\n",
    "    actual : list\n",
    "             A list of sets of elements that are to be predicted \n",
    "             (order doesn't matter in the lists)\n",
    "    predicted : list\n",
    "                A list of lists of predicted elements\n",
    "                (order matters in the lists)\n",
    "    k : int, optional\n",
    "        The maximum number of predicted elements\n",
    "    Returns\n",
    "    -------\n",
    "    score : double\n",
    "            The mean average precision at k over the input lists\n",
    "    \"\"\"\n",
    "    return np.mean([apk(a,p,k) for a,p in zip(actual, predicted)])\n",
    "\n",
    "def average_ndcg_at_k(actual, predicted, k, method=0):\n",
    "    vals = [ ndcg_at_k(act, pred, k, method) for act, pred in zip(actual, predicted)]\n",
    "    return np.mean(vals)\n",
    "\n",
    "def ndcg_at_k(actual, predicted, k, method=0):\n",
    "    \n",
    "    # convert to ratings - actual relevant results give rating of 10, vs 1 for the rest\n",
    "    act_hash = set(actual)    \n",
    "    best_ratings = [ 10 for docid in actual ] + [1 for i in range(0, len(predicted) - len(actual))]\n",
    "    pred_ratings = [ 10 if docid in act_hash else 1 for docid in predicted ]\n",
    "        \n",
    "    dcg_max = dcg_at_k(best_ratings, k, method)\n",
    "    if not dcg_max:\n",
    "        return 0.0\n",
    "    dcg = dcg_at_k(pred_ratings, k, method)\n",
    "    return dcg / dcg_max\n",
    "\n",
    "def dcg_at_k(r, k, method=0):\n",
    "    \"\"\"\n",
    "    Code taken from: https://gist.github.com/bwhite/3726239\n",
    "    \n",
    "    Score is discounted cumulative gain (dcg)\n",
    "    Relevance is positive real values.  Can use binary\n",
    "    as the previous methods.\n",
    "    Example from\n",
    "    http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf\n",
    "    >>> r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]\n",
    "    >>> dcg_at_k(r, 1)\n",
    "    3.0\n",
    "    >>> dcg_at_k(r, 1, method=1)\n",
    "    3.0\n",
    "    >>> dcg_at_k(r, 2)\n",
    "    5.0\n",
    "    >>> dcg_at_k(r, 2, method=1)\n",
    "    4.2618595071429155\n",
    "    >>> dcg_at_k(r, 10)\n",
    "    9.6051177391888114\n",
    "    >>> dcg_at_k(r, 11)\n",
    "    9.6051177391888114\n",
    "    Args:\n",
    "        r: Relevance scores (list or numpy) in rank order\n",
    "            (first element is the first item)\n",
    "        k: Number of results to consider\n",
    "        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]\n",
    "                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]\n",
    "    Returns:\n",
    "        Discounted cumulative gain\n",
    "    \"\"\"\n",
    "    r = np.asfarray(r)[:k]\n",
    "    if r.size:\n",
    "        if method == 0:\n",
    "            return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))\n",
    "        elif method == 1:\n",
    "            return np.sum(r / np.log2(np.arange(2, r.size + 2)))\n",
    "        else:\n",
    "            raise ValueError('method must be 0 or 1.')\n",
    "    return 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.74894708994708992"
      ]
     },
     "execution_count": 249,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Measure results for one set of qf settings\n",
    "score = objective([3,1.5,1.1])\n",
    "score # Score is negative, as scopt tries to minimize function output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Black Box Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Function takes a list of 12 real numbers, and returns a set of solr configuration options\n",
    "def get_solr_params(params):\n",
    "    return {\"qf\" : \"employer^{0}  jobTitle^{1}  jobskills^{2}\".format(*params[0:3])\n",
    "            #\"pf2\" :  \"employer^{0}  jobTitle^{1}  jobSkills^{2}\".format(*params[3:6]), \n",
    "            #\"pf\"  :  \"employer^{0}  jobTitle^{1}  jobSkills^{2}\".format(*params[6:9]) \n",
    "           }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 270,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([0, 1, 2, 3, 4, 5, 6], [7, 8, 9])"
      ]
     },
     "execution_count": 270,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# spit into training and test set of queries\n",
    "sids = list(searches.keys())\n",
    "cutoff = int(0.75* len(sids))\n",
    "train_sids, test_sids = sids[:cutoff], sids[cutoff:]\n",
    "train_sids, test_sids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 271,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Precision cut off\n",
    "PREC_AT = 20\n",
    "# Black box objective function to minimize\n",
    "# This is for the training data\n",
    "def objective(params):\n",
    "    # map list of numbers into solr parameters (just qf in this case)\n",
    "    additional_params = get_solr_params(params)\n",
    "    \n",
    "    predicted, actual =[],[]\n",
    "    for sid in train_sids:\n",
    "        pred, act = get_results_for_search(sid, additional_params[\"qf\"], PREC_AT)\n",
    "        predicted.append(pred)\n",
    "        actual.append(act)\n",
    "    # Compute Mean average precision at 20\n",
    "    return -1.0 * mean_average_precision_at_k(actual, predicted, PREC_AT)\n",
    "    # Can also use NDCG - the version above is tailored for binary judegements\n",
    "    #return -1.0 * average_ndcg_at_k(actual, predicted, PREC_AT)\n",
    "\n",
    "# This is for the test data (held out dataset)\n",
    "def evaluate(params):\n",
    "    # map list of numbers into solr parameters (just qf in this case)\n",
    "    additional_params = get_solr_params(params)\n",
    "    \n",
    "    predicted, actual =[],[]\n",
    "    for sid in test_sids:\n",
    "        pred, act = get_results_for_search(sid, additional_params[\"qf\"], PREC_AT)\n",
    "        predicted.append(pred)\n",
    "        actual.append(act)\n",
    "    # Compute Mean average precision at 20\n",
    "    return -1.0 * mean_average_precision_at_k(actual, predicted, PREC_AT)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Run Optimizer Algorithm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.83492361342361343"
      ]
     },
     "execution_count": 257,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Example of how black box function is called to measure value of parameters (qf settings in this case)\n",
    "score = objective([3, 2.5, 1.5])\n",
    "# Score is negative as -1 * (IR metric), and the skopt library tries to find the parameters to minimize the score \n",
    "score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# simple call back function to print progress while optimizing\n",
    "def callback(res):\n",
    "    call_no = len(res.func_vals)\n",
    "    current_fun = res.func_vals[-1]\n",
    "    print str(call_no).ljust(5) + \"\\t\" + \\\n",
    "        str(-1.0* current_fun).ljust(20) + \"\\t\" + str(map(lambda d: round(d,3), res.x_iters[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The code below runs the sci-kit optimization library and tries to find the set of parameters that minimize the objective function above. We are choosing to map the parameter values to qf values (field boosts), but you can in theory try any configuration setting here that you can test in this way. Some settings, such as changing the config files themselves can be accomplished with a core reload, or in some cases a server restart. Note however that you need the algorithm to run for quite a few iterations to learn effectively from your data, and for some problems, it may not be able to find a near optimal solution."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting at  2016-10-13 13:37:47.798818\n",
      "Run \tCurrent MAP \t\tParameters\n",
      "1    \t0.789787257338      \t[7.633, 4.669, 11.146]\n",
      "2    \t0.57634460211       \t[15.118, 3.984, 9.322]\n",
      "3    \t0.789787257338      \t[3.102, 29.481, 19.532]\n",
      "4    \t0.789787257338      \t[22.993, 17.167, 9.658]\n",
      "5    \t0.789787257338      \t[41.763, 49.444, 30.546]\n",
      "6    \t0.789787257338      \t[46.35, 31.324, 44.14]\n",
      "7    \t0.789787257338      \t[36.349, 34.089, 31.117]\n",
      "8    \t0.789787257338      \t[38.425, 27.613, 12.656]\n",
      "9    \t0.789787257338      \t[13.46, 13.443, 8.997]\n",
      "10   \t0.789787257338      \t[32.201, 18.663, 40.82]\n",
      "11   \t0.789787257338      \t[11.269, 15.497, 29.595]\n",
      "12   \t0.789787257338      \t[20.547, 40.29, 35.926]\n",
      "13   \t0.789787257338      \t[3.638, 11.26, 49.192]\n",
      "14   \t0.789787257338      \t[31.253, 46.627, 37.928]\n",
      "15   \t0.789787257338      \t[42.202, 48.694, 26.05]\n",
      "16   \t0.789787257338      \t[26.195, 25.704, 30.834]\n",
      "17   \t0.789787257338      \t[1.909, 49.891, 49.151]\n",
      "18   \t0.789787257338      \t[15.706, 30.462, 27.224]\n",
      "19   \t0.789787257338      \t[2.68, 8.532, 3.091]\n",
      "20   \t0.789787257338      \t[3.708, 24.603, 38.959]\n",
      "21   \t0.728375454855      \t[19.226, 9.157, 16.54]\n",
      "22   \t0.728375454855      \t[26.191, 14.921, 39.56]\n",
      "23   \t0.789787257338      \t[1.391, 2.871, 46.726]\n",
      "24   \t0.709719507628      \t[12.891, 3.945, 27.206]\n",
      "25   \t0.57634460211       \t[39.588, 4.306, 1.604]\n",
      "26   \t0.789787257338      \t[3.249, 3.978, 16.225]\n",
      "27   \t0.789787257338      \t[16.69, 48.174, 9.223]\n",
      "28   \t0.57634460211       \t[9.509, 2.055, 6.204]\n",
      "29   \t0.57634460211       \t[7.135, 0.578, 35.504]\n",
      "30   \t0.789787257338      \t[0.001, 1.27, 15.887]\n",
      "31   \t0.57634460211       \t[3.419, 0.325, 19.748]\n",
      "32   \t0.789787257338      \t[48.939, 29.199, 48.733]\n",
      "33   \t0.57634460211       \t[28.73, 5.601, 20.555]\n",
      "34   \t0.789787257338      \t[45.542, 30.89, 37.488]\n",
      "35   \t0.789787257338      \t[8.569, 22.665, 31.376]\n",
      "36   \t0.789787257338      \t[6.646, 46.63, 35.397]\n",
      "37   \t0.789787257338      \t[2.325, 17.756, 16.661]\n",
      "38   \t0.789787257338      \t[25.798, 16.771, 13.985]\n",
      "39   \t0.789787257338      \t[8.911, 6.325, 10.43]\n",
      "40   \t0.709719507628      \t[6.497, 1.988, 12.637]\n",
      "41   \t0.789787257338      \t[4.907, 5.265, 2.189]\n",
      "42   \t0.789787257338      \t[1.865, 2.763, 15.76]\n",
      "43   \t0.728375454855      \t[8.432, 4.514, 11.192]\n",
      "44   \t0.57634460211       \t[44.589, 4.986, 10.523]\n",
      "45   \t0.789787257338      \t[1.666, 4.527, 35.625]\n",
      "46   \t0.57634460211       \t[5.675, 0.806, 18.293]\n",
      "47   \t0.789787257338      \t[0.607, 10.608, 46.907]\n",
      "48   \t0.789787257338      \t[32.919, 20.687, 42.497]\n",
      "49   \t0.789787257338      \t[29.812, 21.06, 18.681]\n",
      "50   \t0.789787257338      \t[26.974, 26.03, 13.369]\n",
      "51   \t0.789787257338      \t[1.213, 2.867, 25.221]\n",
      "52   \t0.789787257338      \t[3.311, 3.447, 5.755]\n",
      "53   \t0.789787257338      \t[1.434, 2.976, 9.72]\n",
      "54   \t0.57634460211       \t[3.805, 1.035, 39.879]\n",
      "55   \t0.789787257338      \t[7.097, 30.008, 39.301]\n",
      "56   \t0.789787257338      \t[25.475, 16.37, 47.728]\n",
      "57   \t0.789787257338      \t[39.985, 28.122, 31.249]\n",
      "58   \t0.57634460211       \t[37.455, 5.098, 13.536]\n",
      "59   \t0.789787257338      \t[43.914, 36.802, 21.952]\n",
      "60   \t0.789787257338      \t[32.556, 49.627, 31.495]\n",
      "61   \t0.789787257338      \t[26.191, 27.714, 48.825]\n",
      "62   \t0.789787257338      \t[27.228, 42.453, 0.878]\n",
      "63   \t0.789787257338      \t[14.24, 39.124, 26.839]\n",
      "64   \t0.789787257338      \t[16.463, 13.835, 40.497]\n",
      "65   \t0.728375454855      \t[21.164, 10.936, 48.27]\n",
      "66   \t0.728375454855      \t[20.867, 11.45, 23.094]\n",
      "67   \t0.57634460211       \t[35.272, 5.012, 26.865]\n",
      "68   \t0.728375454855      \t[45.792, 26.401, 11.6]\n",
      "69   \t0.789787257338      \t[17.124, 34.076, 0.088]\n",
      "70   \t0.789787257338      \t[4.036, 5.833, 0.333]\n",
      "71   \t0.789787257338      \t[4.146, 4.596, 1.567]\n",
      "72   \t0.566862364771      \t[49.238, 14.449, 49.105]\n",
      "73   \t0.728375454855      \t[10.682, 5.287, 1.837]\n",
      "74   \t0.701782999691      \t[15.613, 5.668, 1.407]\n",
      "75   \t0.57634460211       \t[17.545, 2.875, 16.856]\n",
      "76   \t0.789787257338      \t[47.88, 43.344, 46.422]\n",
      "77   \t0.701782999691      \t[3.284, 1.233, 15.523]\n",
      "78   \t0.57634460211       \t[6.458, 0.825, 11.828]\n",
      "79   \t0.789787257338      \t[35.758, 47.713, 38.415]\n",
      "80   \t0.789787257338      \t[16.539, 41.749, 25.976]\n",
      "81   \t0.789787257338      \t[4.905, 25.428, 10.925]\n",
      "82   \t0.789787257338      \t[0.335, 4.916, 37.995]\n",
      "83   \t0.789787257338      \t[28.091, 20.628, 19.817]\n",
      "84   \t0.789787257338      \t[2.956, 4.116, 32.588]\n",
      "85   \t0.789787257338      \t[2.429, 1.435, 38.5]\n",
      "86   \t0.789787257338      \t[3.053, 1.86, 38.95]\n",
      "87   \t0.789787257338      \t[49.483, 33.301, 44.404]\n",
      "88   \t0.789787257338      \t[49.118, 49.176, 23.353]\n",
      "89   \t0.789787257338      \t[49.878, 33.831, 45.104]\n",
      "90   \t0.789787257338      \t[18.099, 14.532, 49.317]\n",
      "91   \t0.572735380644      \t[49.669, 14.049, 48.293]\n",
      "92   \t0.789787257338      \t[0.822, 2.53, 7.718]\n",
      "93   \t0.789787257338      \t[0.686, 2.419, 1.253]\n",
      "94   \t0.728375454855      \t[3.094, 1.684, 11.845]\n",
      "95   \t0.789787257338      \t[12.516, 13.298, 4.776]\n",
      "96   \t0.789787257338      \t[1.97, 3.009, 9.946]\n",
      "97   \t0.572735380644      \t[47.755, 13.42, 3.868]\n",
      "98   \t0.789787257338      \t[2.278, 4.854, 12.789]\n",
      "99   \t0.789787257338      \t[0.414, 3.942, 9.38]\n",
      "100  \t0.789787257338      \t[0.06, 1.691, 8.329]\n"
     ]
    }
   ],
   "source": [
    "from skopt import gbrt_minimize\n",
    "import datetime\n",
    "\n",
    "ITERATIONS = 100 # probably want this to be high, 500 calls or more, set to a small value greater than 10 to test it is working\n",
    "min_val, max_val = 0.0, 50.0\n",
    "# min and max for each possible qf value (we read 3 in get_solr_params currently)\n",
    "space  = [(min_val, max_val) for i in range(3)] \n",
    "\n",
    "start = datetime.datetime.now()\n",
    "print \"Starting at \", start\n",
    "print \"Run\",\"\\t\", \"Current MAP\", \"\\t\\t\", \"Parameters\"\n",
    "# run optimizer, which will try to minimize the objective function\n",
    "res = gbrt_minimize(objective,       # the function to minimize\n",
    "                  space,             # the bounds on each dimension of x\n",
    "                  acq=\"LCB\",         # controls how it searches for parameters\n",
    "                  n_calls=ITERATIONS,# the number of evaluations of f including at x0\n",
    "                  random_state=777,  # set to a fixed number if you want this to be deterministic\n",
    "                  n_jobs=-1,         # how many threads (or really python processes due to GIL)\n",
    "                  callback=callback) \n",
    "\n",
    "end = datetime.datetime.now()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The evaluate function below is the same as the objective function, except it tests our newly optimized set of parameters on a different set of queries. This gives a more accurate measure of the performance of the new settings on data points and queries that were not in the training dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 273,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-0.70588235294117652"
      ]
     },
     "execution_count": 273,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# res.fun - function IR metric score (* -1), res.x - the best performing parameters\n",
    "test_score = evaluate(res.x)\n",
    "test_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The results from the training here are much higher than the test set. This is typical for a lot of machine learning \\ optimization problems. If tuning an existing solr installation, you will want to ensure that the IR metrics score on the test set is better than the current production settings before releasing to production."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 276,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "IR Metric @20 Training Data = 0.789787257338\n",
      "IR Metric @20 Test Data     = 0.705882352941\n",
      "\n",
      "Parameters:\n",
      "\temployer^7.63318674507  jobTitle^4.66866284146  jobskills^11.1464049498\n",
      "\n",
      "gbrt_minimize took 14.664135 secs\n"
     ]
    }
   ],
   "source": [
    "print(\"IR Metric @\" + str(PREC_AT) + \" Training Data = \" +  str(-1 * res.fun))\n",
    "print(\"IR Metric @\" + str(PREC_AT) + \" Test Data     = \" +  str(-1 * test_score))\n",
    "print(\"\\nParameters:\\n\\t\"),\n",
    "print get_solr_params(res.x)[\"qf\"]\n",
    "print \"\\ngbrt_minimize took\", (end - start).total_seconds(), \"secs\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}