results/SETI Classification Accuracy Scores.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SETI Test Set Classification Accuracy\n",
    "\n",
    "This notebook provides the code needed to calculate the performance of your signal classification models using the PREVIEW test set (see [Step 1. Get Data notebook](https://github.com/setiQuest/ML4SETI/blob/master/tutorials/Step_1_Get_Data.ipynb))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import csv\n",
    "import operator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class_list = ['brightpixel', 'narrowband', 'narrowbanddrd', 'noise', 'squarepulsednarrowband', 'squiggle', 'squigglesquarepulsednarrowband']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fieldnames = ['uuid'] + class_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Helper functions for parsing the data and using sklearn to print scoring metrics\n",
    "\n",
    "def classChooser(listOfDictionaryScores):\n",
    "    results = []\n",
    "    for row in listOfDictionaryScores:\n",
    "        rowscores = dict((k, float(row[k])) for k in class_list)\n",
    "        maxclass = max(rowscores.iteritems(), key=operator.itemgetter(1))[0]\n",
    "        results.append({'UUID':row['uuid'], 'SIGNAL_CLASSIFICATION':maxclass})\n",
    "        \n",
    "    return results\n",
    "\n",
    "def printsklearnScores(y_true, y_pred, y_prob):\n",
    "    \n",
    "    print sklearn.metrics.classification_report(y_true,y_pred, digits=5)\n",
    "    print sklearn.metrics.confusion_matrix(y_true,y_pred)\n",
    "    print(\"Classification accuracy: %0.6f\" % sklearn.metrics.accuracy_score(y_true,y_pred) )\n",
    "    print(\"Log Loss: %0.6f\" % sklearn.metrics.log_loss(y_true,y_prob) )\n",
    "    \n",
    "    \n",
    "# Takes a .csv scorecard file, compares the results to the preview testset UUID,Class file and\n",
    "# prints the scores. \n",
    "def score(resultsFile):\n",
    "\n",
    "    testSetFile = 'private_list_primary_v3_testset_preview_uuid_class_29june_2017.csv'\n",
    "    \n",
    "    actual_uuid = csv.DictReader(open(testSetFile))\n",
    "    actual_uuid_list = [x for x in actual_uuid]\n",
    "    actual_uuid_list_sorted = sorted(actual_uuid_list, key=lambda k: k['UUID']) \n",
    "\n",
    "    classifier_results = csv.DictReader(open(resultsFile), fieldnames=fieldnames)\n",
    "    classifier_results_list = [x for x in classifier_results]\n",
    "    classifier_results_list_sorted = sorted(classifier_results_list, key=lambda k: k['uuid']) \n",
    "\n",
    "    #yc = classChooser(classifier_results_list_sorted)\n",
    "    #print yc[:5]\n",
    "    \n",
    "    y_true = [x['SIGNAL_CLASSIFICATION'] for x in actual_uuid_list_sorted]\n",
    "    y_pred = [x['SIGNAL_CLASSIFICATION'] for x in classChooser(classifier_results_list_sorted)]\n",
    "    y_prob = [[float(row[cl]) for cl in class_list] for row in classifier_results_list_sorted] \n",
    "    \n",
    "    printsklearnScores(y_true, y_pred, y_prob)\n",
    "   \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scoring a Scorecard\n",
    "\n",
    "The Preview test data set can be obtained in the [Step 1. Get Data notebook](https://github.com/setiQuest/ML4SETI/blob/master/tutorials/Step_1_Get_Data.ipynb).  Using your trained model, you can generate a scorecard for this preview test data set. Your scorecard must be a CSV file with 8 columns. The first column value will contain the UUID and the next 7 will contain the probability estimates for each of the classes that were produced by your model. See the [Judging Information notebook for more information](https://github.com/setiQuest/ML4SETI/blob/master/Judging_Criteria.ipynb).\n",
    "\n",
    "Now you can score the scorecard using this code. [We now are providing the Preview test data set key in order for you to easily produce your own confusion matrix and scoring. This will give you the exact answers for the preview test set, of course.]  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "### Using the Example Scorecard.\n",
    "\n",
    "On the [Judging Information notebook](https://github.com/setiQuest/ML4SETI/blob/master/Judging_Criteria.ipynb) there is a link to download an example scorecard.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                precision    recall  f1-score   support\n",
      "\n",
      "                   brightpixel    0.12349   0.12812   0.12577       320\n",
      "                    narrowband    0.16467   0.14474   0.15406       380\n",
      "                 narrowbanddrd    0.19274   0.19885   0.19574       347\n",
      "                         noise    0.12392   0.12798   0.12592       336\n",
      "        squarepulsednarrowband    0.16298   0.17302   0.16785       341\n",
      "                      squiggle    0.18106   0.17711   0.17906       367\n",
      "squigglesquarepulsednarrowband    0.13043   0.13003   0.13023       323\n",
      "\n",
      "                   avg / total    0.15525   0.15493   0.15495      2414\n",
      "\n",
      "[[41 51 50 38 35 60 45]\n",
      " [47 55 61 65 48 56 48]\n",
      " [42 46 69 52 44 42 52]\n",
      " [47 47 56 43 53 41 49]\n",
      " [42 42 43 47 59 56 52]\n",
      " [60 42 45 64 57 65 34]\n",
      " [53 51 34 38 66 39 42]]\n",
      "Classification accuracy: 0.154930\n",
      "Log Loss: 2.230604\n"
     ]
    }
   ],
   "source": [
    "score('example_scorecard_codechallenge_v3_testset_preview.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<br>\n",
    "### Using the Preview Test Set Key.\n",
    "\n",
    "If I use a scorecard built from the preview test UUID,class CSV file, then I will get a perfect score. With the UUID,class file I created the `private_list_primary_v3_testset_preview_scoreboard_key_29june_2017.csv`. This scorecard will produce a perfect score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                precision    recall  f1-score   support\n",
      "\n",
      "                   brightpixel    1.00000   1.00000   1.00000       320\n",
      "                    narrowband    1.00000   1.00000   1.00000       380\n",
      "                 narrowbanddrd    1.00000   1.00000   1.00000       347\n",
      "                         noise    1.00000   1.00000   1.00000       336\n",
      "        squarepulsednarrowband    1.00000   1.00000   1.00000       341\n",
      "                      squiggle    1.00000   1.00000   1.00000       367\n",
      "squigglesquarepulsednarrowband    1.00000   1.00000   1.00000       323\n",
      "\n",
      "                   avg / total    1.00000   1.00000   1.00000      2414\n",
      "\n",
      "[[320   0   0   0   0   0   0]\n",
      " [  0 380   0   0   0   0   0]\n",
      " [  0   0 347   0   0   0   0]\n",
      " [  0   0   0 336   0   0   0]\n",
      " [  0   0   0   0 341   0   0]\n",
      " [  0   0   0   0   0 367   0]\n",
      " [  0   0   0   0   0   0 323]]\n",
      "Classification accuracy: 1.000000\n",
      "Log Loss: 0.000000\n"
     ]
    }
   ],
   "source": [
    "#Test with the scoreboard key. This should get 100% accuracy\n",
    "score('private_list_primary_v3_testset_preview_scoreboard_key_29june_2017.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Winning Team's Scorecard.\n",
    "\n",
    "I've included the winning team's scorecard submitted to the preview test set scoreboard on July 21 in this repository. The scores for that scorecard are shown below. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                precision    recall  f1-score   support\n",
      "\n",
      "                   brightpixel    0.99262   0.84062   0.91032       320\n",
      "                    narrowband    0.98592   0.92105   0.95238       380\n",
      "                 narrowbanddrd    0.94693   0.97695   0.96170       347\n",
      "                         noise    0.79433   1.00000   0.88538       336\n",
      "        squarepulsednarrowband    0.96923   0.92375   0.94595       341\n",
      "                      squiggle    0.99178   0.98638   0.98907       367\n",
      "squigglesquarepulsednarrowband    0.98738   0.96904   0.97813       323\n",
      "\n",
      "                   avg / total    0.95326   0.94615   0.94693      2414\n",
      "\n",
      "[[269   0   0  51   0   0   0]\n",
      " [  0 350  18   7   5   0   0]\n",
      " [  1   1 339   6   0   0   0]\n",
      " [  0   0   0 336   0   0   0]\n",
      " [  1   4   1  20 315   0   0]\n",
      " [  0   0   0   1   0 362   4]\n",
      " [  0   0   0   2   5   3 313]]\n",
      "Classification accuracy: 0.946147\n",
      "Log Loss: 0.188138\n"
     ]
    }
   ],
   "source": [
    "score(\"results_Effsubsee_best_preview_test_set.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to score your own test set\n",
    "\n",
    "You can use the score functions above with your own test data set parsed from the training data set to measure your model performance. Of course, this let's you test different models and different model parameters more quickly and while keeping the preview test set available for your nearly completed model. \n",
    "\n",
    "The following code will \n",
    "  * show how to split the training data into a training set and test set\n",
    "  * create some fake models to produce some predicted values for the test set\n",
    "  * pass those predicted values to the `printsklearnScores` function above\n",
    "\n",
    "\n",
    "## 1. Split Up the Data\n",
    "First, let's split our data up into a training data set and a test set. We start with the primay small \n",
    "index file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "indexfile = 'public_list_primary_v3_small_21june_2017.csv'\n",
    "indexfile_uuid = csv.DictReader(open(indexfile))\n",
    "indexfile_uuid_list = [x for x in indexfile_uuid]\n",
    "indexfile_uuid_list = sorted(indexfile_uuid_list, key=lambda k: k['UUID'])\n",
    "\n",
    "X = [x['UUID'] for x in indexfile_uuid_list]\n",
    "y = [class_list.index(x['SIGNAL_CLASSIFICATION']) for x in indexfile_uuid_list] #also convert from class name to a number\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Train Your Model\n",
    "In normal operation, you'd then use the `X_train` set of UUIDs to grab the `<UUID>.dat`  data files and produce spectrograms and features. You'd then pass your features, along with `y_train`, which contains\n",
    "the labels, to your model for training. \n",
    "\n",
    "Below, I've coded up two FAKE models. The `randomModel` produces random probabilities. The `perfectModel` actually uses the known values in the `y_test` -- so it will produce a perfect score. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "# Example classes\n",
    "# Your class, of course, would have actual code in the `train` functions and\n",
    "# the predict function would also be different.\n",
    "\n",
    "class randomModel(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def train(self, X_train, y_train):\n",
    "        ## do whatever\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X_test):\n",
    "        y_prob = np.random.rand(len(X_test), len(class_list))\n",
    "        return (y_prob.T / y_prob.sum(axis=1)).T\n",
    "\n",
    "    \n",
    "class perfectModel(object):\n",
    "    def __init__(self):\n",
    "        pass\n",
    "    \n",
    "    def train(self, X_train, y_train):\n",
    "        ## train\n",
    "        pass\n",
    "    \n",
    "    def predict(self, X_test):\n",
    "        encoder = LabelBinarizer()\n",
    "        ytest_np = np.array(y_test).reshape(1,-1)\n",
    "        ytest_onehot = encoder.fit_transform(ytest_np.T)\n",
    "        return ytest_onehot\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3.  Make Predicitons and Score\n",
    "\n",
    "Next, you'd take the `X_test` set of UUIDs, extract the necessary spectrogram and features and pass that to your model\n",
    "in order to predict their class. We use the two fake models from above: `perfectModel` and `randomModel`. \n",
    "\n",
    "Each model.predict function returns a 2d array, M x K, where M is the number of samples in the test set passed into the function and K is the number of classes. The values for each row are the class probability predictions. Obviously, your model should produce a LogLoss and classification accuracy score somewhere between these two values. \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The randomModel class produces random probability estimates\n",
      "[[ 0.20222191  0.15488604  0.04116172  0.05002166  0.1117629   0.2144418\n",
      "   0.22550396]\n",
      " [ 0.03256574  0.22902783  0.22244701  0.15240424  0.20808028  0.04013136\n",
      "   0.11534354]\n",
      " [ 0.26182883  0.10909254  0.09191382  0.02477715  0.11923854  0.24349684\n",
      "   0.14965228]\n",
      " [ 0.06927687  0.13536934  0.2392518   0.24012245  0.0205762   0.12298141\n",
      "   0.17242193]\n",
      " [ 0.1835864   0.20328221  0.02019067  0.05209727  0.1254207   0.20358192\n",
      "   0.21184083]]\n",
      "\n",
      "                                precision    recall  f1-score   support\n",
      "\n",
      "                   brightpixel    0.17544   0.19417   0.18433       103\n",
      "                    narrowband    0.17000   0.17000   0.17000       100\n",
      "                 narrowbanddrd    0.12766   0.11765   0.12245       102\n",
      "                         noise    0.14019   0.17241   0.15464        87\n",
      "        squarepulsednarrowband    0.21429   0.17822   0.19459       101\n",
      "                      squiggle    0.20721   0.22549   0.21596       102\n",
      "squigglesquarepulsednarrowband    0.20000   0.17143   0.18462       105\n",
      "\n",
      "                   avg / total    0.17724   0.17571   0.17571       700\n",
      "\n",
      "[[20 12 11 14 12 15 19]\n",
      " [22 17 14 18 10 10  9]\n",
      " [14 15 12 16 15 19 11]\n",
      " [12 13 15 15  6 14 12]\n",
      " [13 15 15  9 18 16 15]\n",
      " [19 14 13 20  7 23  6]\n",
      " [14 14 14 15 16 14 18]]\n",
      "Classification accuracy: 0.175714\n",
      "Log Loss: 2.274388\n"
     ]
    }
   ],
   "source": [
    "mRandModel = randomModel()\n",
    "mRandModel.train(X_train, y_train)\n",
    "y_prob = mRandModel.predict(X_test)\n",
    "y_true = [class_list[i] for i in y_test]\n",
    "y_pred = [class_list[probarray.argmax()] for probarray in y_prob]\n",
    "\n",
    "print 'The randomModel class produces random probability estimates'\n",
    "print y_prob[:5]\n",
    "print ''\n",
    "\n",
    "printsklearnScores(y_true, y_pred, y_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0 0 0 0 1 0 0]\n",
      " [0 0 0 0 0 0 1]\n",
      " [0 0 0 1 0 0 0]\n",
      " [0 0 0 0 1 0 0]\n",
      " [0 0 0 1 0 0 0]]\n",
      "                                precision    recall  f1-score   support\n",
      "\n",
      "                   brightpixel    1.00000   1.00000   1.00000       103\n",
      "                    narrowband    1.00000   1.00000   1.00000       100\n",
      "                 narrowbanddrd    1.00000   1.00000   1.00000       102\n",
      "                         noise    1.00000   1.00000   1.00000        87\n",
      "        squarepulsednarrowband    1.00000   1.00000   1.00000       101\n",
      "                      squiggle    1.00000   1.00000   1.00000       102\n",
      "squigglesquarepulsednarrowband    1.00000   1.00000   1.00000       105\n",
      "\n",
      "                   avg / total    1.00000   1.00000   1.00000       700\n",
      "\n",
      "[[103   0   0   0   0   0   0]\n",
      " [  0 100   0   0   0   0   0]\n",
      " [  0   0 102   0   0   0   0]\n",
      " [  0   0   0  87   0   0   0]\n",
      " [  0   0   0   0 101   0   0]\n",
      " [  0   0   0   0   0 102   0]\n",
      " [  0   0   0   0   0   0 105]]\n",
      "Classification accuracy: 1.000000\n",
      "Log Loss: 0.000000\n"
     ]
    }
   ],
   "source": [
    "mPerfectModel = perfectModel()\n",
    "mPerfectModel.train(X_train, y_train)\n",
    "\n",
    "y_prob = mPerfectModel.predict(X_test)\n",
    "y_true = [class_list[i] for i in y_test]\n",
    "y_pred = [class_list[probarray.argmax()] for probarray in y_prob]\n",
    "\n",
    "print y_prob[:5]\n",
    "\n",
    "printsklearnScores(y_true, y_pred, y_prob)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}