notebooks/famousFaces_003b_filteredChoice.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Celebrity Multiple Choice - Filtered Version"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import fnmatch\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get logfiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getLogfile(whichfolder, whichexperiment):\n",
    "\n",
    "    loglist = []\n",
    "    for fileName in os.listdir(whichfolder):\n",
    "        if fnmatch.fnmatch(fileName, whichexperiment):\n",
    "            loglist.append(whichfolder+fileName)\n",
    "    loglist.sort()\n",
    "    return loglist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logList = getLogfile('../famousFiltered/experiment/data/','*.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../famousFiltered/experiment/data/famousFilterClean_7.csv',\n",
       " '../famousFiltered/experiment/data/famousFilterClean_8.csv',\n",
       " '../famousFiltered/experiment/data/famousFilterClean_9.csv',\n",
       " '../famousFiltered/experiment/data/famousFilterClean_jb.csv',\n",
       " '../famousFiltered/experiment/data/jb_famFaceFilter_2016_Nov_30_1302.csv']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "logList[-5:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get content from logfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "logFile = logList[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def makeDf(logFile):\n",
    "    \n",
    "    df = pd.read_csv(logFile).drop(0)\n",
    "    \n",
    "    d = {}\n",
    "    for idx in df.index:\n",
    "        k1 = df['ratingK1.response'].loc[idx]\n",
    "        k2 = df['ratingK2.response'].loc[idx]\n",
    "        k3 = df['ratingK3.response'].loc[idx]\n",
    "        if k1 != 'None':\n",
    "            d[idx] = 'act'   \n",
    "        elif k2 != 'None':\n",
    "            d[idx] = k2\n",
    "        elif k3 != 'None':\n",
    "            d[idx] = k3\n",
    "        else:\n",
    "            return\n",
    "\n",
    "    newDf = pd.DataFrame(index=df.index)\n",
    "\n",
    "    newDf['name'] = [x.split('/')[-1].split('.')[0] for x in df['pic']]\n",
    "    condDict = {'smoo15':'smoo','edges5':'edges','gray':'gray'}\n",
    "    newDf['cond'] = [ condDict[x.split('_')[0]] for x in newDf['name'] ]\n",
    "\n",
    "    newDf['familiar'] = [int(x=='1.ja') for x in df['ratingBekannt.response']]\n",
    "    \n",
    "    # make sure there are no missing values\n",
    "    for x in newDf['familiar']:\n",
    "        if x == 'None':\n",
    "            return\n",
    "        \n",
    "    newDf['context'] = pd.DataFrame(d,index=['context']).T\n",
    "    \n",
    "    newDf.index = newDf['name']\n",
    "    return newDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "thisDf = makeDf(logFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>cond</th>\n",
       "      <th>familiar</th>\n",
       "      <th>context</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>gray_Ice-T(OdafinFinTutuola)7_1</th>\n",
       "      <td>gray_Ice-T(OdafinFinTutuola)7_1</td>\n",
       "      <td>gray</td>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gray_OmarEpps(DrEricForeman)3_1</th>\n",
       "      <td>gray_OmarEpps(DrEricForeman)3_1</td>\n",
       "      <td>gray</td>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gray_WillSmith5_1</th>\n",
       "      <td>gray_WillSmith5_1</td>\n",
       "      <td>gray</td>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gray_BritneySpears7_1</th>\n",
       "      <td>gray_BritneySpears7_1</td>\n",
       "      <td>gray</td>\n",
       "      <td>1</td>\n",
       "      <td>music</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gray_WillSmith4_1</th>\n",
       "      <td>gray_WillSmith4_1</td>\n",
       "      <td>gray</td>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                            name  cond  \\\n",
       "name                                                                     \n",
       "gray_Ice-T(OdafinFinTutuola)7_1  gray_Ice-T(OdafinFinTutuola)7_1  gray   \n",
       "gray_OmarEpps(DrEricForeman)3_1  gray_OmarEpps(DrEricForeman)3_1  gray   \n",
       "gray_WillSmith5_1                              gray_WillSmith5_1  gray   \n",
       "gray_BritneySpears7_1                      gray_BritneySpears7_1  gray   \n",
       "gray_WillSmith4_1                              gray_WillSmith4_1  gray   \n",
       "\n",
       "                                 familiar context  \n",
       "name                                               \n",
       "gray_Ice-T(OdafinFinTutuola)7_1         1     act  \n",
       "gray_OmarEpps(DrEricForeman)3_1         1     act  \n",
       "gray_WillSmith5_1                       1     act  \n",
       "gray_BritneySpears7_1                   1   music  \n",
       "gray_WillSmith4_1                       1     act  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "thisDf.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### evaluate context responses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "reference list:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "contextDf = pd.read_csv('../famousCheck/contextList.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "def evalResp(df,contextDf=contextDf):\n",
    "    \n",
    "    respDict = {}\n",
    "\n",
    "    for i in df.index:\n",
    "\n",
    "        resp = df.loc[i]['context']\n",
    "        bekannt = df.loc[i]['familiar']\n",
    "\n",
    "        for idx in contextDf.index:\n",
    "            correct = contextDf.loc[idx]['context']\n",
    "            thisName = contextDf.loc[idx]['name']\n",
    "            \n",
    "            for namePart in thisName.split(' '):\n",
    "                if namePart in i:\n",
    "\n",
    "                    respDict[i] = {'corrAns':correct,\n",
    "                                          'familiarity':bekannt,\n",
    "                                          'corrContext':resp,\n",
    "                                          'context': int(resp in correct)}\n",
    "                    \n",
    "    respDf = pd.DataFrame( respDict ).T\n",
    "    \n",
    "    return respDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "evalDf = evalResp(thisDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>corrAns</th>\n",
       "      <th>corrContext</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>smoo15_RobertSeanLeonard(Wilson)8_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_TomCruise1_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_TomCruise4_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_TomCruise5_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_UrsulavonderLeyen1_1</th>\n",
       "      <td>0</td>\n",
       "      <td>polit</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_UrsulavonderLeyen4_1</th>\n",
       "      <td>0</td>\n",
       "      <td>polit</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_UrsulavonderLeyen5_1</th>\n",
       "      <td>0</td>\n",
       "      <td>polit</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith1_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith4_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith5_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                    context    corrAns corrContext familiarity\n",
       "smoo15_RobertSeanLeonard(Wilson)8_1       1        act         act           0\n",
       "smoo15_TomCruise1_1                       1        act         act           1\n",
       "smoo15_TomCruise4_1                       1        act         act           0\n",
       "smoo15_TomCruise5_1                       1        act         act           1\n",
       "smoo15_UrsulavonderLeyen1_1               0      polit         act           0\n",
       "smoo15_UrsulavonderLeyen4_1               0      polit         act           0\n",
       "smoo15_UrsulavonderLeyen5_1               0      polit         act           0\n",
       "smoo15_WillSmith1_1                       1  act music         act           1\n",
       "smoo15_WillSmith4_1                       1  act music         act           1\n",
       "smoo15_WillSmith5_1                       1  act music         act           1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evalDf.tail(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get rid of unknown faces"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "checkDf = pd.read_csv('../famousCheck/output/famousFamiliarity_%s.csv' % 9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>img</th>\n",
       "      <th>name</th>\n",
       "      <th>bekannt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>./img/SigmarGabriel1.jpg</td>\n",
       "      <td>Sigmar Gabriel</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>./img/BritneySpears5.jpg</td>\n",
       "      <td>Britney Spears</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>./img/EmilyDeschanel3.jpg</td>\n",
       "      <td>Emily Deschanel</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>./img/AlysonHannigan(Lily)1.jpg</td>\n",
       "      <td>Alyson Hannigan</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>./img/BarackObama2.jpg</td>\n",
       "      <td>Barack Obama</td>\n",
       "      <td>ja</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                img             name bekannt\n",
       "37         ./img/SigmarGabriel1.jpg   Sigmar Gabriel      ja\n",
       "38         ./img/BritneySpears5.jpg   Britney Spears      ja\n",
       "39        ./img/EmilyDeschanel3.jpg  Emily Deschanel      ja\n",
       "40  ./img/AlysonHannigan(Lily)1.jpg  Alyson Hannigan      ja\n",
       "41           ./img/BarackObama2.jpg     Barack Obama      ja"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkDf.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cleanUp(evalDf,checkDf=checkDf):\n",
    "    copyDf = evalDf.copy()\n",
    "    for idx in evalDf.index:\n",
    "        #print idx\n",
    "        for j in checkDf.index:\n",
    "            status = (checkDf.loc[j]['bekannt'] == 'nein')\n",
    "            name = checkDf.loc[j]['name'].split(' ')\n",
    "            if status:\n",
    "                for sub in name:\n",
    "                    \n",
    "                    if sub in idx and len(sub)>3:# the >3 is a hack to prevent 'von' and 'der' in von der Leyen to be used for matching\n",
    "                        # this is a hack to prevent Robert Sean Leonard matching Leonard[o] DiCaprio\n",
    "                        if sub == 'Leonard' and 'DiCaprio' in idx:\n",
    "                            pass\n",
    "                        else:\n",
    "                            #print sub,idx\n",
    "                            copyDf.drop(idx,inplace=True)\n",
    "                            break\n",
    "                            \n",
    "    condDict = {'smoo15':'smoo','edges5':'edges','gray':'gray'}\n",
    "    cond = [ condDict[x.split('_')[0]] for x in copyDf.index]                    \n",
    "    copyDf.index = [cond,copyDf.index]\n",
    "    \n",
    "    return copyDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "cleanDf = cleanUp(evalDf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>corrAns</th>\n",
       "      <th>corrContext</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">smoo</th>\n",
       "      <th>smoo15_UrsulavonderLeyen4_1</th>\n",
       "      <td>0</td>\n",
       "      <td>polit</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_UrsulavonderLeyen5_1</th>\n",
       "      <td>0</td>\n",
       "      <td>polit</td>\n",
       "      <td>act</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith1_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith4_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>smoo15_WillSmith5_1</th>\n",
       "      <td>1</td>\n",
       "      <td>act music</td>\n",
       "      <td>act</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 context    corrAns corrContext familiarity\n",
       "smoo smoo15_UrsulavonderLeyen4_1       0      polit         act           0\n",
       "     smoo15_UrsulavonderLeyen5_1       0      polit         act           0\n",
       "     smoo15_WillSmith1_1               1  act music         act           1\n",
       "     smoo15_WillSmith4_1               1  act music         act           1\n",
       "     smoo15_WillSmith5_1               1  act music         act           1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleanDf.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### get percent correct for one participant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makePercent(evalDf,pName):\n",
    "    \n",
    "    sumDf = evalDf.drop(['corrAns','corrContext'],axis=1).groupby(level=0).sum()\n",
    "    countDf = evalDf.drop(['corrAns','corrContext'],axis=1).groupby(level=0).count()\n",
    "    \n",
    "    percentDf = sumDf/countDf\n",
    "    df = pd.DataFrame( percentDf.stack() ).T\n",
    "    df.index = [pName]\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "fuDf = makePercent(cleanDf,'pName')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>pName</th>\n",
       "      <td>0.878788</td>\n",
       "      <td>0.80303</td>\n",
       "      <td>0.954545</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.893939</td>\n",
       "      <td>0.666667</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          edges                  gray                  smoo            \n",
       "        context familiarity   context familiarity   context familiarity\n",
       "pName  0.878788     0.80303  0.954545    0.969697  0.893939    0.666667"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fuDf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### jb split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "jbRecogDf = pd.read_csv('../famousCheck/recognizableList.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeRecogSplit(thisDf,jbRecogDf=jbRecogDf):\n",
    "    \n",
    "    df = thisDf.copy()\n",
    "    \n",
    "    # for each entry in the response df\n",
    "    for i in df.index:\n",
    "        thisName = i[1]\n",
    "        # for each name in the recoglist\n",
    "        for j in jbRecogDf.index:\n",
    "            thisRecog = jbRecogDf.loc[j]['name']\n",
    "            thisStatus = jbRecogDf.loc[j]['recognizable']\n",
    "            # if the names match\n",
    "            if thisRecog in thisName:\n",
    "                # set the status of the \n",
    "                df.set_value(i,'jbRecog',thisStatus)\n",
    "           \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makePercentJb(evalDf,pName):\n",
    "    \n",
    "    myDf = evalDf.drop(['corrAns','corrContext'],axis=1)\n",
    "    myDf['type'] = [myDf.index.levels[0][x] for x in myDf.index.labels[0] ]\n",
    "    sumDf = myDf.groupby(['jbRecog','type']).sum()\n",
    "    countDf = myDf.groupby(['jbRecog','type']).count()\n",
    "    \n",
    "    percentDf = sumDf/countDf\n",
    "    percentDf = percentDf.unstack(1).reorder_levels((1,0),axis=1).sort_index(axis=1)\n",
    "    return percentDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jbRecog</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>0.566667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.909091</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.969697</td>\n",
       "      <td>0.727273</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "type        edges                  gray                  smoo            \n",
       "          context familiarity   context familiarity   context familiarity\n",
       "jbRecog                                                                  \n",
       "JB       0.800000    0.666667  0.933333    0.933333  0.800000    0.566667\n",
       "JB*      0.969697    0.909091  1.000000    1.000000  0.969697    0.727273"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makePercentJb(makeRecogSplit(cleanDf),'pName')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeParticipant(logFile,contextDf=contextDf):\n",
    "    \n",
    "    pName = logFile.split('/')[-1].split('_')[0]\n",
    "\n",
    "    thisDf = makeDf(logFile)\n",
    "    evalDf = evalResp(thisDf)\n",
    "    checkDf = pd.read_csv('../famousCheck/output/famousFamiliarity_%s.csv' % pName)\n",
    "    cleanDf = cleanUp(evalDf,checkDf)\n",
    "    \n",
    "    if pName == 'jb':\n",
    "        thisDf = makePercentJb(makeRecogSplit(cleanDf),pName)\n",
    "    else:\n",
    "        thisDf = makePercent(cleanDf,pName)\n",
    "    \n",
    "    return thisDf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.520833</td>\n",
       "      <td>0.729167</td>\n",
       "      <td>0.666667</td>\n",
       "      <td>0.604167</td>\n",
       "      <td>0.520833</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       edges                  gray                  smoo            \n",
       "     context familiarity   context familiarity   context familiarity\n",
       "11  0.666667    0.520833  0.729167    0.666667  0.604167    0.520833"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makeParticipant(logList[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jbRecog</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.696970</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.606061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.694444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "type        edges                  gray                  smoo            \n",
       "          context familiarity   context familiarity   context familiarity\n",
       "jbRecog                                                                  \n",
       "JB       0.818182    0.696970  0.939394    0.939394  0.818182    0.606061\n",
       "JB*      0.972222    0.916667  1.000000    1.000000  0.972222    0.694444"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makeParticipant(logList[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### do this for all participants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def makeBigDf(logList,contextDf=contextDf):\n",
    "    bigDf = pd.DataFrame()\n",
    "    for logFile in logList:\n",
    "        #print '\\n',logFile\n",
    "        try:\n",
    "            thisDf = makeParticipant(logFile,contextDf=contextDf)\n",
    "            bigDf = pd.concat([bigDf,thisDf])\n",
    "        except:\n",
    "            print '!!!!',logFile\n",
    "            \n",
    "    return bigDf*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!!!! ../famousFiltered/experiment/data/10_famFaceFilter_2017_Feb_06_1253.csv\n",
      "!!!! ../famousFiltered/experiment/data/18_famFaceFilter_2017_Feb_16_1655.csv\n",
      "!!!! ../famousFiltered/experiment/data/19_famFaceFilter_2017_Feb_20_1110.csv\n",
      "!!!! ../famousFiltered/experiment/data/23_famFaceFilter_2017_Feb_23_1717.csv\n",
      "!!!! ../famousFiltered/experiment/data/24_famFaceFilter_2017_Feb_24_1121.csv\n",
      "!!!! ../famousFiltered/experiment/data/26_famFaceFilter_2017_Feb_28_1246.csv\n",
      "!!!! ../famousFiltered/experiment/data/27_famFaceFilter_2017_Mrz_01_1642.csv\n",
      "!!!! ../famousFiltered/experiment/data/3_famFaceFilter_2017_Jan_25_1257.csv\n",
      "!!!! ../famousFiltered/experiment/data/5_famFaceFilter_2017_Jan_26_1056.csv\n",
      "!!!! ../famousFiltered/experiment/data/6_famFaceFilter_2017_Jan_27_1049.csv\n",
      "!!!! ../famousFiltered/experiment/data/7_famFaceFilter_2017_Jan_30_1503.csv\n",
      "!!!! ../famousFiltered/experiment/data/8_famFaceFilter_2017_Feb_01_1709.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_1.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_10.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_11.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_12.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_13.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_14.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_15.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_16.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_17.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_18.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_19.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_2.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_20.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_21.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_22.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_23.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_24.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_25.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_26.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_27.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_3.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_4.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_5.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_6.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_7.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_8.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_9.csv\n",
      "!!!! ../famousFiltered/experiment/data/famousFilterClean_jb.csv\n"
     ]
    }
   ],
   "source": [
    "bigDf = makeBigDf(logList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>92.982456</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>92.982456</td>\n",
       "      <td>98.245614</td>\n",
       "      <td>91.228070</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.611111</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>96.969697</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>86.363636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>81.818182</td>\n",
       "      <td>69.696970</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>60.606061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>97.222222</td>\n",
       "      <td>91.666667</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.222222</td>\n",
       "      <td>69.444444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          edges                    gray                    smoo            \n",
       "        context familiarity     context familiarity     context familiarity\n",
       "2    100.000000   92.982456  100.000000   92.982456   98.245614   91.228070\n",
       "4    100.000000  100.000000   98.611111  100.000000  100.000000  100.000000\n",
       "9    100.000000   98.484848   96.969697   98.484848   89.393939   86.363636\n",
       "JB    81.818182   69.696970   93.939394   93.939394   81.818182   60.606061\n",
       "JB*   97.222222   91.666667  100.000000  100.000000   97.222222   69.444444"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigDf.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### now we merge with the logfile version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "altDf = pd.read_csv('../famousFiltered/output/filteredChoiceLog.csv',index_col=[0],header=[0,1])\n",
    "altDf.index = [str(x) for x in altDf.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>type</th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>jbRecog</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.696970</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0.939394</td>\n",
       "      <td>0.818182</td>\n",
       "      <td>0.606061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.916667</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.972222</td>\n",
       "      <td>0.694444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "type        edges                  gray                  smoo            \n",
       "          context familiarity   context familiarity   context familiarity\n",
       "jbRecog                                                                  \n",
       "JB       0.818182    0.696970  0.939394    0.939394  0.818182    0.606061\n",
       "JB*      0.972222    0.916667  1.000000    1.000000  0.972222    0.694444"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "makeParticipant(logList[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.611111</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.611111</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>92.592593</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>92.592593</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>98.148148</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>94.444444</td>\n",
       "      <td>98.148148</td>\n",
       "      <td>98.148148</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.296296</td>\n",
       "      <td>100.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>93.939394</td>\n",
       "      <td>87.878788</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>90.909091</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>89.393939</td>\n",
       "      <td>86.363636</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>96.969697</td>\n",
       "      <td>98.484848</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         smoo                   edges                   gray            \n",
       "      context familiarity     context familiarity    context familiarity\n",
       "5  100.000000  100.000000   98.611111  100.000000  98.611111  100.000000\n",
       "6   92.592593   94.444444   92.592593   94.444444  98.148148  100.000000\n",
       "7   94.444444   98.148148   98.148148  100.000000  96.296296  100.000000\n",
       "8   93.939394   87.878788   90.909091   89.393939  95.454545   90.909091\n",
       "9   89.393939   86.363636  100.000000   98.484848  96.969697   98.484848"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "altDf.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### checking if the two versions give identical results  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "1\n",
      "20\n",
      "21\n",
      "22\n",
      "25\n",
      "2\n",
      "4\n",
      "9\n",
      "JB\n",
      "JB*\n"
     ]
    }
   ],
   "source": [
    "for entry in bigDf.index:\n",
    "    try:\n",
    "        print entry\n",
    "        print bigDf.loc[entry].round(10) == altDf.loc[entry].round(10)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merging the two"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mergeDf = pd.concat([bigDf,altDf]).groupby(level=0).last().sort_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>98.148148</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.296296</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>98.148148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>90.909091</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>87.878788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>96.969697</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>86.363636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>81.818182</td>\n",
       "      <td>69.696970</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>60.606061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>97.222222</td>\n",
       "      <td>91.666667</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.222222</td>\n",
       "      <td>69.444444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          edges                    gray                   smoo            \n",
       "        context familiarity     context familiarity    context familiarity\n",
       "7     98.148148  100.000000   96.296296  100.000000  94.444444   98.148148\n",
       "8     90.909091   89.393939   95.454545   90.909091  93.939394   87.878788\n",
       "9    100.000000   98.484848   96.969697   98.484848  89.393939   86.363636\n",
       "JB    81.818182   69.696970   93.939394   93.939394  81.818182   60.606061\n",
       "JB*   97.222222   91.666667  100.000000  100.000000  97.222222   69.444444"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mergeDf.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mergeDf.to_csv('../famousFiltered/output/filteredChoice.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">edges</th>\n",
       "      <th colspan=\"2\" halign=\"left\">gray</th>\n",
       "      <th colspan=\"2\" halign=\"left\">smoo</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "      <th>context</th>\n",
       "      <th>familiarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>98.148148</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>96.296296</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>94.444444</td>\n",
       "      <td>98.148148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>90.909091</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>95.454545</td>\n",
       "      <td>90.909091</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>87.878788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>96.969697</td>\n",
       "      <td>98.484848</td>\n",
       "      <td>89.393939</td>\n",
       "      <td>86.363636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB</th>\n",
       "      <td>81.818182</td>\n",
       "      <td>69.696970</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>93.939394</td>\n",
       "      <td>81.818182</td>\n",
       "      <td>60.606061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>JB*</th>\n",
       "      <td>97.222222</td>\n",
       "      <td>91.666667</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>97.222222</td>\n",
       "      <td>69.444444</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          edges                    gray                   smoo            \n",
       "        context familiarity     context familiarity    context familiarity\n",
       "7     98.148148  100.000000   96.296296  100.000000  94.444444   98.148148\n",
       "8     90.909091   89.393939   95.454545   90.909091  93.939394   87.878788\n",
       "9    100.000000   98.484848   96.969697   98.484848  89.393939   86.363636\n",
       "JB    81.818182   69.696970   93.939394   93.939394  81.818182   60.606061\n",
       "JB*   97.222222   91.666667  100.000000  100.000000  97.222222   69.444444"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv('../famousFiltered/output/filteredChoice.csv',index_col=[0],header=[0,1]).tail()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}