remove_outliers.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## remove_outliers \n",
    "\n",
    "Goal is to remove any request submitted before the shift starts, or after the shift ends. This should only be a problem with past data, as anything newer has the room password feature."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run labqueue_common.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = preprocess_dates_in_csv(pd.read_csv(\"labqueue_anon.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'Unnamed: 0.1', 'course', 'description', 'is_open',\n",
       "       'location', 'pk', 'time_accepted', 'time_closed', 'time_created',\n",
       "       ...\n",
       "       'sem_week_6', 'sem_week_fallspringbreak', 'sem_week_7', 'sem_week_8',\n",
       "       'sem_week_9', 'sem_week_10', 'sem_week_11', 'sem_week_12',\n",
       "       'sem_week_13', 'sem_week_14'],\n",
       "      dtype='object', length=103)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>course</th>\n",
       "      <th>description</th>\n",
       "      <th>is_open</th>\n",
       "      <th>location</th>\n",
       "      <th>pk</th>\n",
       "      <th>time_accepted</th>\n",
       "      <th>time_closed</th>\n",
       "      <th>time_created</th>\n",
       "      <th>...</th>\n",
       "      <th>sem_week_6</th>\n",
       "      <th>sem_week_fallspringbreak</th>\n",
       "      <th>sem_week_7</th>\n",
       "      <th>sem_week_8</th>\n",
       "      <th>sem_week_9</th>\n",
       "      <th>sem_week_10</th>\n",
       "      <th>sem_week_11</th>\n",
       "      <th>sem_week_12</th>\n",
       "      <th>sem_week_13</th>\n",
       "      <th>sem_week_14</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>TOY decode program</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8</td>\n",
       "      <td>2017-10-24 19:11:00</td>\n",
       "      <td>2017-10-24 19:45:00</td>\n",
       "      <td>2017-10-24 19:10:00</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>cos126</td>\n",
       "      <td>Program not printing output.</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9</td>\n",
       "      <td>2017-10-24 19:16:00</td>\n",
       "      <td>2017-10-24 19:55:00</td>\n",
       "      <td>2017-10-24 19:16:00</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>cos126</td>\n",
       "      <td>TOY</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10</td>\n",
       "      <td>2017-10-24 19:26:00</td>\n",
       "      <td>2017-10-24 19:40:00</td>\n",
       "      <td>2017-10-24 19:25:00</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>cos217</td>\n",
       "      <td>midterms</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>2017-10-24 19:33:00</td>\n",
       "      <td>2017-10-24 20:24:00</td>\n",
       "      <td>2017-10-24 19:33:00</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>cos217</td>\n",
       "      <td>midtermszn</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>2017-10-24 19:33:00</td>\n",
       "      <td>2017-10-24 19:52:00</td>\n",
       "      <td>2017-10-24 19:33:00</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Unnamed: 0  Unnamed: 0.1  course                   description  \\\n",
       "Unnamed: 0                                                                   \n",
       "0                    0             0  cos126            TOY decode program   \n",
       "1                    1             1  cos126  Program not printing output.   \n",
       "2                    2             2  cos126                           TOY   \n",
       "3                    3             3  cos217                      midterms   \n",
       "4                    4             4  cos217                    midtermszn   \n",
       "\n",
       "            is_open location  pk       time_accepted         time_closed  \\\n",
       "Unnamed: 0                                                                 \n",
       "0             False      NaN   8 2017-10-24 19:11:00 2017-10-24 19:45:00   \n",
       "1             False      NaN   9 2017-10-24 19:16:00 2017-10-24 19:55:00   \n",
       "2             False      NaN  10 2017-10-24 19:26:00 2017-10-24 19:40:00   \n",
       "3             False      NaN  11 2017-10-24 19:33:00 2017-10-24 20:24:00   \n",
       "4             False      NaN  12 2017-10-24 19:33:00 2017-10-24 19:52:00   \n",
       "\n",
       "                  time_created  ... sem_week_6 sem_week_fallspringbreak  \\\n",
       "Unnamed: 0                      ...                                       \n",
       "0          2017-10-24 19:10:00  ...          1                        0   \n",
       "1          2017-10-24 19:16:00  ...          1                        0   \n",
       "2          2017-10-24 19:25:00  ...          1                        0   \n",
       "3          2017-10-24 19:33:00  ...          1                        0   \n",
       "4          2017-10-24 19:33:00  ...          1                        0   \n",
       "\n",
       "            sem_week_7  sem_week_8  sem_week_9  sem_week_10  sem_week_11  \\\n",
       "Unnamed: 0                                                                 \n",
       "0                    0           0           0            0            0   \n",
       "1                    0           0           0            0            0   \n",
       "2                    0           0           0            0            0   \n",
       "3                    0           0           0            0            0   \n",
       "4                    0           0           0            0            0   \n",
       "\n",
       "            sem_week_12  sem_week_13  sem_week_14  \n",
       "Unnamed: 0                                         \n",
       "0                     0            0            0  \n",
       "1                     0            0            0  \n",
       "2                     0            0            0  \n",
       "3                     0            0            0  \n",
       "4                     0            0            0  \n",
       "\n",
       "[5 rows x 103 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>course</th>\n",
       "      <th>description</th>\n",
       "      <th>is_open</th>\n",
       "      <th>location</th>\n",
       "      <th>pk</th>\n",
       "      <th>time_accepted</th>\n",
       "      <th>time_closed</th>\n",
       "      <th>time_created</th>\n",
       "      <th>...</th>\n",
       "      <th>sem_week_6</th>\n",
       "      <th>sem_week_fallspringbreak</th>\n",
       "      <th>sem_week_7</th>\n",
       "      <th>sem_week_8</th>\n",
       "      <th>sem_week_9</th>\n",
       "      <th>sem_week_10</th>\n",
       "      <th>sem_week_11</th>\n",
       "      <th>sem_week_12</th>\n",
       "      <th>sem_week_13</th>\n",
       "      <th>sem_week_14</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4120</th>\n",
       "      <td>4120.0</td>\n",
       "      <td>4120.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>randomized queue</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4129.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 14:52:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4122</th>\n",
       "      <td>4122.0</td>\n",
       "      <td>4122.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>deques and randomized queues assignment - need...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4131.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 14:57:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4123</th>\n",
       "      <td>4123.0</td>\n",
       "      <td>4123.0</td>\n",
       "      <td>cos217</td>\n",
       "      <td>problem with exit_failure</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4132.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:04:00</td>\n",
       "      <td>2018-02-17 14:58:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4124</th>\n",
       "      <td>4124.0</td>\n",
       "      <td>4124.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>sever\\r\\ndeque</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4133.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:19:00</td>\n",
       "      <td>2018-02-17 14:59:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4125</th>\n",
       "      <td>4125.0</td>\n",
       "      <td>4125.0</td>\n",
       "      <td>cos217</td>\n",
       "      <td>server</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4134.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:36:00</td>\n",
       "      <td>2018-02-17 14:59:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4126</th>\n",
       "      <td>4126.0</td>\n",
       "      <td>4126.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>Password: server</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4135.0</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>2018-02-17 15:10:00</td>\n",
       "      <td>2018-02-17 14:59:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4127</th>\n",
       "      <td>4127.0</td>\n",
       "      <td>4127.0</td>\n",
       "      <td>cos217</td>\n",
       "      <td>help w spaces and new line character</td>\n",
       "      <td>0.0</td>\n",
       "      <td>122</td>\n",
       "      <td>4136.0</td>\n",
       "      <td>2018-02-17 15:02:00</td>\n",
       "      <td>2018-02-17 15:31:00</td>\n",
       "      <td>2018-02-17 15:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4129</th>\n",
       "      <td>4129.0</td>\n",
       "      <td>4129.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>nullpointerexception</td>\n",
       "      <td>0.0</td>\n",
       "      <td>122</td>\n",
       "      <td>4138.0</td>\n",
       "      <td>2018-02-17 15:04:00</td>\n",
       "      <td>2018-02-17 15:21:00</td>\n",
       "      <td>2018-02-17 15:00:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4130</th>\n",
       "      <td>4130.0</td>\n",
       "      <td>4130.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>PW: Server. Failing several tests relating to ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4139.0</td>\n",
       "      <td>2018-02-17 15:04:00</td>\n",
       "      <td>2018-02-17 15:23:00</td>\n",
       "      <td>2018-02-17 15:01:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4131</th>\n",
       "      <td>4131.0</td>\n",
       "      <td>4131.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>server\\r\\n\\r\\nhelp with starting deques assign...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4140.0</td>\n",
       "      <td>2018-02-17 15:05:00</td>\n",
       "      <td>2018-02-17 15:16:00</td>\n",
       "      <td>2018-02-17 15:02:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Unnamed: 0  Unnamed: 0.1  course  \\\n",
       "Unnamed: 0                                     \n",
       "4120            4120.0        4120.0  cos226   \n",
       "4122            4122.0        4122.0  cos226   \n",
       "4123            4123.0        4123.0  cos217   \n",
       "4124            4124.0        4124.0  cos226   \n",
       "4125            4125.0        4125.0  cos217   \n",
       "4126            4126.0        4126.0  cos226   \n",
       "4127            4127.0        4127.0  cos217   \n",
       "4129            4129.0        4129.0  cos226   \n",
       "4130            4130.0        4130.0  cos226   \n",
       "4131            4131.0        4131.0  cos226   \n",
       "\n",
       "                                                  description  is_open  \\\n",
       "Unnamed: 0                                                               \n",
       "4120                                         randomized queue      0.0   \n",
       "4122        deques and randomized queues assignment - need...      0.0   \n",
       "4123                                problem with exit_failure      0.0   \n",
       "4124                                           sever\\r\\ndeque      0.0   \n",
       "4125                                                   server      0.0   \n",
       "4126                                         Password: server      0.0   \n",
       "4127                     help w spaces and new line character      0.0   \n",
       "4129                                     nullpointerexception      0.0   \n",
       "4130        PW: Server. Failing several tests relating to ...      0.0   \n",
       "4131        server\\r\\n\\r\\nhelp with starting deques assign...      0.0   \n",
       "\n",
       "           location      pk       time_accepted         time_closed  \\\n",
       "Unnamed: 0                                                            \n",
       "4120            121  4129.0 2018-02-17 15:01:00 2018-02-17 15:01:00   \n",
       "4122            121  4131.0 2018-02-17 15:01:00 2018-02-17 15:01:00   \n",
       "4123            121  4132.0 2018-02-17 15:01:00 2018-02-17 15:04:00   \n",
       "4124            121  4133.0 2018-02-17 15:01:00 2018-02-17 15:19:00   \n",
       "4125            121  4134.0 2018-02-17 15:01:00 2018-02-17 15:36:00   \n",
       "4126            121  4135.0 2018-02-17 15:01:00 2018-02-17 15:10:00   \n",
       "4127            122  4136.0 2018-02-17 15:02:00 2018-02-17 15:31:00   \n",
       "4129            122  4138.0 2018-02-17 15:04:00 2018-02-17 15:21:00   \n",
       "4130            121  4139.0 2018-02-17 15:04:00 2018-02-17 15:23:00   \n",
       "4131            121  4140.0 2018-02-17 15:05:00 2018-02-17 15:16:00   \n",
       "\n",
       "                  time_created  ... sem_week_6 sem_week_fallspringbreak  \\\n",
       "Unnamed: 0                      ...                                       \n",
       "4120       2018-02-17 14:52:00  ...        0.0                      0.0   \n",
       "4122       2018-02-17 14:57:00  ...        0.0                      0.0   \n",
       "4123       2018-02-17 14:58:00  ...        0.0                      0.0   \n",
       "4124       2018-02-17 14:59:00  ...        0.0                      0.0   \n",
       "4125       2018-02-17 14:59:00  ...        0.0                      0.0   \n",
       "4126       2018-02-17 14:59:00  ...        0.0                      0.0   \n",
       "4127       2018-02-17 15:00:00  ...        0.0                      0.0   \n",
       "4129       2018-02-17 15:00:00  ...        0.0                      0.0   \n",
       "4130       2018-02-17 15:01:00  ...        0.0                      0.0   \n",
       "4131       2018-02-17 15:02:00  ...        0.0                      0.0   \n",
       "\n",
       "            sem_week_7  sem_week_8  sem_week_9  sem_week_10  sem_week_11  \\\n",
       "Unnamed: 0                                                                 \n",
       "4120               0.0         0.0         0.0          0.0          0.0   \n",
       "4122               0.0         0.0         0.0          0.0          0.0   \n",
       "4123               0.0         0.0         0.0          0.0          0.0   \n",
       "4124               0.0         0.0         0.0          0.0          0.0   \n",
       "4125               0.0         0.0         0.0          0.0          0.0   \n",
       "4126               0.0         0.0         0.0          0.0          0.0   \n",
       "4127               0.0         0.0         0.0          0.0          0.0   \n",
       "4129               0.0         0.0         0.0          0.0          0.0   \n",
       "4130               0.0         0.0         0.0          0.0          0.0   \n",
       "4131               0.0         0.0         0.0          0.0          0.0   \n",
       "\n",
       "            sem_week_12  sem_week_13  sem_week_14  \n",
       "Unnamed: 0                                         \n",
       "4120                0.0          0.0          0.0  \n",
       "4122                0.0          0.0          0.0  \n",
       "4123                0.0          0.0          0.0  \n",
       "4124                0.0          0.0          0.0  \n",
       "4125                0.0          0.0          0.0  \n",
       "4126                0.0          0.0          0.0  \n",
       "4127                0.0          0.0          0.0  \n",
       "4129                0.0          0.0          0.0  \n",
       "4130                0.0          0.0          0.0  \n",
       "4131                0.0          0.0          0.0  \n",
       "\n",
       "[10 rows x 103 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.where(df[\"isSaturday\"]==1).dropna()[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>course</th>\n",
       "      <th>description</th>\n",
       "      <th>is_open</th>\n",
       "      <th>location</th>\n",
       "      <th>pk</th>\n",
       "      <th>time_accepted</th>\n",
       "      <th>time_closed</th>\n",
       "      <th>time_created</th>\n",
       "      <th>...</th>\n",
       "      <th>sem_week_6</th>\n",
       "      <th>sem_week_fallspringbreak</th>\n",
       "      <th>sem_week_7</th>\n",
       "      <th>sem_week_8</th>\n",
       "      <th>sem_week_9</th>\n",
       "      <th>sem_week_10</th>\n",
       "      <th>sem_week_11</th>\n",
       "      <th>sem_week_12</th>\n",
       "      <th>sem_week_13</th>\n",
       "      <th>sem_week_14</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3926</th>\n",
       "      <td>3926.0</td>\n",
       "      <td>3926.0</td>\n",
       "      <td>cos217</td>\n",
       "      <td>helpp</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>3935.0</td>\n",
       "      <td>2018-02-13 20:57:00</td>\n",
       "      <td>2018-02-13 20:57:00</td>\n",
       "      <td>2018-02-13 20:29:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3945</th>\n",
       "      <td>3945.0</td>\n",
       "      <td>3945.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>creating math.random numbers</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>3954.0</td>\n",
       "      <td>2018-02-13 21:56:00</td>\n",
       "      <td>2018-02-13 21:56:00</td>\n",
       "      <td>2018-02-13 21:45:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3953</th>\n",
       "      <td>3953.0</td>\n",
       "      <td>3953.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>error messages</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>3962.0</td>\n",
       "      <td>2018-02-13 22:16:00</td>\n",
       "      <td>2018-02-13 22:16:00</td>\n",
       "      <td>2018-02-13 22:14:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4003</th>\n",
       "      <td>4003.0</td>\n",
       "      <td>4003.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>assigment 1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4012.0</td>\n",
       "      <td>2018-02-14 21:52:00</td>\n",
       "      <td>2018-02-14 21:52:00</td>\n",
       "      <td>2018-02-14 21:48:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4026</th>\n",
       "      <td>4026.0</td>\n",
       "      <td>4026.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>I'm trying to figure out how to use arrays to ...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>121</td>\n",
       "      <td>4035.0</td>\n",
       "      <td>2018-02-15 19:59:00</td>\n",
       "      <td>2018-02-15 19:59:00</td>\n",
       "      <td>2018-02-15 19:54:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19147</th>\n",
       "      <td>19147.0</td>\n",
       "      <td>19147.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>idk how to do this/I haven't started sorry</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Lewis 122</td>\n",
       "      <td>19302.0</td>\n",
       "      <td>2019-03-24 21:16:00</td>\n",
       "      <td>2019-03-24 21:16:00</td>\n",
       "      <td>2019-03-24 20:50:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19148</th>\n",
       "      <td>19148.0</td>\n",
       "      <td>19148.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>Chuck I need help</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Lewis 122</td>\n",
       "      <td>19303.0</td>\n",
       "      <td>2019-03-24 21:40:00</td>\n",
       "      <td>2019-03-24 21:40:00</td>\n",
       "      <td>2019-03-24 20:59:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19219</th>\n",
       "      <td>19219.0</td>\n",
       "      <td>19219.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>decoding</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Lewis 122</td>\n",
       "      <td>19374.0</td>\n",
       "      <td>2019-03-25 20:34:00</td>\n",
       "      <td>2019-03-25 20:34:00</td>\n",
       "      <td>2019-03-25 20:33:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19340</th>\n",
       "      <td>19340.0</td>\n",
       "      <td>19340.0</td>\n",
       "      <td>cos226</td>\n",
       "      <td>implementing range</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Lewis 121</td>\n",
       "      <td>19495.0</td>\n",
       "      <td>2019-03-28 20:44:00</td>\n",
       "      <td>2019-03-28 20:44:00</td>\n",
       "      <td>2019-03-28 20:37:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19373</th>\n",
       "      <td>19373.0</td>\n",
       "      <td>19373.0</td>\n",
       "      <td>cos126</td>\n",
       "      <td>lfsr</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Lewis 121</td>\n",
       "      <td>19528.0</td>\n",
       "      <td>2019-03-28 22:50:00</td>\n",
       "      <td>2019-03-28 22:50:00</td>\n",
       "      <td>2019-03-28 22:29:00</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1028 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            Unnamed: 0  Unnamed: 0.1  course  \\\n",
       "Unnamed: 0                                     \n",
       "3926            3926.0        3926.0  cos217   \n",
       "3945            3945.0        3945.0  cos126   \n",
       "3953            3953.0        3953.0  cos126   \n",
       "4003            4003.0        4003.0  cos126   \n",
       "4026            4026.0        4026.0  cos126   \n",
       "...                ...           ...     ...   \n",
       "19147          19147.0       19147.0  cos126   \n",
       "19148          19148.0       19148.0  cos126   \n",
       "19219          19219.0       19219.0  cos126   \n",
       "19340          19340.0       19340.0  cos226   \n",
       "19373          19373.0       19373.0  cos126   \n",
       "\n",
       "                                                  description  is_open  \\\n",
       "Unnamed: 0                                                               \n",
       "3926                                                    helpp      0.0   \n",
       "3945                             creating math.random numbers      0.0   \n",
       "3953                                           error messages      0.0   \n",
       "4003                                              assigment 1      0.0   \n",
       "4026        I'm trying to figure out how to use arrays to ...      0.0   \n",
       "...                                                       ...      ...   \n",
       "19147              idk how to do this/I haven't started sorry      0.0   \n",
       "19148                                       Chuck I need help      0.0   \n",
       "19219                                                decoding      0.0   \n",
       "19340                                      implementing range      0.0   \n",
       "19373                                                    lfsr      0.0   \n",
       "\n",
       "             location       pk       time_accepted         time_closed  \\\n",
       "Unnamed: 0                                                               \n",
       "3926              121   3935.0 2018-02-13 20:57:00 2018-02-13 20:57:00   \n",
       "3945              121   3954.0 2018-02-13 21:56:00 2018-02-13 21:56:00   \n",
       "3953              121   3962.0 2018-02-13 22:16:00 2018-02-13 22:16:00   \n",
       "4003              121   4012.0 2018-02-14 21:52:00 2018-02-14 21:52:00   \n",
       "4026              121   4035.0 2018-02-15 19:59:00 2018-02-15 19:59:00   \n",
       "...               ...      ...                 ...                 ...   \n",
       "19147       Lewis 122  19302.0 2019-03-24 21:16:00 2019-03-24 21:16:00   \n",
       "19148       Lewis 122  19303.0 2019-03-24 21:40:00 2019-03-24 21:40:00   \n",
       "19219       Lewis 122  19374.0 2019-03-25 20:34:00 2019-03-25 20:34:00   \n",
       "19340       Lewis 121  19495.0 2019-03-28 20:44:00 2019-03-28 20:44:00   \n",
       "19373       Lewis 121  19528.0 2019-03-28 22:50:00 2019-03-28 22:50:00   \n",
       "\n",
       "                  time_created  ... sem_week_6 sem_week_fallspringbreak  \\\n",
       "Unnamed: 0                      ...                                       \n",
       "3926       2018-02-13 20:29:00  ...        0.0                      0.0   \n",
       "3945       2018-02-13 21:45:00  ...        0.0                      0.0   \n",
       "3953       2018-02-13 22:14:00  ...        0.0                      0.0   \n",
       "4003       2018-02-14 21:48:00  ...        0.0                      0.0   \n",
       "4026       2018-02-15 19:54:00  ...        0.0                      0.0   \n",
       "...                        ...  ...        ...                      ...   \n",
       "19147      2019-03-24 20:50:00  ...        0.0                      1.0   \n",
       "19148      2019-03-24 20:59:00  ...        0.0                      1.0   \n",
       "19219      2019-03-25 20:33:00  ...        0.0                      0.0   \n",
       "19340      2019-03-28 20:37:00  ...        0.0                      0.0   \n",
       "19373      2019-03-28 22:29:00  ...        0.0                      0.0   \n",
       "\n",
       "            sem_week_7  sem_week_8  sem_week_9  sem_week_10  sem_week_11  \\\n",
       "Unnamed: 0                                                                 \n",
       "3926               0.0         0.0         0.0          0.0          0.0   \n",
       "3945               0.0         0.0         0.0          0.0          0.0   \n",
       "3953               0.0         0.0         0.0          0.0          0.0   \n",
       "4003               0.0         0.0         0.0          0.0          0.0   \n",
       "4026               0.0         0.0         0.0          0.0          0.0   \n",
       "...                ...         ...         ...          ...          ...   \n",
       "19147              0.0         0.0         0.0          0.0          0.0   \n",
       "19148              0.0         0.0         0.0          0.0          0.0   \n",
       "19219              1.0         0.0         0.0          0.0          0.0   \n",
       "19340              1.0         0.0         0.0          0.0          0.0   \n",
       "19373              1.0         0.0         0.0          0.0          0.0   \n",
       "\n",
       "            sem_week_12  sem_week_13  sem_week_14  \n",
       "Unnamed: 0                                         \n",
       "3926                0.0          0.0          0.0  \n",
       "3945                0.0          0.0          0.0  \n",
       "3953                0.0          0.0          0.0  \n",
       "4003                0.0          0.0          0.0  \n",
       "4026                0.0          0.0          0.0  \n",
       "...                 ...          ...          ...  \n",
       "19147               0.0          0.0          0.0  \n",
       "19148               0.0          0.0          0.0  \n",
       "19219               0.0          0.0          0.0  \n",
       "19340               0.0          0.0          0.0  \n",
       "19373               0.0          0.0          0.0  \n",
       "\n",
       "[1028 rows x 103 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.where(df[\"time_accepted\"]==df[\"time_closed\"]).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_to_starting_hour = dict()\n",
    "label_to_starting_hour[\"isMonday\"] = 19\n",
    "label_to_starting_hour[\"isTuesday\"] = 19\n",
    "label_to_starting_hour[\"isWednesday\"] = 19\n",
    "label_to_starting_hour[\"isThursday\"] = 19\n",
    "label_to_starting_hour[\"isFriday\"] = 19\n",
    "label_to_starting_hour[\"isSaturday\"] = 15\n",
    "label_to_starting_hour[\"isSunday\"] = 17"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_outliers():\n",
    "    outlier_index = list()\n",
    "    for k, v in label_to_starting_hour.items():\n",
    "        outlier_index.append(df.where(df[k]==1).where(df[\"time_accepted\"]==df[\"time_closed\"]).where(df[\"time_created\"].apply(get_hour) < v).dropna().index.values)\n",
    "    return outlier_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "outliers = get_outliers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_indexes = np.concatenate(outliers).ravel().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop(drop_indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>course</th>\n",
       "      <th>description</th>\n",
       "      <th>is_open</th>\n",
       "      <th>location</th>\n",
       "      <th>pk</th>\n",
       "      <th>time_accepted</th>\n",
       "      <th>time_closed</th>\n",
       "      <th>time_created</th>\n",
       "      <th>...</th>\n",
       "      <th>sem_week_6</th>\n",
       "      <th>sem_week_fallspringbreak</th>\n",
       "      <th>sem_week_7</th>\n",
       "      <th>sem_week_8</th>\n",
       "      <th>sem_week_9</th>\n",
       "      <th>sem_week_10</th>\n",
       "      <th>sem_week_11</th>\n",
       "      <th>sem_week_12</th>\n",
       "      <th>sem_week_13</th>\n",
       "      <th>sem_week_14</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>0 rows × 103 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Unnamed: 0, Unnamed: 0.1, course, description, is_open, location, pk, time_accepted, time_closed, time_created, wait_time, help_time, TA_anon, student_anon, year, month, day, day_created, hour_accepted, minute_accepted, hour_created, minute_created, hour_closed, minute_closed, day_of_week, week_of_year, wait_time_in_minutes, help_time_in_minutes, Color, Marker, cos109, cos126, cos217, cos226, isMonday, isTuesday, isWednesday, isThursday, isFriday, isSaturday, isSunday, woy2, woy3, woy6, woy7, woy8, woy9, woy10, woy11, woy12, woy13, woy14, woy15, woy16, woy17, woy18, woy19, woy20, woy37, woy38, woy39, woy40, woy41, woy42, woy43, woy44, woy45, woy46, woy47, woy48, woy49, woy50, wait_time_90m_before_submit, wait_time_30m_before_submit, help_time_90m_before_submit, help_time_30m_before_submit, num_in_queue_before_us, time_created_floor, TAs_this_hour, day_before_217_due, day_of_217_due, day_before_226_due, day_of_226_due, day_before_126_due, day_of_126_due, assignment_due_next_day, assignment_due_day_of, more_TAs_than_queue, sem_week_1, sem_week_2, sem_week_3, sem_week_4, sem_week_5, sem_week_6, sem_week_fallspringbreak, sem_week_7, sem_week_8, sem_week_9, sem_week_10, sem_week_11, ...]\n",
       "Index: []\n",
       "\n",
       "[0 rows x 103 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.where(df[\"isTuesday\"]==1).where(df[\"time_accepted\"]==df[\"time_closed\"]).where(df[\"time_created\"].apply(get_hour) < 19).dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Additionally, we want to remove specific instances in the test data where the wait time of that individual was more than 2x every other person in that shift's wait time. This could correspond to students repeatedly putting themselves on the queue right after being helped. I manually searched for these pks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "pks_to_remove = [18392, 19049, 18774, 19011, 18974, 19051, 18798, 19210, 19328]\n",
    "df = df.where(~df[\"pk\"].isin(pks_to_remove)).dropna()\n",
    "\n",
    "df.to_csv(\"labqueue_anon_1.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert len(drop_indexes) == 60"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}