t5_threshold.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import numpy as np\n",
    "from sklearn.cluster import KMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "infernece_result_path = './outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction_raw.json'\n",
    "# infernece_result_path = './outputs/eval_ehrsql_mimic3_t5_base_schema__mimic3_valid/prediction_raw.json'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[result] 1122 lines loaded\n"
     ]
    }
   ],
   "source": [
    "num_workers = -1\n",
    "with open(infernece_result_path, 'r') as f:\n",
    "    data = json.load(f)\n",
    "print(f'[result] {len(data)} lines loaded')\n",
    "\n",
    "data_id = []\n",
    "query_real = []\n",
    "query_pred = []\n",
    "entropy = []\n",
    "impossible = []\n",
    "for idx_, line in data.items():\n",
    "    data_id.append(idx_)\n",
    "    query_real.append(line['real'])\n",
    "    query_pred.append(line['pred'])\n",
    "    entropy.append(max(line['sequence_entropy']))\n",
    "    impossible.append(line['is_impossible'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# k-means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_threshold_kmeans(entropy):\n",
    "    \n",
    "    kmeans = KMeans(n_clusters=2, random_state=0).fit(np.expand_dims(entropy, axis=1))\n",
    "    \n",
    "    zero_low = min(np.array(entropy)[kmeans.labels_==0])\n",
    "    zero_high = max(np.array(entropy)[kmeans.labels_==0])\n",
    "    one_low = min(np.array(entropy)[kmeans.labels_==1])\n",
    "    one_high = max(np.array(entropy)[kmeans.labels_==1])\n",
    "    \n",
    "    if one_high > zero_high:\n",
    "        return (zero_high + one_low)/2\n",
    "    else:\n",
    "        return (zero_low + one_high)/2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.744074136018753"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_threshold_kmeans(entropy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mimic3 k-means: 0.744074136018753\n",
    "# mimic3_schema k-means: 0.8807361125946045"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# percentile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.14923561])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.percentile(entropy, q=[67.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# mimic3 67 percentile: 0.14923561\n",
    "# mimic3_schema 67 percentile: 0.09932577"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# choosing a threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T5/abstain_with_entropy.py:18: UserWarning: Threshold value is not set! All predictions are sent to the database.\n",
      "  warnings.warn(\"Threshold value is not set! All predictions are sent to the database.\")\n",
      "{\n",
      "  \"precision_ans\": 67.74,\n",
      "  \"recall_ans\": 100.0,\n",
      "  \"f1_ans\": 80.77,\n",
      "  \"precision_exec\": 65.24,\n",
      "  \"recall_exec\": 96.32,\n",
      "  \"f1_exec\": 77.79\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold -1\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 83.84,\n",
      "  \"recall_ans\": 97.63,\n",
      "  \"f1_ans\": 90.21,\n",
      "  \"precision_exec\": 82.03,\n",
      "  \"recall_exec\": 95.53,\n",
      "  \"f1_exec\": 88.27\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.744074136018753\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 94.81,\n",
      "  \"recall_ans\": 93.82,\n",
      "  \"f1_ans\": 94.31,\n",
      "  \"precision_exec\": 93.88,\n",
      "  \"recall_exec\": 92.89,\n",
      "  \"f1_exec\": 93.39\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.14923561\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# high-precision setting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 95.82,\n",
      "  \"recall_ans\": 93.42,\n",
      "  \"f1_ans\": 94.6,\n",
      "  \"precision_exec\": 95.14,\n",
      "  \"recall_exec\": 92.76,\n",
      "  \"f1_exec\": 93.94\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.1\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 98.46,\n",
      "  \"recall_ans\": 8.42,\n",
      "  \"f1_ans\": 15.52,\n",
      "  \"precision_exec\": 98.46,\n",
      "  \"recall_exec\": 8.42,\n",
      "  \"f1_exec\": 15.52\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.000050\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 97.78,\n",
      "  \"recall_ans\": 5.79,\n",
      "  \"f1_ans\": 10.93,\n",
      "  \"precision_exec\": 97.78,\n",
      "  \"recall_exec\": 5.79,\n",
      "  \"f1_exec\": 10.93\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.000040\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"precision_ans\": 100.0,\n",
      "  \"recall_ans\": 3.68,\n",
      "  \"f1_ans\": 7.11,\n",
      "  \"precision_exec\": 100.0,\n",
      "  \"recall_exec\": 3.68,\n",
      "  \"f1_exec\": 7.11\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "!python T5/abstain_with_entropy.py --infernece_result_path outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid --input_file prediction_raw.json --output_file prediction.json --threshold 0.000030\n",
    "!python evaluate.py --db_path ./dataset/ehrsql/mimic_iii/mimic_iii.db --data_file dataset/ehrsql/mimic_iii/valid.json --pred_file ./outputs/eval_ehrsql_mimic3_t5_base__mimic3_valid/prediction.json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# precision_exec: 100.0 (valid) >= 99.0\n",
    "# set the threshold to 0.000030 and now ready to evaluate on the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}