Kappa_scores.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9508d05-42d1-43d9-84de-5d140c4ffb1f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lets take a look at the kappa scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4c32caad-bb63-4252-8a91-be108d5163e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter, defaultdict\n",
    "from prodigy.components.db import connect\n",
    "from prodigy import set_hashes\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 447,
   "id": "7af5ed83-2eaa-4783-a9db-a41b505a01cd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Counter({'asg64': 300, 'dc2249': 250, 'rat4': 75})\n"
     ]
    }
   ],
   "source": [
    "input_hashes = defaultdict(list)\n",
    "counts = Counter()\n",
    "db = connect()\n",
    "answer_dict = defaultdict(lambda: {'accept':0, 'reject':0, 'ignore':0})\n",
    "examples = db.get_dataset(\"n2c2_assessment\")\n",
    "examples = [set_hashes(eg) for eg in examples]\n",
    "for eg in examples:\n",
    "    session = eg[\"_session_id\"].split(\"-\")[1] if \"_session_id\" in eg else \"n/a\"\n",
    "    answer_dict[session][eg['answer']] +=1\n",
    "    counts[session] += 1\n",
    "print(counts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 448,
   "id": "dab54b65-940d-4bc3-b9b1-c994c638e56d",
   "metadata": {},
   "outputs": [],
   "source": [
    "annotations_df = pd.DataFrame(examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 449,
   "id": "57f68a30-f514-4d7e-b621-c5a8f3a1a91c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"DC = annotations_df[annotations_df['_annotator_id'] == 'n2c2_planSubsection-dc2249']\\nAG = annotations_df[annotations_df['_annotator_id'] == 'n2c2_planSubsection-asg64']\""
      ]
     },
     "execution_count": 449,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''DC = annotations_df[annotations_df['_annotator_id'] == 'n2c2_planSubsection-dc2249']\n",
    "AG = annotations_df[annotations_df['_annotator_id'] == 'n2c2_planSubsection-asg64']'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 450,
   "id": "04c452f1-cb7a-4db8-801d-c218b23e358c",
   "metadata": {},
   "outputs": [],
   "source": [
    "DC = annotations_df[annotations_df['_annotator_id'] == 'n2c2_assessment-dc2249']\n",
    "AG = annotations_df[annotations_df['_annotator_id'] == 'n2c2_assessment-asg64']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 451,
   "id": "99496ef2-e701-4114-9f28-a607a18ba37c",
   "metadata": {},
   "outputs": [],
   "source": [
    "input_hashes = list(set(DC['_input_hash'].tolist() + AG['_input_hash'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 452,
   "id": "dc6de619-92ce-4b53-bf3a-095db8f4816b",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-1596265469,\n",
       " 1392997892,\n",
       " -220794875,\n",
       " 1514401796,\n",
       " 1396486152,\n",
       " 375916553,\n",
       " -1638053366,\n",
       " 1370902027,\n",
       " -1916174838,\n",
       " 840248333,\n",
       " -1879121905,\n",
       " -282588144,\n",
       " -1022002157,\n",
       " -1519498733,\n",
       " -134292458,\n",
       " -2056580585,\n",
       " -1351545830,\n",
       " 153219101,\n",
       " 1223879711,\n",
       " -680471009,\n",
       " -28723168,\n",
       " -1344517086,\n",
       " 571081251,\n",
       " 505572898,\n",
       " 920593957,\n",
       " 663308325,\n",
       " 1966029866,\n",
       " 1385212461,\n",
       " 1536314928,\n",
       " -833489358,\n",
       " 362977331,\n",
       " -1121631694,\n",
       " -372850123,\n",
       " 123433014,\n",
       " 1104646710,\n",
       " -1685422025,\n",
       " -604504013,\n",
       " -1730468293,\n",
       " -1861296065,\n",
       " 1230097472,\n",
       " 1699806273,\n",
       " 12708420,\n",
       " -2010786231,\n",
       " 2101303371,\n",
       " -1322690997,\n",
       " -1592103342,\n",
       " -1668282281,\n",
       " -638123944,\n",
       " 897941592,\n",
       " -1210327460,\n",
       " -17959331,\n",
       " -1745139618,\n",
       " 904082530,\n",
       " -599862173,\n",
       " 2062438506,\n",
       " 2056702061,\n",
       " 1870317680,\n",
       " 722656881,\n",
       " -1053598606,\n",
       " 1030337651,\n",
       " 559190132,\n",
       " 699653748,\n",
       " 1006400116,\n",
       " 1714639991,\n",
       " 1913661560,\n",
       " 1352781433,\n",
       " -1634382214,\n",
       " 885883003,\n",
       " 156165756,\n",
       " 349432957,\n",
       " -1623115143,\n",
       " -490089854,\n",
       " 1509383299,\n",
       " 833883781,\n",
       " 292982921,\n",
       " 698539657,\n",
       " -1069797748,\n",
       " 214019726,\n",
       " 6881423,\n",
       " 851516558,\n",
       " 1118918802,\n",
       " -1697283435,\n",
       " 1339059861,\n",
       " -924291431,\n",
       " -184490854,\n",
       " 908474011,\n",
       " -1072961890,\n",
       " 88129695,\n",
       " 1736715425,\n",
       " 1823839393,\n",
       " -844028248,\n",
       " 1395201193,\n",
       " 1425987242,\n",
       " 2140450477,\n",
       " -700591954,\n",
       " -1498830670,\n",
       " 231419059,\n",
       " -1152828744,\n",
       " 1091719353,\n",
       " 487177406,\n",
       " -1327347010,\n",
       " 825940674,\n",
       " -1535075645,\n",
       " 1701151940,\n",
       " -21733690,\n",
       " 1335093449,\n",
       " -1957543731,\n",
       " -1581395250,\n",
       " 309816015,\n",
       " -1166193456,\n",
       " 1113681105,\n",
       " -1470325033,\n",
       " 349120728,\n",
       " -1874149673,\n",
       " -561455392,\n",
       " 770517217,\n",
       " 771326178,\n",
       " -1958182174,\n",
       " -300054298,\n",
       " -1058721049,\n",
       " -1877941528,\n",
       " 1869569766,\n",
       " -1949835546,\n",
       " -319463701,\n",
       " -276334356,\n",
       " 1374273255,\n",
       " -1056685842,\n",
       " -1000284436,\n",
       " 333226225,\n",
       " -2012114702,\n",
       " 356231411,\n",
       " -1161605389,\n",
       " 560855289,\n",
       " 836307194,\n",
       " 320561660,\n",
       " -1513955586,\n",
       " 578402560,\n",
       " -133820668,\n",
       " 1836821253,\n",
       " -2082683131,\n",
       " -604601081,\n",
       " 1787553545,\n",
       " -72425204,\n",
       " -1944330483,\n",
       " 1088104204,\n",
       " -1066055409,\n",
       " 154946320,\n",
       " -47335657,\n",
       " -255533287,\n",
       " -1601918182,\n",
       " 1156632347,\n",
       " -1327874279,\n",
       " 65755933,\n",
       " -171069154,\n",
       " -1222704353,\n",
       " 228789536,\n",
       " -2103080154,\n",
       " 1264551208,\n",
       " 131576105,\n",
       " 2024132906,\n",
       " -433674454,\n",
       " -1287068374,\n",
       " -1337766611,\n",
       " 210608946,\n",
       " -1883029198,\n",
       " 1364798259,\n",
       " -2066125515,\n",
       " 1970507575,\n",
       " -2017190596,\n",
       " 1430105406,\n",
       " 1095010624,\n",
       " -276025022,\n",
       " -1883379389,\n",
       " 1528612163,\n",
       " -393548475,\n",
       " 129978694,\n",
       " 394848579,\n",
       " -1441842360,\n",
       " 1754923848,\n",
       " -1580309688,\n",
       " -447764661,\n",
       " 1991806796,\n",
       " 515349837,\n",
       " 808617809,\n",
       " 134055762,\n",
       " -185945775,\n",
       " 219140949,\n",
       " -1985110698,\n",
       " -73290923,\n",
       " -746464937,\n",
       " -1454678695,\n",
       " -312719013,\n",
       " 2043229022,\n",
       " -1281554082,\n",
       " 1898075486,\n",
       " 132621426,\n",
       " 1808529762,\n",
       " 1140884322,\n",
       " 1249072993,\n",
       " 2098118503,\n",
       " -1554000534,\n",
       " -1027303573,\n",
       " -2072736405,\n",
       " 1895531882,\n",
       " -338711183,\n",
       " -861152398,\n",
       " 54909300,\n",
       " 485089653,\n",
       " -887582347,\n",
       " 1148563830,\n",
       " -190062726,\n",
       " 107585915,\n",
       " 460330875,\n",
       " 1394502525,\n",
       " -1694209156,\n",
       " -1133612160,\n",
       " -1338731133,\n",
       " 385109895,\n",
       " -1222774904,\n",
       " 1398105481,\n",
       " 29643148,\n",
       " -1842017394,\n",
       " 279561615,\n",
       " -1934052978,\n",
       " 1502624658,\n",
       " -245770861,\n",
       " 142729108,\n",
       " -578053227,\n",
       " 248080276,\n",
       " 339756440,\n",
       " -1060110438,\n",
       " -1280813157,\n",
       " -867412582,\n",
       " -1499431010,\n",
       " -125576800,\n",
       " 585650082,\n",
       " -1414189149,\n",
       " 2115840420,\n",
       " 1575531944,\n",
       " 1030893992,\n",
       " 1908132777,\n",
       " 547202989,\n",
       " 457223086,\n",
       " 514697647,\n",
       " 1152602032,\n",
       " 50953649,\n",
       " -1549901900,\n",
       " -1641491020,\n",
       " -91973708,\n",
       " 38781878,\n",
       " -281737800,\n",
       " -721620556,\n",
       " 536312762,\n",
       " 671820219,\n",
       " 1898317244,\n",
       " 2059898303,\n",
       " -1140264511,\n",
       " -1167719998,\n",
       " -1460100157,\n",
       " 1516558275,\n",
       " 171308996,\n",
       " 142621126,\n",
       " -1138039865,\n",
       " -1808285752,\n",
       " 2088379852,\n",
       " 1066363341,\n",
       " -1424784946,\n",
       " 333522895,\n",
       " 1590477264,\n",
       " 165147600,\n",
       " -1290448944,\n",
       " 577104336,\n",
       " -1086059567,\n",
       " -1614904361,\n",
       " 1139473915,\n",
       " 59380698,\n",
       " -925833248,\n",
       " -1797576735,\n",
       " -499955231,\n",
       " 951696867,\n",
       " -106646046,\n",
       " -1749584411,\n",
       " 988858855,\n",
       " -908525080,\n",
       " -2034571289,\n",
       " -1721007126,\n",
       " 344783851,\n",
       " 609651687,\n",
       " -1519719443,\n",
       " -417475090,\n",
       " -38276114,\n",
       " 2046438386,\n",
       " -1095705614,\n",
       " 995606516,\n",
       " -94332427,\n",
       " -642727944,\n",
       " -1576552455,\n",
       " -1374689286,\n",
       " -576509957,\n",
       " -637262852]"
      ]
     },
     "execution_count": 452,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_hashes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 453,
   "id": "6140edaa-9682-496d-89bb-3301156de94a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "300"
      ]
     },
     "execution_count": 453,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_hashes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "8c60ff9e-7e9b-4e80-b0b9-d979e01600ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "id": "36b40cec-062d-49e5-9622-08b297e9ecf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2,\n",
       "  'end': 7,\n",
       "  'text': 'STEMI',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 1561130997,\n",
       "  'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'label': 'problem'},\n",
       " {'start': 23,\n",
       "  'end': 54,\n",
       "  'token_start': 6,\n",
       "  'token_end': 15,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 123,\n",
       "  'end': 140,\n",
       "  'token_start': 39,\n",
       "  'token_end': 42,\n",
       "  'label': 'event_related_to_problem'},\n",
       " {'start': 340,\n",
       "  'end': 354,\n",
       "  'token_start': 89,\n",
       "  'token_end': 90,\n",
       "  'label': 'complication_related_to_problem'}]"
      ]
     },
     "execution_count": 454,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_AG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "id": "32afccc4-0f6d-4ce9-807b-36f439522bf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import defaultdict\n",
    "assessment_labels = {x:idx for idx, x in enumerate(['primary_problem','secondary_problem','primary_symptom','primary_sign'])}\n",
    "planSubsection_labels = {x:idx for idx, x in enumerate(['problem','complication_related_to_problem','event_related_to_problem','organ_failure_related_to_problem'])}\n",
    "IAA_f1_score = {}\n",
    "saved_annotations = []\n",
    "for h in input_hashes:\n",
    "    AG_df_hash = AG[AG['_input_hash'] == h]\n",
    "    DC_df_hash = DC[DC['_input_hash'] == h]\n",
    "    if len(DC_df_hash) == 0 or len(AG_df_hash) == 0:\n",
    "        continue\n",
    "    AG_df_hash = AG_df_hash.iloc[0]# get spans\n",
    "    DC_df_hash = DC_df_hash.iloc[0]# get spans\n",
    "    # if spans in the df, if spans are not NA and if the len of the span is larger than 0\n",
    "    ann1 = type(AG_df_hash['spans']) == float or len(AG_df_hash['spans']) == 0\n",
    "    ann2 = type(DC_df_hash['spans']) == float or len(DC_df_hash['spans']) == 0\n",
    "    if ann1 or ann2:\n",
    "        if len(saved_annotations) < 6:\n",
    "            saved_annotations.append({'DC': AG_df_hash['spans'], 'AG': DC_df_hash['spans'], 'text': DC_df_hash['text']})\n",
    "        continue\n",
    "    if len(saved_annotations) < 10:\n",
    "        saved_annotations.append({'DC': AG_df_hash['spans'], 'AG': DC_df_hash['spans'], 'text': DC_df_hash['text']})\n",
    "    lst_AG = AG_df_hash['spans']\n",
    "    lst_DC = DC_df_hash['spans']\n",
    "    # extract the list elements that have overlap (This may cause some issue where an annotation from a1 is overlapped 2 times by a2)\n",
    "    # I am going to just count it as another y value\n",
    "    # so what we want to do is capture the intervals for the spans, then check to see if the spans are different. in 3 ways\n",
    "    #   i) binary absolute match (EXACT match on spans, no match), \n",
    "    #  ii) any overlap (if there is any overlap then its an agreement) and \n",
    "    # iii) weighted overlap by jaccard distance/character numbers (amount of overlap is > 50%)\n",
    "    spans_AG = [[x['start'], x['end'], {'AG': assessment_labels[x['label']]}] for x in lst_AG]\n",
    "    spans_DC = [[x['start'], x['end'], {'DC': assessment_labels[x['label']]}] for x in lst_DC]\n",
    "    # now we can update the annotator-label dict with both annotators on the merged\n",
    "    spans = spans_AG + spans_DC\n",
    "    if len(spans) == 0:# in the event the spans are empty\n",
    "        continue\n",
    "    spans.sort(key=lambda x: x[0])# sort them by the first element\n",
    "    merged = []\n",
    "    for interval in spans:\n",
    "        # if the list of merged intervals is empty or if the current\n",
    "        # interval does not overlap with the previous, simply append it.\n",
    "        if not merged or merged[-1][1] < interval[0]:\n",
    "            merged.append(interval)\n",
    "        else:\n",
    "        # otherwise, there is overlap, so we merge the current and previous\n",
    "        # intervals.\n",
    "            merged[-1][1] = max(merged[-1][1], interval[1])\n",
    "            merged[-1][2].update(interval[2])\n",
    "    lst = []\n",
    "    for x in merged:\n",
    "        x[2]['interval'] = [x[0], x[1]]\n",
    "        lst.append(x[2])\n",
    "    # test code, if we got here then we should save last dfs\n",
    "    last_AG = AG_df_hash\n",
    "    last_DC = DC_df_hash\n",
    "    tmp_df = pd.DataFrame(lst).set_index('interval').fillna(-1)\n",
    "    IAA_f1_score[h] = f1_score(tmp_df['AG'], tmp_df['DC'], average='weighted')\n",
    "\n",
    "    '''for ag_idx, ag in enumerate(lst_AG):\n",
    "        for dc_idx, dc in enumerate(lst_DC):\n",
    "            s1 = ag['start']\n",
    "            s2 = dc['start']\n",
    "            e1 = ag['end']\n",
    "            e2 = dc['end']\n",
    "            # any overlap\n",
    "            if (s1 <= e2) and (s2 <= e1):\n",
    "                matches[h]['binary_absolute'].append(True if s1 == s2 and e1 == e2 else False)\n",
    "                matches[h]['any_overlap'].append(True) # true because we are in this if statement\n",
    "                matches[h]['levenshtein_distance'].append(nltk.edit_distance(AG_df_hash.iloc[ag_idx,]['text'][s1:e1+1], DC_df_hash.iloc[dc_idx]['text'][s2:e2+1]))\n",
    "                matches[h]['label_match'].append([lst_AG[ag_idx]['label'], lst_DC[dc_idx]['label']])\n",
    "                matches[h]['annotator_a_span'].append(ag)\n",
    "                matches[h]['annotator_b_span'].append(dc)'''\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 490,
   "id": "2336bf2a-98d2-4ba3-a080-10186aaf12d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ALTERED MENTAL STATUS (NOT DELIRIUM)\n",
      " .H/O CARDIOMYOPATHY, OTHER\n",
      " RESPIRATORY FAILURE, ACUTE (NOT ARDS/[**Doctor Last Name 76**])\n",
      "   This is a 75 yo M with [**Last Name (LF) 7287**], [**First Name3 (LF) **] 15%, afib on coumadin, s/p pacer\n",
      "   placement, admitted from [**Hospital1 **] with RLL pna and respiratory distress.\n",
      "Leukocytosis.  Likely secondary to SBO, but will evaluate for\n",
      "   infectious sources.  Also has infiltrate on CXR, ?aspiration\n",
      "   pneumonitis versus pneumonia.\n",
      "   - f/u urine, blood, sputum cultures\n",
      "   - monitor fever curve, wbc\n",
      "   - hold off on Abx for now, but consider if febrile or has worsening\n",
      "   respiratory symptoms\n",
      "Ms [**Known lastname 11751**] is a 45 yo woman with pmh of DM1, CAD s/p CABG and PCI,\n",
      "   ESRD s/p living-related kidney transplant in [**10-31**], and recent\n",
      "   hospitalization for septic arthritis here with acute renal failure,\n",
      "   acute diastolic heart failure with newly diagnosed severe MR and\n",
      "   pulmonary hypertension.\n",
      "84 yoM with a history of CAD, ischemic cardiomyopathy, PVD and [**Hospital 2092**]\n",
      "   transferred to unit with septic shock.\n",
      "This [**Age over 90 **]M with CAD, dementia, afib, and prior strokes presents with\n",
      "   hypotension and respiratory distress.\n",
      "Assesment: 71 y/o M with h/o recent right MCA CVA stroke hospital\n",
      "   course c/b bilateral pulmonary emboli, VAP s/p treatment with zosyn,\n",
      "   now admitted with hemorrhagic 1.1cm focus in right temporal lobe and\n",
      "   fever and hypotension with sepsis as likely etiology\n",
      "73 year old male with CAD, aortic stenosis, VT/VF arrest s/p ICD and\n",
      "   ablation, CHF (EF 20%) s/p BiV pacer, Afib on coumadin, CRI who\n",
      "   presents after recent discharge with multiple episodes of VT.\n",
      "71 year old male with crypotogenic cirrhosis, HCC s/p cyberknife,\n",
      "   started on lasix and spironolactone in [**Month (only) 2877**] for hepatic\n",
      "   encephalopathy and worsening edema and ascites with reported 25lb\n",
      "   weight loss and resolution of peripheral edema and improvement in\n",
      "   abdominal girth who presents with 1 week of weakness, found to have\n",
      "   hypotension and hyponatremia.\n",
      "22yo woman with h/o IDDM admitted with DKA most likely due to poor\n",
      "   medication compliance.\n",
      "56yo man w/ hx of MVP, now presenting w/ fevers, chills and malaise, w/\n",
      "   cough prod of clr/bld streaked sputum and worsening MR.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[None, None, None, None, None, None, None, None, None, None]"
      ]
     },
     "execution_count": 490,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "[print(x['text']) for x in saved_annotations]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "id": "a2de4fd5-9dbd-4f2f-9201-dbf0bb0beb40",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('random_10_samples_n2c2.json', 'w') as outfile:\n",
    "    json.dump(saved_annotations, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 477,
   "id": "706e342b-9916-4c62-8227-23633c8d27ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6200941620141693"
      ]
     },
     "execution_count": 477,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(list(IAA_f1_score.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "id": "599ce54f-5178-4923-b2b1-84a69c919f81",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2,\n",
       "  'end': 7,\n",
       "  'text': 'STEMI',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 1561130997,\n",
       "  'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'label': 'problem'},\n",
       " {'start': 23,\n",
       "  'end': 54,\n",
       "  'token_start': 6,\n",
       "  'token_end': 15,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 123,\n",
       "  'end': 140,\n",
       "  'token_start': 39,\n",
       "  'token_end': 42,\n",
       "  'label': 'event_related_to_problem'},\n",
       " {'start': 340,\n",
       "  'end': 354,\n",
       "  'token_start': 89,\n",
       "  'token_end': 90,\n",
       "  'label': 'complication_related_to_problem'}]"
      ]
     },
     "execution_count": 440,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_AG['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 442,
   "id": "7f0576fd-acd2-47ad-90b6-6845459f4168",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2,\n",
       "  'end': 7,\n",
       "  'text': 'STEMI',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 1561130997,\n",
       "  'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'label': 'problem'},\n",
       " {'start': 10,\n",
       "  'end': 54,\n",
       "  'token_start': 4,\n",
       "  'token_end': 15,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 57,\n",
       "  'end': 78,\n",
       "  'token_start': 18,\n",
       "  'token_end': 24,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 81,\n",
       "  'end': 92,\n",
       "  'token_start': 27,\n",
       "  'token_end': 31,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 159,\n",
       "  'end': 208,\n",
       "  'token_start': 48,\n",
       "  'token_end': 55,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 211,\n",
       "  'end': 252,\n",
       "  'token_start': 58,\n",
       "  'token_end': 66,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 255,\n",
       "  'end': 328,\n",
       "  'token_start': 69,\n",
       "  'token_end': 82,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 331,\n",
       "  'end': 362,\n",
       "  'token_start': 85,\n",
       "  'token_end': 92,\n",
       "  'label': 'event_related_to_problem'},\n",
       " {'start': 367,\n",
       "  'end': 403,\n",
       "  'token_start': 95,\n",
       "  'token_end': 97,\n",
       "  'label': 'event_related_to_problem'}]"
      ]
     },
     "execution_count": 442,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "last_DC['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 441,
   "id": "79856940-2a6a-40e0-bdc7-48b9abdc2b9b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "619    [{'start': 0, 'end': 3, 'token_start': 0, 'tok...\n",
       "Name: spans, dtype: object"
      ]
     },
     "execution_count": 441,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG_df_hash['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "id": "3590bdee-c371-4d51-8049-327a501531dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Series([], Name: spans, dtype: object)"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC_df_hash['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 428,
   "id": "875854ca-4859-422f-bcb0-785a73cb415b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AG</th>\n",
       "      <th>DC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interval</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>[2, 7]</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[10, 54]</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[57, 78]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[81, 92]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[123, 140]</th>\n",
       "      <td>2.0</td>\n",
       "      <td>-1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[159, 208]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[211, 252]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[255, 328]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[331, 362]</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[367, 403]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             AG   DC\n",
       "interval            \n",
       "[2, 7]      0.0  0.0\n",
       "[10, 54]    1.0  1.0\n",
       "[57, 78]   -1.0  1.0\n",
       "[81, 92]   -1.0  1.0\n",
       "[123, 140]  2.0 -1.0\n",
       "[159, 208] -1.0  1.0\n",
       "[211, 252] -1.0  1.0\n",
       "[255, 328] -1.0  1.0\n",
       "[331, 362]  1.0  2.0\n",
       "[367, 403] -1.0  2.0"
      ]
     },
     "execution_count": 428,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 406,
   "id": "3a43b7df-8e3e-460f-8390-1a30faaa9d6e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "text             # HIV - CD4 300s, VL undetectable.  concern fo...\n",
       "_input_hash                                              651456513\n",
       "_task_hash                                              -111059571\n",
       "_is_binary                                                   False\n",
       "tokens           [{'text': '#', 'start': 0, 'end': 1, 'id': 0, ...\n",
       "_view_id                                                ner_manual\n",
       "spans            [{'start': 2, 'end': 5, 'text': 'HIV', 'source...\n",
       "answer                                                      accept\n",
       "_timestamp                                              1655754986\n",
       "_annotator_id                           n2c2_planSubsection-dc2249\n",
       "_session_id                             n2c2_planSubsection-dc2249\n",
       "Name: 379, dtype: object"
      ]
     },
     "execution_count": 406,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC_df_hash"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 405,
   "id": "6d6bcb4e-f148-4a1d-b8a6-02d1b6112c29",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([False, False, False, False])"
      ]
     },
     "execution_count": 405,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isna(DC_df_hash['spans'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 404,
   "id": "bb550f0b-399b-4202-b037-97abb7ffe754",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 404,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isna(DC.iloc[2]['spans'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 415,
   "id": "2665f52d-60cf-484c-8432-efde31629d22",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 415,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(DC.iloc[2]['spans']) == float"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "22d8597f-5c80-4ada-b15c-7d17608892c2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 380,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(DC_df_hash) == 0 or len(AG_df_hash) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 378,
   "id": "eeeee1b2-993b-45da-afd3-fb1ae5405494",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-378-07a57ab3d153>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[1;33m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDC_df_hash\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'spans'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mbool\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1583\u001b[0m             )\n\u001b[0;32m   1584\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1585\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__nonzero__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1586\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1587\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m__nonzero__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1533\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1534\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__nonzero__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1535\u001b[1;33m         raise ValueError(\n\u001b[0m\u001b[0;32m   1536\u001b[0m             \u001b[1;34mf\"The truth value of a {type(self).__name__} is ambiguous. \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1537\u001b[0m             \u001b[1;34m\"Use a.empty, a.bool(), a.item(), a.any() or a.all().\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
     ]
    }
   ],
   "source": [
    "(pd.isna(DC_df_hash['spans']) == False).bool()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 336,
   "id": "4df0e1bd-41e4-451d-8f87-d84c7e5a0ce4",
   "metadata": {},
   "outputs": [],
   "source": [
    "# if spans in the df, if spans are not NA and if the len of the span is larger than 0\n",
    "ann1 = (('spans' in AG_df_hash) and (pd.isna(AG_df_hash['spans']) == False).bool() and (len(AG_df_hash['spans'].values[0]) > 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "id": "5b193cea-da38-459a-9a86-176f3928165c",
   "metadata": {},
   "outputs": [
    {
     "ename": "ValueError",
     "evalue": "The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all().",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-346-46f51a292fb5>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mann2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'spans'\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mDC_df_hash\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misna\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDC_df_hash\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'spans'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mand\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mDC_df_hash\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'spans'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36mbool\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1583\u001b[0m             )\n\u001b[0;32m   1584\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1585\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__nonzero__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1586\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1587\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m__nonzero__\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m   1533\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mfinal\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1534\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__nonzero__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1535\u001b[1;33m         raise ValueError(\n\u001b[0m\u001b[0;32m   1536\u001b[0m             \u001b[1;34mf\"The truth value of a {type(self).__name__} is ambiguous. \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1537\u001b[0m             \u001b[1;34m\"Use a.empty, a.bool(), a.item(), a.any() or a.all().\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: The truth value of a Series is ambiguous. Use a.empty, a.bool(), a.item(), a.any() or a.all()."
     ]
    }
   ],
   "source": [
    "ann2 = (('spans' in DC_df_hash) and (pd.isna(DC_df_hash['spans']) == False).bool() and (len(DC_df_hash['spans'].values[0]) > 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 389,
   "id": "42c0ad2c-ef12-4173-9c7b-877b376d70f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 389,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checks if is na and if the len == 0. if any of these two are true continue as there is no annotation for the sample\n",
    "pd.isna(DC.iloc[2]['spans']) or len(DC.loc[448]['spans']) == 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "id": "e2ab75a5-aa6b-4858-a8a8-484beac12fa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isna(DC.iloc[2]['spans'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "ee198721-1a05-4132-ae16-5092e82de1a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 363,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC.iloc[2]['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "id": "a39e0802-16d0-4625-b54c-fcc75ee76805",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 348,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC_df_hash['spans'].any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "1fe28e9b-f386-4a91-9142-3c260219eefe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 349,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG_df_hash['spans'].any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f392b23f-95a3-4e09-9b99-094ba4b32dca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "id": "05a5fe9a-5823-4096-8411-9bf733cf5a25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 339,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ann1 and ann2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 321,
   "id": "af5e40b2-7d46-4561-ae0d-bf5e73714bda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6    True\n",
       "Name: spans, dtype: bool"
      ]
     },
     "execution_count": 321,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.isna(DC_df_hash['spans'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 322,
   "id": "9d386c81-a470-47c3-823c-d130ecacf329",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6    NaN\n",
       "Name: spans, dtype: object"
      ]
     },
     "execution_count": 322,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC_df_hash['spans']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 294,
   "id": "0e0e3998-802a-4dca-a447-49de7b119b2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 294,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(AG_df_hash['spans'].values[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "id": "73870ab6-66b1-4cdd-84b2-3523bcb119c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = pd.DataFrame(lst).set_index('interval').fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "id": "63ffd67a-75fa-4ff8-b7f0-26a12afea594",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = tmp_df.set_index('interval')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "id": "2693d6e8-f460-4a86-b09c-0519a3702f8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp_df = tmp_df.fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 256,
   "id": "5f354f4c-e101-4289-b509-299c1ccabd39",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AG</th>\n",
       "      <th>DC</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>interval</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>[2, 5]</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[8, 16]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[18, 33]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>[36, 64]</th>\n",
       "      <td>-1.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           AG  DC\n",
       "interval         \n",
       "[2, 5]    0.0   0\n",
       "[8, 16]  -1.0   1\n",
       "[18, 33] -1.0   1\n",
       "[36, 64] -1.0   1"
      ]
     },
     "execution_count": 256,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "id": "1596b779-e1d7-445e-a52c-48ceb580d67f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.25"
      ]
     },
     "execution_count": 252,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1_score(tmp_df['AG'], tmp_df['DC'], average='weighted')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "id": "aa66398e-5a89-4f69-a886-5495d436735c",
   "metadata": {},
   "outputs": [],
   "source": [
    "x[2]['interval'] = [x[1], x[2]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "79796e0f-46a9-4727-8fb7-e5d7a249cc05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cs</th>\n",
       "      <th>ce</th>\n",
       "      <th>AG</th>\n",
       "      <th>DC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>problem</td>\n",
       "      <td>problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>33</td>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>64</td>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cs  ce       AG                               DC\n",
       "0   2   5  problem                          problem\n",
       "1   8  16      NaN  complication_related_to_problem\n",
       "2  18  33      NaN  complication_related_to_problem\n",
       "3  36  64      NaN  complication_related_to_problem"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = [{'cs':2, 'ce':5, 'AG': 'problem', 'DC': 'problem'},\n",
    " {'cs':8, 'ce':16, 'DC': 'complication_related_to_problem'},\n",
    " {'cs':18, 'ce':33, 'DC': 'complication_related_to_problem'},\n",
    " {'cs':36, 'ce':64, 'DC': 'complication_related_to_problem'}]\n",
    "pd.DataFrame(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "id": "086a6318-84e8-43fc-8552-e07ea0910cda",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AG</th>\n",
       "      <th>DC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>problem</td>\n",
       "      <td>problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>complication_related_to_problem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        AG                               DC\n",
       "0  problem                          problem\n",
       "1      NaN  complication_related_to_problem\n",
       "2      NaN  complication_related_to_problem\n",
       "3      NaN  complication_related_to_problem"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame([x[2] for x in merged])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "id": "1f17baa4-928a-4eff-b68f-e8e18d56d2fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>{'AG': 'problem', 'DC': 'problem'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>{'DC': 'complication_related_to_problem'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>33</td>\n",
       "      <td>{'DC': 'complication_related_to_problem'}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>64</td>\n",
       "      <td>{'DC': 'complication_related_to_problem'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    0   1                                          2\n",
       "0   2   5         {'AG': 'problem', 'DC': 'problem'}\n",
       "1   8  16  {'DC': 'complication_related_to_problem'}\n",
       "2  18  33  {'DC': 'complication_related_to_problem'}\n",
       "3  36  64  {'DC': 'complication_related_to_problem'}"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "id": "9826fb31-915d-4397-a64e-1a58adc2aea6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "DC_test = ['problem', 'complication_related_to_problem', 'complication_related_to_problem', 'complication', 'complication']\n",
    "AG_test = ['problem', 'problem', 'complication_related_to_problem', 'complication', 'complication']\n",
    "cm = confusion_matrix(DC_test, AG_test, labels=['problem', 'complication_related_to_problem', 'complication'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "id": "ea381dca-2a40-4a15-927e-77761e5bc588",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1, 0, 0],\n",
       "       [1, 1, 0],\n",
       "       [0, 0, 2]], dtype=int64)"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "id": "c12d7799-dda2-4a2e-80eb-91415002eff8",
   "metadata": {},
   "outputs": [],
   "source": [
    "sum_records = sum(sum(cm))\n",
    "sum_agreement = np.trace(cm)\n",
    "P_0 = sum_agreement/sum_records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "id": "000ca842-9a07-477f-97bd-9e41ce86b6d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 0], dtype=int64)"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cm[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "id": "0e652baf-74a0-4ece-8438-4baf54464d6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "DC_annotation_sum = np.sum(cm, axis=0)\n",
    "AG_annotation_sum = np.sum(cm, axis=1)\n",
    "P_e = []\n",
    "for dc, ag in zip(DC_annotation_sum, AG_annotation_sum):\n",
    "    P_e.append((dc/sum_records) * (ag/sum_records))\n",
    "P_e = sum(P_e)\n",
    "K = (P_0-P_e)/(1-P_e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "id": "b043c411-be7f-4b1d-a5be-14653aa9c8fd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7058823529411765"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "42b63e51-31c3-4f95-abaa-64ab8cb74cff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2,\n",
       "  'end': 5,\n",
       "  'text': 'HIV',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 651456513,\n",
       "  'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'label': 'problem'},\n",
       " {'start': 8,\n",
       "  'end': 16,\n",
       "  'token_start': 3,\n",
       "  'token_end': 4,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 18,\n",
       "  'end': 33,\n",
       "  'token_start': 6,\n",
       "  'token_end': 7,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 36,\n",
       "  'end': 64,\n",
       "  'token_start': 10,\n",
       "  'token_end': 13,\n",
       "  'label': 'complication_related_to_problem'}]"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_DC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "id": "ec47aa78-890d-4b76-bd14-5f0a85122f71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([list([{'token_start': 1, 'token_end': 1, 'start': 2, 'end': 5, 'text': 'HIV', 'label': 'problem', 'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/', 'input_hash': 651456513}])],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG_df_hash['spans'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "69fb88b0-2095-432e-9aee-d5833bd582b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'start': 2,\n",
       "  'end': 5,\n",
       "  'text': 'HIV',\n",
       "  'label': 'problem',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 651456513}]"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_AG"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "4fc41147-7dbf-4e6e-a709-b048ef741363",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2,\n",
       "  'end': 5,\n",
       "  'text': 'HIV',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 651456513,\n",
       "  'token_start': 1,\n",
       "  'token_end': 1,\n",
       "  'label': 'problem'},\n",
       " {'start': 8,\n",
       "  'end': 16,\n",
       "  'token_start': 3,\n",
       "  'token_end': 4,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 18,\n",
       "  'end': 33,\n",
       "  'token_start': 6,\n",
       "  'token_end': 7,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 36,\n",
       "  'end': 64,\n",
       "  'token_start': 10,\n",
       "  'token_end': 13,\n",
       "  'label': 'complication_related_to_problem'}]"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lst_DC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "57e90887-d1c4-4fad-b0a0-7e0497545d3c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77ed0a41-4148-48f3-acc8-b1af2f88a701",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "3b84f433-cd51-4971-a545-cde9bada5932",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'# h/o PE, on coumadin: Remote chance that a transient PE could explain\\n   current presentation of hypoxia and hypotension.\\n   -continue coumadin with goal INR [**1-30**]'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG[AG['_input_hash'] == 1760190987]['text'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "7e407a87-3f50-4251-af4b-4f41d1530ffe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>_input_hash</th>\n",
       "      <th>_task_hash</th>\n",
       "      <th>_is_binary</th>\n",
       "      <th>tokens</th>\n",
       "      <th>_view_id</th>\n",
       "      <th>spans</th>\n",
       "      <th>answer</th>\n",
       "      <th>_timestamp</th>\n",
       "      <th>_annotator_id</th>\n",
       "      <th>_session_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td># h/o PE, on coumadin: Remote chance that a tr...</td>\n",
       "      <td>1760190987</td>\n",
       "      <td>-1754486048</td>\n",
       "      <td>False</td>\n",
       "      <td>[{'text': '#', 'start': 0, 'end': 1, 'id': 0, ...</td>\n",
       "      <td>ner_manual</td>\n",
       "      <td>[{'start': 98, 'end': 105, 'token_start': 21, ...</td>\n",
       "      <td>accept</td>\n",
       "      <td>1655226936</td>\n",
       "      <td>n2c2_planSubsection-asg64</td>\n",
       "      <td>n2c2_planSubsection-asg64</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  text  _input_hash  \\\n",
       "175  # h/o PE, on coumadin: Remote chance that a tr...   1760190987   \n",
       "\n",
       "     _task_hash  _is_binary  \\\n",
       "175 -1754486048       False   \n",
       "\n",
       "                                                tokens    _view_id  \\\n",
       "175  [{'text': '#', 'start': 0, 'end': 1, 'id': 0, ...  ner_manual   \n",
       "\n",
       "                                                 spans  answer  _timestamp  \\\n",
       "175  [{'start': 98, 'end': 105, 'token_start': 21, ...  accept  1655226936   \n",
       "\n",
       "                 _annotator_id                _session_id  \n",
       "175  n2c2_planSubsection-asg64  n2c2_planSubsection-asg64  "
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG[AG['_input_hash'] == 1760190987]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "3cddd92c-06bc-4f92-9c87-1ce067638015",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 98,\n",
       "  'end': 105,\n",
       "  'token_start': 21,\n",
       "  'token_end': 21,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 110,\n",
       "  'end': 121,\n",
       "  'token_start': 23,\n",
       "  'token_end': 23,\n",
       "  'label': 'complication_related_to_problem'}]"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "AG[AG['_input_hash'] == 1760190987]['spans'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "71d82371-76fe-45c5-bb49-fef9a7e4d9cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'# h/o PE, on coumadin: Remote chance that a transient PE could explain\\n   current presentation of hypoxia and hypotension.\\n   -continue coumadin with goal INR [**1-30**]'"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC[DC['_input_hash'] == 1760190987]['text'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "e9af00ad-6311-44c4-9a5e-3b09f5e79983",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'start': 2, 'end': 8, 'token_start': 1, 'token_end': 4, 'label': 'problem'},\n",
       " {'start': 13,\n",
       "  'end': 21,\n",
       "  'token_start': 7,\n",
       "  'token_end': 7,\n",
       "  'label': 'complication_related_to_problem'},\n",
       " {'start': 98,\n",
       "  'end': 105,\n",
       "  'text': 'hypoxia',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 1760190987,\n",
       "  'token_start': 21,\n",
       "  'token_end': 21,\n",
       "  'label': 'event_related_to_problem'},\n",
       " {'start': 110,\n",
       "  'end': 121,\n",
       "  'text': 'hypotension',\n",
       "  'source': '/home/kl533/n2c2/models/plan_subsection_model/model-best-fixed/',\n",
       "  'input_hash': 1760190987,\n",
       "  'token_start': 23,\n",
       "  'token_end': 23,\n",
       "  'label': 'event_related_to_problem'},\n",
       " {'start': 126,\n",
       "  'end': 158,\n",
       "  'token_start': 26,\n",
       "  'token_end': 30,\n",
       "  'label': 'event_related_to_problem'}]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "DC[DC['_input_hash'] == 1760190987]['spans'].values[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "a758e3c5-35ba-40cc-9388-0eab8c2d0121",
   "metadata": {},
   "outputs": [
    {
     "ename": "IndexError",
     "evalue": "list index out of range",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-51-22a56fb01ebc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mAG\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miloc\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'spans'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mIndexError\u001b[0m: list index out of range"
     ]
    }
   ],
   "source": [
    "AG.iloc[1]['spans'][4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a187fcc5-5fb6-485b-81a1-833d0db57f0c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}