Upload_to_Hub.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2db774a5-3a8f-43f3-b1ca-bfb7fd896259",
   "metadata": {},
   "outputs": [],
   "source": [
    "from huggingface_hub import notebook_login"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3e7220dc-530a-4bf2-9ac7-098e1c9d605f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "232b0fd00e394460bbfdee83eab2e9d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "VBox(children=(HTML(value='<center> <img\\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.sv…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "notebook_login()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "abb13d6e-7421-440f-9008-4b7f01edf55f",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import GPT2Tokenizer, GPTNeoForSequenceClassification\n",
    "from transformers import DataCollatorWithPadding\n",
    "from transformers import logging\n",
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "from transformers import RobertaForSequenceClassification, BertForSequenceClassification\n",
    "from transformers import TFRobertaForSequenceClassification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d37ac067-b293-44e0-9e10-796a1cf39387",
   "metadata": {},
   "outputs": [],
   "source": [
    "pt_model = RobertaForSequenceClassification.from_pretrained(\"/home/vs428/project/n2c2_models/roberta_large_ner_so/roberta_large_ner_so/outputs/roberta_large/checkpoint-384/\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b555178f-ab47-4e74-8012-fcebd21f7056",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-01-04 00:33:11.668046: W tensorflow/compiler/xla/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcusolver.so.11'; dlerror: libcusolver.so.11: cannot open shared object file: No such file or directory\n",
      "2023-01-04 00:33:11.668201: W tensorflow/core/common_runtime/gpu/gpu_device.cc:1934] Cannot dlopen some GPU libraries. Please make sure the missing libraries mentioned above are installed properly if you would like to use GPU. Follow the guide at https://www.tensorflow.org/install/gpu for how to download and setup the required libraries for your platform.\n",
      "Skipping registering GPU devices...\n",
      "2023-01-04 00:33:11.669328: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaForSequenceClassification: ['roberta.embeddings.position_ids']\n",
      "- This IS expected if you are initializing TFRobertaForSequenceClassification from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing TFRobertaForSequenceClassification from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "All the weights of TFRobertaForSequenceClassification were initialized from the PyTorch model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaForSequenceClassification for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "tf_model = TFRobertaForSequenceClassification.from_pretrained(\"/home/vs428/project/n2c2_models/roberta_large_ner_so/roberta_large_ner_so/outputs/roberta_large/checkpoint-384/\", from_pt=True)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bf247bb7-9f28-4602-9418-e9d5494a7eb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFRobertaForSequenceClassification.\n",
      "\n",
      "All the layers of TFRobertaForSequenceClassification were initialized from the model checkpoint at /home/vs428/project/n2c2_models/roberta_large_ner_so/tf_model.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaForSequenceClassification for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "tf_model = TFRobertaForSequenceClassification.from_pretrained(\"/home/vs428/project/n2c2_models/roberta_large_ner_so/tf_model\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "54b88e7a-cd2a-4db6-a1ab-486d5173d1e4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/vsocrates/n2c2-soap-entailment/commit/0b6bca418737b10e401688d95a38bba2feb6ac0d', commit_message='Upload RobertaForSequenceClassification', commit_description='', oid='0b6bca418737b10e401688d95a38bba2feb6ac0d', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt_model.push_to_hub(\"n2c2-soap-entailment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2ebf836b-1885-48d4-b272-7178c34ae213",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_model.push_to_hub(\"n2c2-soap-entailment\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6abe2925-b20a-40ec-a533-616c6221a474",
   "metadata": {},
   "source": [
    "# Fix Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "47bf1853-6aeb-4e91-8483-8ae5389df7c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "92304a7b106046c88821ceb7b9c48b7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.42G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from transformers import AutoConfig, AutoModelForSequenceClassification\n",
    "\n",
    "label2id = {'Not Relevant':3, 'Neither':2, 'Indirect':1, 'Direct':0}\n",
    "id2label = {v:k for k,v in label2id.items()}\n",
    "# define model checkpoint - can be the same model that you already have on the hub\n",
    "model_ckpt = \"vsocrates/n2c2-soap-entailment\"\n",
    "# define config\n",
    "config = AutoConfig.from_pretrained(model_ckpt, label2id=label2id, id2label=id2label)\n",
    "# load model with config\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_ckpt, config=config)\n",
    "# export model\n",
    "model.save_pretrained(\"/home/vs428/project/n2c2_models/roberta_large_ner_so/roberta_large_ner_so/upload_relabeled\")\n",
    "# \"/home/vs428/project/n2c2_models/roberta_large_ner_so/tf_model\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2e039fdf-d5ef-468a-b1c3-acf668d4a29c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/vsocrates/n2c2-soap-entailment/commit/ac58c4b99d855db07d3245e4b5fc4c8963ac0e7a', commit_message='Upload RobertaForSequenceClassification', commit_description='', oid='ac58c4b99d855db07d3245e4b5fc4c8963ac0e7a', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.push_to_hub(\"n2c2-soap-entailment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "d1b91f61-bc77-4120-8007-9997ea0c0238",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e63a41a644db49548319311dc8e7092c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/956 [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56c691a406654eb7adc0c7afa2f43c92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Downloading:   0%|          | 0.00/1.42G [00:00<?, ?B/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "All model checkpoint layers were used when initializing TFRobertaForSequenceClassification.\n",
      "\n",
      "All the layers of TFRobertaForSequenceClassification were initialized from the model checkpoint at vsocrates/n2c2-soap-entailment.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaForSequenceClassification for predictions without further training.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoConfig, TFAutoModelForSequenceClassification\n",
    "\n",
    "label2id = {'Not Relevant':3, 'Neither':2, 'Indirect':1, 'Direct':0}\n",
    "id2label = {v:k for k,v in label2id.items()}\n",
    "# define model checkpoint - can be the same model that you already have on the hub\n",
    "model_ckpt = \"vsocrates/n2c2-soap-entailment\"\n",
    "# define config\n",
    "config = AutoConfig.from_pretrained(model_ckpt, label2id=label2id, id2label=id2label)\n",
    "# load model with config\n",
    "model = TFAutoModelForSequenceClassification.from_pretrained(model_ckpt, config=config)\n",
    "# export model\n",
    "model.save_pretrained(\"/home/vs428/project/n2c2_models/roberta_large_ner_so/roberta_large_ner_so/tf_upload_relabeled\")\n",
    "# \"/home/vs428/project/n2c2_models/roberta_large_ner_so/tf_model\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "18736dfe-af7c-472d-8c77-ebc8b23e843a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.push_to_hub(\"n2c2-soap-entailment\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fcbc435f-fcf2-456b-ab2f-1da6d5827ccc",
   "metadata": {},
   "source": [
    "# Also Upload Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d4fe2954-ae56-4f26-af10-c17d166ae089",
   "metadata": {},
   "outputs": [],
   "source": [
    "from functools import partial\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8e6c0a84-e6b3-4a9a-b017-2ee39624badc",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import spacy\n",
    "\n",
    "import shap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f227cc01-b6e9-4b41-8fd0-71e0b3b77790",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "98e8042c-ea05-463c-83d8-0f35ed160a14",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'7.0.0'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pyarrow.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6434237d-1ae1-41d5-aff5-197dd3d8fe33",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import transformers\n",
    "from transformers import AutoTokenizer, AutoModel, AutoModelForMaskedLM, AutoModelForSequenceClassification\n",
    "from transformers import GPT2Tokenizer, GPTNeoForSequenceClassification\n",
    "from transformers import DataCollatorWithPadding\n",
    "from transformers import logging\n",
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "from transformers import RobertaForSequenceClassification, BertForSequenceClassification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "acbb600e-fd82-4edb-96af-15864b600b9e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "from datasets import Value, ClassLabel, Features, DatasetDict\n",
    "from datasets import load_metric\n",
    "import evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "70d26901-c132-4e04-97de-fcc37d831c37",
   "metadata": {},
   "outputs": [],
   "source": [
    "from omegaconf import DictConfig, OmegaConf\n",
    "import hydra"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "33f4a6f7-d724-4a95-a372-ef959d9c09a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "from preprocessing.cleaning_utils import *\n",
    "from train_utils.metrics import *\n",
    "from train_utils.plot_utils import *\n",
    "from train_utils.custom_trainer import *"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79e6e6be-2b1a-4fbe-9c72-927a3c69ca61",
   "metadata": {},
   "source": [
    "\n",
    "# Load in Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9f8ebbee-fab8-46cc-a266-abab3796b178",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_model_path = \"/home/vs428/project/models/RoBERTa-large-PM-M3-Voc/RoBERTa-large-PM-M3-Voc-hf\"\n",
    "model_path = \"/home/vs428/project/n2c2_models/roberta_large_ner_so/roberta_large_ner_so/outputs/roberta_large/checkpoint-384/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "96858c96-a4b2-424a-a8ff-dbc395b639a2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "tokenizer = transformers.AutoTokenizer.from_pretrained(base_model_path, use_fast=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "eaa1f625-5c20-43cd-b068-7757c46170f2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PreTrainedTokenizerFast(name_or_path='/home/vs428/project/models/RoBERTa-large-PM-M3-Voc/RoBERTa-large-PM-M3-Voc-hf', vocab_size=50000, model_max_len=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'bos_token': AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'eos_token': AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'unk_token': AddedToken(\"<unk>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'sep_token': AddedToken(\"</s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'pad_token': AddedToken(\"<pad>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'cls_token': AddedToken(\"<s>\", rstrip=False, lstrip=False, single_word=False, normalized=True), 'mask_token': AddedToken(\"<mask>\", rstrip=False, lstrip=True, single_word=False, normalized=True)})"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b9782bf-0fa6-4d7e-a12d-df126307da02",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "c8567e1a-556a-47de-b9b4-33dde90a5c63",
   "metadata": {},
   "source": [
    "# Read in Dev Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "aabe2a5d-b2b3-4cd7-9ba5-fb9e0fc55738",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = \"/home/vs428/project/n2c2/2022/Data/dev_so.csv\"\n",
    "spacy_assessment = \"/home/vs428/project/n2c2_spacy_models/assessment_model_v2/model-best\"\n",
    "spacy_plan = \"/home/vs428/project/n2c2_spacy_models/plan_subsection_model_v3/model-best\"\n",
    "spacy_so = \"/home/vs428/project/n2c2_spacy_models/so_model/model-best\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "62c127c2-dd8f-4a67-9d73-3af0a24e9382",
   "metadata": {},
   "outputs": [],
   "source": [
    "dev = pd.read_csv(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "49a491d0-f887-4d2f-bf94-501eb424d5ac",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ROW ID</th>\n",
       "      <th>HADM ID</th>\n",
       "      <th>Assessment</th>\n",
       "      <th>Plan Subsection</th>\n",
       "      <th>Relation</th>\n",
       "      <th>HADM_ID</th>\n",
       "      <th>S</th>\n",
       "      <th>O</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>576487</td>\n",
       "      <td>112508</td>\n",
       "      <td>A 60 year old woman with recurrent ALL with CN...</td>\n",
       "      <td>CNS VRE:  S/P Omaya removal, on linezolid for ...</td>\n",
       "      <td>Direct</td>\n",
       "      <td>112508</td>\n",
       "      <td>NGT placed/TF started  Femoral line changed o...</td>\n",
       "      <td>Last dose of Antibiotics:  Meropenem - [**2176...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>576487</td>\n",
       "      <td>112508</td>\n",
       "      <td>A 60 year old woman with recurrent ALL with CN...</td>\n",
       "      <td>LEUKOCYTOSIS:   GCSF vs culture negative bacte...</td>\n",
       "      <td>Direct</td>\n",
       "      <td>112508</td>\n",
       "      <td>NGT placed/TF started  Femoral line changed o...</td>\n",
       "      <td>Last dose of Antibiotics:  Meropenem - [**2176...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>576487</td>\n",
       "      <td>112508</td>\n",
       "      <td>A 60 year old woman with recurrent ALL with CN...</td>\n",
       "      <td>ALL:  Currently day +43 s/p hyper-CVAD.  She i...</td>\n",
       "      <td>Indirect</td>\n",
       "      <td>112508</td>\n",
       "      <td>NGT placed/TF started  Femoral line changed o...</td>\n",
       "      <td>Last dose of Antibiotics:  Meropenem - [**2176...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>576487</td>\n",
       "      <td>112508</td>\n",
       "      <td>A 60 year old woman with recurrent ALL with CN...</td>\n",
       "      <td>DRUG RASH:  Appears improved today.  Evidence ...</td>\n",
       "      <td>Indirect</td>\n",
       "      <td>112508</td>\n",
       "      <td>NGT placed/TF started  Femoral line changed o...</td>\n",
       "      <td>Last dose of Antibiotics:  Meropenem - [**2176...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>576487</td>\n",
       "      <td>112508</td>\n",
       "      <td>A 60 year old woman with recurrent ALL with CN...</td>\n",
       "      <td>ELEVATED LIVER ENZYMES:  Consistent with shock...</td>\n",
       "      <td>Neither</td>\n",
       "      <td>112508</td>\n",
       "      <td>NGT placed/TF started  Femoral line changed o...</td>\n",
       "      <td>Last dose of Antibiotics:  Meropenem - [**2176...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>592</th>\n",
       "      <td>679058</td>\n",
       "      <td>187377</td>\n",
       "      <td>This is a 67 year old male h/o HTN, systolic H...</td>\n",
       "      <td># Hct Drop: Likely represents GI bleed though ...</td>\n",
       "      <td>Neither</td>\n",
       "      <td>187377</td>\n",
       "      <td>Chest Pain/Multivessel CAD [**4-4**] CP at [*...</td>\n",
       "      <td>Last dose of Antibiotics:  Infusions:  Nitrogl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>679058</td>\n",
       "      <td>187377</td>\n",
       "      <td>This is a 67 year old male h/o HTN, systolic H...</td>\n",
       "      <td># HTN: BP well controlled at this time. Trying...</td>\n",
       "      <td>Indirect</td>\n",
       "      <td>187377</td>\n",
       "      <td>Chest Pain/Multivessel CAD [**4-4**] CP at [*...</td>\n",
       "      <td>Last dose of Antibiotics:  Infusions:  Nitrogl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>594</th>\n",
       "      <td>679058</td>\n",
       "      <td>187377</td>\n",
       "      <td>This is a 67 year old male h/o HTN, systolic H...</td>\n",
       "      <td># Hyperlipdemia: cont statin</td>\n",
       "      <td>Neither</td>\n",
       "      <td>187377</td>\n",
       "      <td>Chest Pain/Multivessel CAD [**4-4**] CP at [*...</td>\n",
       "      <td>Last dose of Antibiotics:  Infusions:  Nitrogl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>595</th>\n",
       "      <td>679058</td>\n",
       "      <td>187377</td>\n",
       "      <td>This is a 67 year old male h/o HTN, systolic H...</td>\n",
       "      <td># Tobacco Abuse: Deferrred nicotine patch at t...</td>\n",
       "      <td>Neither</td>\n",
       "      <td>187377</td>\n",
       "      <td>Chest Pain/Multivessel CAD [**4-4**] CP at [*...</td>\n",
       "      <td>Last dose of Antibiotics:  Infusions:  Nitrogl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>596</th>\n",
       "      <td>679058</td>\n",
       "      <td>187377</td>\n",
       "      <td>This is a 67 year old male h/o HTN, systolic H...</td>\n",
       "      <td>ICU Care\\n   Nutrition:\\n   Glycemic Control:\\...</td>\n",
       "      <td>Not Relevant</td>\n",
       "      <td>187377</td>\n",
       "      <td>Chest Pain/Multivessel CAD [**4-4**] CP at [*...</td>\n",
       "      <td>Last dose of Antibiotics:  Infusions:  Nitrogl...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>597 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     ROW ID  HADM ID                                         Assessment  \\\n",
       "0    576487   112508  A 60 year old woman with recurrent ALL with CN...   \n",
       "1    576487   112508  A 60 year old woman with recurrent ALL with CN...   \n",
       "2    576487   112508  A 60 year old woman with recurrent ALL with CN...   \n",
       "3    576487   112508  A 60 year old woman with recurrent ALL with CN...   \n",
       "4    576487   112508  A 60 year old woman with recurrent ALL with CN...   \n",
       "..      ...      ...                                                ...   \n",
       "592  679058   187377  This is a 67 year old male h/o HTN, systolic H...   \n",
       "593  679058   187377  This is a 67 year old male h/o HTN, systolic H...   \n",
       "594  679058   187377  This is a 67 year old male h/o HTN, systolic H...   \n",
       "595  679058   187377  This is a 67 year old male h/o HTN, systolic H...   \n",
       "596  679058   187377  This is a 67 year old male h/o HTN, systolic H...   \n",
       "\n",
       "                                       Plan Subsection      Relation  HADM_ID  \\\n",
       "0    CNS VRE:  S/P Omaya removal, on linezolid for ...        Direct   112508   \n",
       "1    LEUKOCYTOSIS:   GCSF vs culture negative bacte...        Direct   112508   \n",
       "2    ALL:  Currently day +43 s/p hyper-CVAD.  She i...      Indirect   112508   \n",
       "3    DRUG RASH:  Appears improved today.  Evidence ...      Indirect   112508   \n",
       "4    ELEVATED LIVER ENZYMES:  Consistent with shock...       Neither   112508   \n",
       "..                                                 ...           ...      ...   \n",
       "592  # Hct Drop: Likely represents GI bleed though ...       Neither   187377   \n",
       "593  # HTN: BP well controlled at this time. Trying...      Indirect   187377   \n",
       "594                       # Hyperlipdemia: cont statin       Neither   187377   \n",
       "595  # Tobacco Abuse: Deferrred nicotine patch at t...       Neither   187377   \n",
       "596  ICU Care\\n   Nutrition:\\n   Glycemic Control:\\...  Not Relevant   187377   \n",
       "\n",
       "                                                     S  \\\n",
       "0     NGT placed/TF started  Femoral line changed o...   \n",
       "1     NGT placed/TF started  Femoral line changed o...   \n",
       "2     NGT placed/TF started  Femoral line changed o...   \n",
       "3     NGT placed/TF started  Femoral line changed o...   \n",
       "4     NGT placed/TF started  Femoral line changed o...   \n",
       "..                                                 ...   \n",
       "592   Chest Pain/Multivessel CAD [**4-4**] CP at [*...   \n",
       "593   Chest Pain/Multivessel CAD [**4-4**] CP at [*...   \n",
       "594   Chest Pain/Multivessel CAD [**4-4**] CP at [*...   \n",
       "595   Chest Pain/Multivessel CAD [**4-4**] CP at [*...   \n",
       "596   Chest Pain/Multivessel CAD [**4-4**] CP at [*...   \n",
       "\n",
       "                                                     O  \n",
       "0    Last dose of Antibiotics:  Meropenem - [**2176...  \n",
       "1    Last dose of Antibiotics:  Meropenem - [**2176...  \n",
       "2    Last dose of Antibiotics:  Meropenem - [**2176...  \n",
       "3    Last dose of Antibiotics:  Meropenem - [**2176...  \n",
       "4    Last dose of Antibiotics:  Meropenem - [**2176...  \n",
       "..                                                 ...  \n",
       "592  Last dose of Antibiotics:  Infusions:  Nitrogl...  \n",
       "593  Last dose of Antibiotics:  Infusions:  Nitrogl...  \n",
       "594  Last dose of Antibiotics:  Infusions:  Nitrogl...  \n",
       "595  Last dose of Antibiotics:  Infusions:  Nitrogl...  \n",
       "596  Last dose of Antibiotics:  Infusions:  Nitrogl...  \n",
       "\n",
       "[597 rows x 8 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dev"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "937fe816-00a2-464f-b0fa-d58365176a1a",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using custom data configuration default-86e8ba7bd129054c\n",
      "Found cached dataset csv (/home/vs428/.cache/huggingface/datasets/csv/default-86e8ba7bd129054c/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a496b8b12444d9b932617e8b6758512",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading cached processed dataset at /home/vs428/.cache/huggingface/datasets/csv/default-86e8ba7bd129054c/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-e3d1802179efc530.arrow\n",
      "Loading cached processed dataset at /home/vs428/.cache/huggingface/datasets/csv/default-86e8ba7bd129054c/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-0c3ea877a77644f2.arrow\n",
      "[W095] Model 'en_pipeline' (0.0.0) was trained with spaCy v3.3 and may not be 100% compatible with the current version (3.4.2). If you see errors or degraded performance, download a newer compatible model or retrain your custom model with the current spaCy version. For more details and available updates, run: python -m spacy validate\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ROW ID': 576487, 'HADM ID': 112508, 'Assessment': 'A 60 year old woman with recurrent ALL with CNS involvement s/p Omaya\\n   removal due to VRE contamination & SDH evacuation.  She is now doing\\n   well and awake s/p extubation, afebrile and her WBC count is trending\\n   downward.', 'Plan Subsection': 'CNS VRE:  S/P Omaya removal, on linezolid for greater CNS penetration\\n   despite risk of CNS penetration. No TEE performed.\\n   - continue linezolid (D1=[**4-16**]), course to be determined by TEE (4 vs 8\\n   weeks) gentamicin D/C\\nd per ID\\n   - f/u culture data and sensitivities\\n   - TEE deferred at this time.\\n   - f/u tigacycline sensitivities for long-term therapy', 'label': 0, 'S': ' NGT placed/TF started  Femoral line changed over wire by IR  c diff x 2 negative, 3rd pending  d/c\\'ed gentamicin per ID Sulfa (Sulfonamides)  Rash;  Penicillins  Rash;  Latex  Hives; Wheezing  Red Dye  Headache; \"thra  Darvon (Oral) (Propoxyphene Hcl)  Nausea/Vomiting  Percodan (Oral) (Oxycodone Hcl/Oxycodon Ter/Asa)  Nausea/Vomiting  Aspirin  Nausea/Vomiting  Aspartame  Unknown;  Fentanyl  Arrhythmia/PalpfReview of systems is unchanged from admission except as noted below  Review of systems:', 'O': 'Last dose of Antibiotics:  Meropenem - [**2176-4-18**] 07:42 AM  Gentamicin - [**2176-4-20**] 08:00 AM  Micafungin - [**2176-4-20**] 07:35 PM  Linezolid - [**2176-4-21**] 02:30 AM  Acyclovir - [**2176-4-21**] 06:22 AM  Infusions:  Other ICU medications:  Pantoprazole (Protonix) - [**2176-4-20**] 08:00 AM  Heparin Sodium (Prophylaxis) - [**2176-4-20**] 01:16 PM  Other medications:Flowsheet Data as of [**2176-4-21**] 07:16 AM  Vital signs  Hemodynamic monitoring  Fluid balance  24 hours  Since 12 AM  Tmax: 36 C (96.8  Tcurrent: 36 C (96.8  HR: 98 (82 - 105) bpm  BP: 128/83(101) {95/60(74) - 128/83(101)} mmHg  RR: 18 (15 - 28) insp/min  SpO2: 95%  Heart rhythm: SR (Sinus Rhythm)  Height: 63 Inch  Total In:  1,500 mL  532 mL  PO:  TF:  86 mL  IVF:  1,500 mL  447 mL  Blood products:  Total out:  2,044 mL  335 mL  Urine:  2,044 mL  335 mL  NG:  Stool:  Drains:  Balance:  -544 mL  197 mL  Respiratory support  O2 Delivery Device: None  SpO2: 95%ABG: ///18/ Eyes / Conjunctiva: pupils equal and reactive, alert, appropriate,  following commands  Head, Ears, Nose, Throat: PERRL  Cardiovascular: normal S1/S2  Peripheral Vascular: DP pulses intact  Respiratory / Chest: lungs clear anteriorly  Abdominal: Soft, Bowel sounds present, ostomy bag in place with brown  liquid stool, no apparent abdominal tenderness  Extremities: warm  Skin: diffuse maculopapular erythematous, blanching rash on back,  appears less erythematous than yesterday  Neurologic: following commands and answering appropriately but slowly73 K/uL  11.0 g/dL  111 mg/dL  0.9 mg/dL  18 mEq/L  3.6 mEq/L  28 mg/dL  109 mEq/L  137 mEq/L  32.2 %  23.7 K/uLCa++:7.6 mg/dL, Mg++:1.6 mg/dL, PO4:3.2 mg/dL  Differential-Neuts:87.0 %, Band:5.0 %, Lymph:3.0 %, Mono:1.0 %, Eos:0.0  %,WBC  30.3  27.0  32.1  23.7  Hct  36.0  29.9  29.8  31.3  32.2  Plt  90  75  73  73  Cr  0.9  0.9  0.9  0.9  TCO2  17  16  18  18  21  Glucose  157  157  137  117  111  ALT: 161 AP: 455 Tbili: 1.1  AST: 52 LDH: 721  PT: 15.2 PTT: 34.3 INR: 1.3  Fibrinogen: 484Imaging: NoneMicrobiology: [**4-21**]: Cath tip culture P  [**Date range (1) 6157**] BCx pending'}\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a51b5d940f04b9ebe812f7fe660f128",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/597 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43f5c9ec890a4b069e9b6cf261593dae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/597 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "# create hf Dataset\n",
    "classes = ['Not Relevant', 'Neither', 'Indirect', 'Direct']\n",
    "\n",
    "# instead we will use the raw text for now\n",
    "features = Features({\n",
    "    'ROW ID':Value(\"int64\"),\n",
    "    'HADM ID':Value(\"int64\"),\n",
    "    'Assessment':Value(\"string\"),\n",
    "    'Plan Subsection':Value(\"string\"),\n",
    "    \"Relation\":Value(\"string\"),\n",
    "    \"S\":Value(\"string\"),\n",
    "    \"O\":Value(\"string\")        \n",
    "\n",
    "}) \n",
    "\n",
    "dataset = load_dataset(\"csv\", data_files={\n",
    "                            # \"train\":cfg.data.n2c2_data_dir + \"train_so.csv\",\n",
    "                            \"valid\":data_path,\n",
    "                            # \"train\":cfg.data.n2c2_data_dir + \"train.csv\",\n",
    "                            # \"valid\":cfg.data.n2c2_data_dir + \"dev.csv\",\n",
    "\n",
    "                        },\n",
    "                       features=features)\n",
    "\n",
    "# create encoded class labels and rename\n",
    "label2id = {'Not Relevant':3, 'Neither':2, 'Indirect':1, 'Direct':0}\n",
    "id2label = {v:k for k,v in label2id.items()}\n",
    "\n",
    "dataset = dataset.class_encode_column(\"Relation\")\n",
    "dataset = dataset.align_labels_with_mapping(label2id, \"Relation\")\n",
    "dataset = dataset.rename_column(\"Relation\", \"label\")\n",
    "\n",
    "# dataset['valid'] = dataset['valid'].shard(num_shards=20, index=0)\n",
    "\n",
    "# dataset = dataset.map(partial(add_SO_sections))\n",
    "# print(\"AFTER TRAIN _SO sections\")\n",
    "print(dataset['valid'][0])\n",
    "\n",
    "spacy.require_gpu()\n",
    "nlp_assessment = spacy.load(spacy_assessment, exclude=\"parser\")\n",
    "nlp_plan = spacy.load(spacy_plan, exclude=\"parser\")\n",
    "\n",
    "dataset['valid'] = dataset['valid'].map(partial(add_ner_assessment, nlp=nlp_assessment))\n",
    "dataset['valid'] = dataset['valid'].map(partial(add_ner_plan, nlp=nlp_plan))        \n",
    "\n",
    "# we ASSUME that the ner labels we want are lowercase, UNLIKE the standard ones in the model\n",
    "spans = [x for x in nlp_plan.get_pipe(\"ner\").labels if x.islower()] + [x for x in nlp_assessment.get_pipe(\"ner\").labels if x.islower()]\n",
    "\n",
    "tokens = []\n",
    "for span in spans:\n",
    "    tokens.append(\"<\" + span + \">\")\n",
    "    tokens.append(\"</\" + span + \">\")            \n",
    "\n",
    "# add the span tags to the vocab\n",
    "_ = tokenizer.add_tokens(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c0507fde-41a3-4dd8-a06c-3648d6525fba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CommitInfo(commit_url='https://huggingface.co/vsocrates/n2c2-soap-entailment/commit/59c09d100ad030f2e542a40c145801a6bb544960', commit_message='Upload tokenizer', commit_description='', oid='59c09d100ad030f2e542a40c145801a6bb544960', pr_url=None, pr_revision=None, pr_num=None)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.push_to_hub(\"n2c2-soap-entailment\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84c5d7c4-e343-4dfa-bbc8-40a825bd5717",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eac375cf-8e73-409d-9899-dcbb6d28966c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0d9ffb82-f548-4630-99e6-01db23bbe823",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c076473c-0d33-43c6-a2f6-4abcd6ac40a2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "284dd4d7-b77a-4e74-adb5-8a930b7f36f0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}