data_ingestion.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading BADM modules (docx and pdf) into Pinecone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import required libraries\n",
    "\n",
    "from langchain.chains import RetrievalQA\n",
    "from langchain.llms import OpenAI\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load API keys from globally-availabe .env file\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "load_dotenv(dotenv_path='/mnt/project/chatbotai/huggingface_cache/internal_api_keys.env', override=True)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading one docx file into the dataloader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convert /mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567/BADM 567 MOOC 1 Module 1 Word Transcript.docx -> /tmp/tmpes8iq7vy/BADM 567 MOOC 1 Module 1 Word Transcript.docx using filter : MS Word 2007 XML\n"
     ]
    }
   ],
   "source": [
    "from langchain.document_loaders import UnstructuredWordDocumentLoader\n",
    "\n",
    "loader = UnstructuredWordDocumentLoader(\"/mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567/BADM 567 MOOC 1 Module 1 Word Transcript.docx\")\n",
    "data = loader.load()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the entire directory into the loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "detectron2 is not installed. Cannot use the hi_res partitioning strategy. Falling back to partitioning with the fast strategy.\n",
      "detectron2 is not installed. Cannot use the hi_res partitioning strategy. Falling back to partitioning with the fast strategy.\n",
      "detectron2 is not installed. Cannot use the hi_res partitioning strategy. Falling back to partitioning with the fast strategy.\n",
      "detectron2 is not installed. Cannot use the hi_res partitioning strategy. Falling back to partitioning with the fast strategy.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "convert /mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567/BAMD 567 MOOC 1 Module 3 Word Transcript.docx -> /tmp/tmp9prafnjt/BAMD 567 MOOC 1 Module 3 Word Transcript.docx using filter : MS Word 2007 XML\n",
      "convert /mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567/BADM 567 MOOC 1 Module 4 Word Transcript.docx -> /tmp/tmp4fabzdg9/BADM 567 MOOC 1 Module 4 Word Transcript.docx using filter : MS Word 2007 XML\n",
      "convert /mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567/BADM 567 MOOC 1 Module 1 Word Transcript.docx -> /tmp/tmpq6l8dxl0/BADM 567 MOOC 1 Module 1 Word Transcript.docx using filter : MS Word 2007 XML\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "8"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from langchain.document_loaders import DirectoryLoader\n",
    "\n",
    "loader = DirectoryLoader(\"/mnt/project/chatbotai/asmita/non-public-datasets/raw_data/badm_567\")\n",
    "docs = loader.load()\n",
    "len(docs)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initializing tokenizer and text splitter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/avd6/utils/miniconda3/envs/langchain/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from langchain import text_splitter\n",
    "from transformers import AutoTokenizer\n",
    "from langchain.text_splitter import CharacterTextSplitter\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained('OpenAssistant/oasst-sft-1-pythia-12b')\n",
    "\n",
    "doc_splitter = CharacterTextSplitter.from_huggingface_tokenizer(tokenizer, chunk_size=682, chunk_overlap=100)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Creating a list of document-wise IDs and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'file_name': 'BADM 567 MOOC 1 Module 2 Word Transcript.docx', 'module_name': 'Module 2: Process Configurations and Metrics'}, {'file_name': 'BADM 567 MOOC2 Module 1 Word Transcript.pdf', 'module_name': 'Module 1: Infrastructure for Organizational Learning'}, {'file_name': 'BADM 567 MOOC2 Module 2 Word Transcript.pdf', 'module_name': 'Module 2: Process Improvement Projects in Continuous'}, {'file_name': 'BADM 567 MOOC2 Module 3 Word Transcript.pdf', 'module_name': 'Module 3: Measurement and Analysis for Process Improvement'}, {'file_name': 'BADM 567 MOOC2 Module 4 Word Transcript.pdf', 'module_name': 'Module 4: Designing Improvements and Sustaining Changes'}, {'file_name': 'BAMD 567 MOOC 1 Module 3 Word Transcript.docx', 'module_name': 'Module 3: Inventory Management '}, {'file_name': 'BADM 567 MOOC 1 Module 4 Word Transcript.docx', 'module_name': 'Module 4: Supply Chain Management '}, {'file_name': 'BADM 567 MOOC 1 Module 1 Word Transcript.docx', 'module_name': 'Module 1: Operations Strategy'}]\n",
      "['BADM_567_MOOC_1_Module_2', 'BADM_567_MOOC2_Module_1', 'BADM_567_MOOC2_Module_2', 'BADM_567_MOOC2_Module_3', 'BADM_567_MOOC2_Module_4', 'BAMD_567_MOOC_1_Module_3', 'BADM_567_MOOC_1_Module_4', 'BADM_567_MOOC_1_Module_1']\n"
     ]
    }
   ],
   "source": [
    "# creating metadata for the docs\n",
    "\n",
    "metadatas = []\n",
    "doc_ids = []\n",
    "\n",
    "for i in range(len(docs)):\n",
    "    source = docs[i].metadata['source']\n",
    "    content_list = docs[i].page_content[:100].split(\"\\n\")\n",
    "    module_name = [ele for ele in  content_list if re.match('Module\\s\\d:', ele)][0]\n",
    "    \n",
    "    file_name = source.split(\"/\")[-1]\n",
    "    metadatas.append({'file_name': file_name, 'module_name': module_name})\n",
    "\n",
    "    title = file_name.split(\" \")[:-2]\n",
    "    id = \"_\".join(title)\n",
    "    doc_ids.append(id)\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Splitting the individual docs into chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_documents = []\n",
    "\n",
    "for i in range(len(docs)):\n",
    "    doc = docs[i].page_content\n",
    "    texts = doc_splitter.split_text(doc)\n",
    "    split_documents.append(texts)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initiate embeddings stuff"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No sentence-transformers model found with name /home/avd6/.cache/torch/sentence_transformers/intfloat_e5-large. Creating a new one with MEAN pooling.\n"
     ]
    }
   ],
   "source": [
    "from langchain.vectorstores import Pinecone\n",
    "from langchain.embeddings import HuggingFaceEmbeddings\n",
    "import pinecone\n",
    "\n",
    "model_name = \"intfloat/e5-large\"\n",
    "embeddings = HuggingFaceEmbeddings(model_name=model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HuggingFaceEmbeddings(client=SentenceTransformer(\n",
       "  (0): Transformer({'max_seq_length': 512, 'do_lower_case': False}) with Transformer model: BertModel \n",
       "  (1): Pooling({'word_embedding_dimension': 1024, 'pooling_mode_cls_token': False, 'pooling_mode_mean_tokens': True, 'pooling_mode_max_tokens': False, 'pooling_mode_mean_sqrt_len_tokens': False})\n",
       "), model_name='intfloat/e5-large')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a chunk-wise list of IDs, embeddings, and metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate the (id, vector, metadata) list\n",
    "\n",
    "document_embeddings = []\n",
    "document_metadatas = []\n",
    "document_ids = []\n",
    "# iterate over documents\n",
    "for i in range(len(split_documents)):\n",
    "    doc = split_documents[i]\n",
    "    metadata = metadatas[i]\n",
    "    doc_id = doc_ids[i]\n",
    "\n",
    "    # iterate over chunks of a single module document\n",
    "    for j in range(len(doc)):\n",
    "        chunk = doc[j]\n",
    "        chunk_embedding = embeddings.embed_documents([chunk])\n",
    "\n",
    "        # create chunk wise IDs and metadata\n",
    "        id = doc_id + \"_chunk_\" + str(j)\n",
    "        metadata[\"chunk\"] = j\n",
    "        \n",
    "        # append to respective lists\n",
    "        document_embeddings.append(chunk_embedding[0])\n",
    "        document_ids.append(id)\n",
    "        document_metadatas.append(metadata)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157\n",
      "157\n",
      "157\n"
     ]
    }
   ],
   "source": [
    "print(len(document_metadatas))\n",
    "print(len(document_ids))\n",
    "print(len(document_embeddings))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Zipping the IDs, embeddings, and metadata together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "records = zip(document_ids, document_embeddings, document_metadatas)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Initiating Pinecone things"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IndexDescription(name='uiuc-chatbot-deduped', metric='cosine', replicas=1, dimension=1024.0, shards=1, pods=1, pod_type='p1.x1', status={'ready': True, 'state': 'Ready'}, metadata_config=None, source_collection='')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# upserting the embeddings tuple to pinecone\n",
    "\n",
    "pinecone_index = pinecone.Index(\"uiuc-chatbot-deduped\")\n",
    "pinecone.init(api_key=os.environ['PINECONE_API_KEY'], environment=\"us-east4-gcp\")\n",
    "\n",
    "pinecone.describe_index(\"uiuc-chatbot-deduped\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Upserting the zipped data to Pinecone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'upserted_count': 157}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pinecone_index.upsert(vectors=records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# what do we have after cleaning?\n",
    "# a nested list of all the documents and a list of their individual metadatas\n",
    "\n",
    "# todo next: create IDs of each file and generate embeddings using the open assistant tokenizer\n",
    "# upload the (ID, embedding, metadata) to pinecone --> ask Kastan if we need to create a separate index\n",
    "\n",
    "# try llamahub too?"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}