examples/nlp_for_materials/nlp_for_materials.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "37d96e78",
   "metadata": {},
   "source": [
    "# Material Science on Documents - Quick start"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4edb626f",
   "metadata": {},
   "source": [
    "## Getting started\n",
    "\n",
    "The [Deep Search Toolkit](https://ds4sd.github.io/deepsearch-toolkit/) allows document conversion with the following few lines of code. It's that simple! For more info or step-by-step guide:\n",
    "- Visit https://ds4sd.github.io/deepsearch-toolkit/guide/convert_doc/\n",
    "- Follow this example notebook"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a8f9c441",
   "metadata": {},
   "source": [
    "### Set notebook parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b01a4fd1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      "Project key:  1234567890abcdefghijklmnopqrstvwyz123456\n"
     ]
    }
   ],
   "source": [
    "from dsnotebooks.settings import ProjectNotebookSettings\n",
    "\n",
    "# notebook settings auto-loaded from .env / env vars\n",
    "notebook_settings = ProjectNotebookSettings()\n",
    "\n",
    "PROFILE_NAME = notebook_settings.profile  # the profile to use\n",
    "PROJ_KEY = notebook_settings.proj_key  # the project to use\n",
    "\n",
    "# default project_key = 1234567890abcdefghijklmnopqrstvwyz123456"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "239dc0f1",
   "metadata": {},
   "source": [
    "### Import example dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "502cdef8",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-02T12:14:25.377422Z",
     "start_time": "2022-08-02T12:14:25.152485Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import deepsearch as ds\n",
    "\n",
    "from pathlib import Path\n",
    "from zipfile import ZipFile\n",
    "\n",
    "from deepsearch.documents.core.export import export_to_markdown\n",
    "from IPython.display import display, Markdown, HTML, display_html"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e6e4dcda",
   "metadata": {},
   "source": [
    "### Connect to Deep Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f44fbf08",
   "metadata": {},
   "outputs": [],
   "source": [
    "api = ds.CpsApi.from_env(profile_name=PROFILE_NAME)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6f1200c5-1138-4491-bc33-3b2d5aabe949",
   "metadata": {},
   "source": [
    "## Convert Document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ec83eb0b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-02T12:14:49.216045Z",
     "start_time": "2022-08-02T12:14:25.380757Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processing input:     : 100%|\u001b[38;2;15;98;254m██████████████████████████████\u001b[0m| 1/1 [00:00<00:00, 206.31it/s]\u001b[38;2;15;98;254m                                                                                                                                                            \u001b[0m\n",
      "Submitting input:     : 100%|\u001b[38;2;15;98;254m██████████████████████████████\u001b[0m| 1/1 [00:03<00:00,  3.48s/it]\u001b[38;2;15;98;254m                                                                                                                                                             \u001b[0m\n",
      "Converting input:     : 100%|\u001b[38;2;15;98;254m██████████████████████████████\u001b[0m| 1/1 [00:23<00:00, 23.54s/it]\u001b[38;2;15;98;254m                                                                                                                                                             \u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Total documents': 1, 'Successfully converted documents': 1}\n"
     ]
    }
   ],
   "source": [
    "output_dir = Path(\"./converted_docs\")\n",
    "\n",
    "fname = \"20140197356.pdf\"\n",
    "\n",
    "documents = ds.convert_documents(\n",
    "    api=api,\n",
    "    proj_key=PROJ_KEY,\n",
    "    source_path=f\"../../data/samples/{fname}\",\n",
    "    progress_bar=True,\n",
    ")\n",
    "documents.download_all(result_dir=output_dir)\n",
    "info = documents.generate_report(result_dir=output_dir)\n",
    "print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "382c4869-cca9-43fc-8052-c0ab7e9c175d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "writing converted_docs/20140197356.json\n",
      "writing converted_docs/20140197356.md\n"
     ]
    }
   ],
   "source": [
    "# Iterare output files and visualize the output\n",
    "for output_file in output_dir.rglob(\"json*.zip\"):\n",
    "    with ZipFile(output_file) as archive:\n",
    "        all_files = archive.namelist()\n",
    "        for name in all_files:\n",
    "            if not name.endswith(\".json\"):\n",
    "                continue\n",
    "\n",
    "            # basename = name.rstrip('.json')\n",
    "            doc_json = json.loads(archive.read(name))\n",
    "\n",
    "            ofile = output_dir / name\n",
    "            print(f\"writing {ofile}\")\n",
    "            with ofile.open(\"w\") as fw:\n",
    "                fw.write(json.dumps(doc_json, indent=2))\n",
    "\n",
    "            doc_md = export_to_markdown(doc_json)\n",
    "\n",
    "            ofile = output_dir / name.replace(\".json\", \".md\")\n",
    "            print(f\"writing {ofile}\")\n",
    "            with ofile.open(\"w\") as fw:\n",
    "                fw.write(doc_md)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "b19f7678-b650-484b-a994-150d0c4ec3a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# display last document\n",
    "# display(Markdown(doc_md))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6784c8a9-4b96-4385-a04e-40ddbf6c613f",
   "metadata": {},
   "source": [
    "## Find materials in a local PDF Document"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb458eb7-4c19-403f-a615-270c38542d20",
   "metadata": {},
   "source": [
    "### load models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9099530a-1912-4651-9a40-76c4f4cf22af",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " -> already downloaded part-of-speech\n",
      " -> already downloaded reference\n",
      " -> already downloaded material\n",
      " -> already downloaded language\n",
      " -> already downloaded name\n",
      " -> already downloaded semantic\n",
      " -> already downloaded geoloc\n"
     ]
    }
   ],
   "source": [
    "from deepsearch_glm.utils.load_pretrained_models import load_pretrained_nlp_models\n",
    "\n",
    "from deepsearch_glm.nlp_utils import (\n",
    "    extract_references_from_doc,\n",
    "    init_nlp_model,\n",
    "    list_nlp_model_configs,\n",
    ")\n",
    "\n",
    "from deepsearch_glm.glm_utils import (\n",
    "    create_glm_config_from_docs,\n",
    "    create_glm_dir,\n",
    "    create_glm_from_docs,\n",
    "    expand_terms,\n",
    "    load_glm,\n",
    "    read_edges_in_dataframe,\n",
    "    read_nodes_in_dataframe,\n",
    "    show_query_result,\n",
    ")\n",
    "\n",
    "from tabulate import tabulate\n",
    "\n",
    "models = load_pretrained_nlp_models(verbose=True, force=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b574c77b-58a5-4259-b8e9-03ce7129c3bf",
   "metadata": {},
   "source": [
    "### Run the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1c995060-aee3-4b6c-ba8b-dce6288e44dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "ifile = \"./converted_docs/20140197356.json\"\n",
    "\n",
    "with open(ifile) as fr:\n",
    "    doc = json.load(fr)\n",
    "\n",
    "model = init_nlp_model(\"language;term;material\")\n",
    "model.set_loglevel(\"INFO\")\n",
    "\n",
    "res = model.apply_on_doc(doc)\n",
    "\n",
    "insts = pd.DataFrame(res[\"instances\"][\"data\"], columns=res[\"instances\"][\"headers\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "01771757-70c3-44cb-824c-1fd9b716a99f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "          type           subtype                                      name   subj_path\n",
      "1097  material  complex_chemical                          2-methylpropyl).  #/texts/48\n",
      "1173  material  complex_chemical    2,4,7-trimethyloctadec-5-yne-4, 7-diol  #/texts/49\n",
      "1216  material  complex_chemical  2,5,8,11-tetramethyldodec-6-yne-5,8-diol  #/texts/49\n",
      "1673  material  complex_chemical                          1-naphthoic acid  #/texts/58\n",
      "1677  material  complex_chemical                          2-naphthoic acid  #/texts/58\n",
      "1720  material  complex_chemical                   1,2,3,4-butanetetracar›  #/texts/58\n"
     ]
    }
   ],
   "source": [
    "# print(insts.columns)\n",
    "\n",
    "materials = insts[\n",
    "    (insts[\"type\"] == \"material\") & (insts[\"subtype\"] == \"complex_chemical\")\n",
    "][[\"type\", \"subtype\", \"name\", \"subj_path\"]]\n",
    "print(materials.to_string())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "327f126e-1007-42db-9b9b-7768ea3c0c80",
   "metadata": {},
   "source": [
    "## Extracting materials from Document collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7b78cb3e-25cb-4ab7-8533-e839623bb52f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>Type</th>\n",
       "      <th>Num entries</th>\n",
       "      <th>Date</th>\n",
       "      <th>Coords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AAAI</td>\n",
       "      <td>Document</td>\n",
       "      <td>16.02K</td>\n",
       "      <td>2023-08-29</td>\n",
       "      <td>default/aaai</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>Document</td>\n",
       "      <td>55.28K</td>\n",
       "      <td>2023-08-22</td>\n",
       "      <td>default/acl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Annual Reports</td>\n",
       "      <td>Document</td>\n",
       "      <td>107.38K</td>\n",
       "      <td>2024-01-12</td>\n",
       "      <td>default/annual-report</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>arXiv abstracts</td>\n",
       "      <td>Document</td>\n",
       "      <td>2.37M</td>\n",
       "      <td>2023-12-07</td>\n",
       "      <td>default/arxiv-abstract</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>arXiv category taxonomy</td>\n",
       "      <td>Record</td>\n",
       "      <td>155</td>\n",
       "      <td>2023-12-05</td>\n",
       "      <td>default/arxiv-category</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>arXiv full documents</td>\n",
       "      <td>Document</td>\n",
       "      <td>2.29M</td>\n",
       "      <td>2023-10-29</td>\n",
       "      <td>default/arxiv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>BioRxiv</td>\n",
       "      <td>Document</td>\n",
       "      <td>357.76K</td>\n",
       "      <td>2023-11-09</td>\n",
       "      <td>default/biorxiv</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Brenda</td>\n",
       "      <td>Record</td>\n",
       "      <td>7.12K</td>\n",
       "      <td>2023-01-03</td>\n",
       "      <td>default/brenda</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>ChEMBL</td>\n",
       "      <td>Record</td>\n",
       "      <td>2.11M</td>\n",
       "      <td>2023-01-03</td>\n",
       "      <td>default/chembl</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>ChemRxiv</td>\n",
       "      <td>Document</td>\n",
       "      <td>8.82K</td>\n",
       "      <td>2023-11-23</td>\n",
       "      <td>default/chemrxiv</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      Name      Type Num entries        Date  \\\n",
       "0                     AAAI  Document      16.02K  2023-08-29   \n",
       "1            ACL Anthology  Document      55.28K  2023-08-22   \n",
       "2           Annual Reports  Document     107.38K  2024-01-12   \n",
       "3          arXiv abstracts  Document       2.37M  2023-12-07   \n",
       "4  arXiv category taxonomy    Record         155  2023-12-05   \n",
       "5     arXiv full documents  Document       2.29M  2023-10-29   \n",
       "6                  BioRxiv  Document     357.76K  2023-11-09   \n",
       "7                   Brenda    Record       7.12K  2023-01-03   \n",
       "8                   ChEMBL    Record       2.11M  2023-01-03   \n",
       "9                 ChemRxiv  Document       8.82K  2023-11-23   \n",
       "\n",
       "                   Coords  \n",
       "0            default/aaai  \n",
       "1             default/acl  \n",
       "2   default/annual-report  \n",
       "3  default/arxiv-abstract  \n",
       "4  default/arxiv-category  \n",
       "5           default/arxiv  \n",
       "6         default/biorxiv  \n",
       "7          default/brenda  \n",
       "8          default/chembl  \n",
       "9        default/chemrxiv  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from numerize.numerize import numerize\n",
    "\n",
    "# Fetch list of all data collections\n",
    "collections = api.elastic.list()\n",
    "collections.sort(key=lambda c: c.name.lower())\n",
    "\n",
    "# Visualize summary table\n",
    "results = [\n",
    "    {\n",
    "        \"Name\": c.name,\n",
    "        \"Type\": c.metadata.type,\n",
    "        \"Num entries\": numerize(c.documents),\n",
    "        \"Date\": c.metadata.created.strftime(\"%Y-%m-%d\"),\n",
    "        \"Coords\": f\"{c.source.elastic_id}/{c.source.index_key}\",\n",
    "    }\n",
    "    for c in collections\n",
    "]\n",
    "display(pd.DataFrame(results[0:10]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d5394460-3427-4a89-a6e3-d655b28b0412",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "#-found documents:  RunQueryResult(outputs={'data_outputs': [], 'data_count': 2, 'data_aggs': {'deepsearch_total_size': {'value': 713388.0}}}, next_pages={}, timings=RunQueryResult.QueryTimings(overall=0.28376056300476193, tasks={'0_ElasticQuery': RunQueryResult.QueryTimings.TaskTimings(overall=0.2827732330188155, details={})}))\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "3it [00:02,  1.13it/s]                                                                                                                                                                                                                                "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished fetching all data. Total is 2 records.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "from copy import deepcopy\n",
    "\n",
    "from deepsearch.cps.client.components.elastic import ElasticDataCollectionSource\n",
    "from deepsearch.cps.queries import DataQuery\n",
    "\n",
    "\n",
    "# Input query\n",
    "search_query = '\"SUBSTITUTED 6-PHENYLNICOTINIC ACIDS AND THEIR USE\"'\n",
    "data_collection = ElasticDataCollectionSource(\n",
    "    elastic_id=\"default\", index_key=\"patent-uspto\"\n",
    ")\n",
    "page_size = 50\n",
    "\n",
    "# Prepare the data query\n",
    "query = DataQuery(\n",
    "    search_query,  # The search query to be executed\n",
    "    # source=[\"description.title\", \"description.authors\", \"identifiers\"], # Which fields of documents we want to fetch\n",
    "    limit=page_size,  # The size of each request page\n",
    "    coordinates=data_collection,  # The data collection to be queries\n",
    ")\n",
    "\n",
    "\n",
    "# [Optional] Compute the number of total results matched. This can be used to monitor the pagination progress.\n",
    "count_query = deepcopy(query)\n",
    "count_query.paginated_task.parameters[\"limit\"] = 0\n",
    "count_results = api.queries.run(count_query)\n",
    "expected_total = count_results.outputs[\"data_count\"]\n",
    "expected_pages = (\n",
    "    expected_total + page_size - 1\n",
    ") // page_size  # this is simply a ceiling formula\n",
    "\n",
    "print(f\"#-found documents: \", count_results)\n",
    "\n",
    "# Iterate through all results by fetching `page_size` results at the same time\n",
    "documents = []\n",
    "cursor = api.queries.run_paginated_query(query)\n",
    "for result_page in tqdm(cursor, total=expected_pages):\n",
    "    # Iterate through the results of a single page, and add to the total list\n",
    "    for row in result_page.outputs[\"data_outputs\"]:\n",
    "        documents.append(row[\"_source\"])\n",
    "\n",
    "print(f\"Finished fetching all data. Total is {len(documents)} records.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "8717419b-6d96-4848-bbbf-3d4149406872",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " US8143411B2: text-2\n",
      "\n",
      "The present application relates to novel substituted 6-phenylnicotinic acid derivatives, to\n",
      "processes for their preparation, to their use for the treatment and/or prophylaxis of diseases and\n",
      "to their use for preparing medicaments for the treatment and/or prophylaxis of diseases, preferably\n",
      "for the treatment and/or prophylaxis of cardiovascular disorders, in particular dyslipidaemias,\n",
      "arteriosclerosis and heart failure. \n",
      "\n",
      "       type           subtype                    name subj_path\n",
      "2  material  complex_chemical  6-phenylnicotinic acid         #\n",
      "\n",
      " US8143411B2: text-3\n",
      "\n",
      "The present invention relates to novel substituted 6-phenylnicotinic acid derivatives, to processes\n",
      "for their preparation, to their use for the treatment and/or prophylaxis of diseases and to their\n",
      "use for preparing medicaments for the treatment and/or prophylaxis of diseases, preferably for the\n",
      "treatment and/or prophylaxis of cardiovascular diseases, in particular dyslipidaemias,\n",
      "arteriosclerosis and heart failure. \n",
      "\n",
      "       type           subtype                    name subj_path\n",
      "2  material  complex_chemical  6-phenylnicotinic acid         #\n",
      "\n",
      " US8143411B2: text-4\n",
      "\n",
      "In spite of many successful therapies, cardiovascular disorders remain a serious public health\n",
      "problem. Treatment with statins, which inhibit HMG-CoA reductase, very successfully lowers both LDL\n",
      "cholesterol (LDL-C) plasma concentrations and the mortality of patients at risk; however, convincing\n",
      "treatment strategies for the therapy of patients having an unfavourable HDL-C/LDL-C ratio and/or\n",
      "hypertriglyceridaemia are still not available to date. \n",
      "\n",
      "        type          subtype         name subj_path\n",
      "8   material  simple_chemical      statins         #\n",
      "13  material  simple_chemical  cholesterol         #\n",
      "\n",
      " US8143411B2: text-5\n",
      "\n",
      "Currently, in addition to niacin, fibrates are the only therapy option for patients of these risk\n",
      "groups. They lower elevated triglyceride levels by 20-50%, reduce LDL-C by 10-15%, change the LDL\n",
      "particle size of atherogenic LDL of low density to less atherogenic LDL of normal density and\n",
      "increase the HDL concentration by 10-15%. \n",
      "\n",
      "       type          subtype          name subj_path\n",
      "2  material  simple_chemical        niacin         #\n",
      "9  material  simple_chemical  triglyceride         #\n",
      "\n",
      " US20100234432A1: text-2\n",
      "\n",
      "The present application relates to novel substituted 6-phenylnicotinic acid derivatives, to\n",
      "processes for their preparation, to their use for the treatment and/or prophylaxis of diseases and\n",
      "to their use for preparing medicaments for the treatment and/or prophylaxis of diseases, preferably\n",
      "for the treatment and/or prophylaxis of cardiovascular disorders, in particular dyslipidaemias,\n",
      "arteriosclerosis and heart failure. \n",
      "\n",
      "       type           subtype                    name subj_path\n",
      "2  material  complex_chemical  6-phenylnicotinic acid         #\n",
      "\n",
      " US20100234432A1: text-3\n",
      "\n",
      "The present invention relates to novel substituted 6-phenylnicotinic acid derivatives, to processes\n",
      "for their preparation, to their use for the treatment and/or prophylaxis of diseases and to their\n",
      "use for preparing medicaments for the treatment and/or prophylaxis of diseases, preferably for the\n",
      "treatment and/or prophylaxis of cardiovascular diseases, in particular dyslipidaemias,\n",
      "arteriosclerosis and heart failure. \n",
      "\n",
      "       type           subtype                    name subj_path\n",
      "2  material  complex_chemical  6-phenylnicotinic acid         #\n",
      "\n",
      " US20100234432A1: text-4\n",
      "\n",
      "In spite of many successful therapies, cardiovascular disorders remain a serious public health\n",
      "problem. Treatment with statins, which inhibit HMG-CoA reductase, very successfully lowers both LDL\n",
      "cholesterol (LDL-C) plasma concentrations and the mortality of patients at risk; however, convincing\n",
      "treatment strategies for the therapy of patients having an unfavourable HDL-C/LDL-C ratio and/or\n",
      "hypertriglyceridaemia are still not available to date. \n",
      "\n",
      "        type          subtype         name subj_path\n",
      "8   material  simple_chemical      statins         #\n",
      "13  material  simple_chemical  cholesterol         #\n",
      "\n",
      " US20100234432A1: text-5\n",
      "\n",
      "Currently, in addition to niacin, fibrates are the only therapy option for patients of these risk\n",
      "groups. They lower elevated triglyceride levels by 20-50%, reduce LDL-C by 10-15%, change the LDL\n",
      "particle size of atherogenic LDL of low density to less atherogenic LDL of normal density and\n",
      "increase the HDL concentration by 10-15%. \n",
      "\n",
      "       type          subtype          name subj_path\n",
      "2  material  simple_chemical        niacin         #\n",
      "9  material  simple_chemical  triglyceride         #\n"
     ]
    }
   ],
   "source": [
    "import textwrap\n",
    "\n",
    "# Create a TextWrapper object\n",
    "wrapper = textwrap.TextWrapper(width=100)  # Set the desired width\n",
    "\n",
    "model = init_nlp_model(\"language;term;material\")\n",
    "model.set_loglevel(\"INFO\")\n",
    "\n",
    "max_items = 5\n",
    "\n",
    "for doc in documents:\n",
    "\n",
    "    dname = doc[\"file-info\"][\"filename\"]\n",
    "\n",
    "    for i, item in enumerate(doc[\"main-text\"]):\n",
    "\n",
    "        if \"text\" not in item:\n",
    "            continue\n",
    "\n",
    "        if i > max_items:\n",
    "            break\n",
    "\n",
    "        res = model.apply_on_text(item[\"text\"])\n",
    "\n",
    "        insts = pd.DataFrame(\n",
    "            res[\"instances\"][\"data\"], columns=res[\"instances\"][\"headers\"]\n",
    "        )\n",
    "\n",
    "        materials = insts[insts[\"type\"] == \"material\"][\n",
    "            [\"type\", \"subtype\", \"name\", \"subj_path\"]\n",
    "        ]\n",
    "\n",
    "        if len(materials) > 0:\n",
    "            lines = wrapper.wrap(item[\"text\"])\n",
    "            print(f\"\\n {dname}: text-{i}\\n\")\n",
    "            print(\"\\n\".join(lines), \"\\n\")\n",
    "            print(materials.to_string())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "05280444-d67c-4828-bfc2-bb3f4bc8698e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adf0256c-f8f9-42d8-957d-a5c54f388a75",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "23663f76e1e243f0a6319b8ef58f504b6b45c83666dfefd3138ba8cf69ab01fa"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}