03_datasets_mongo_lemmas.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a133453",
   "metadata": {},
   "source": [
    "# Микродиахроническое исследование русских приставок методами дистрибутивной семантики\n",
    "## Автор: Елизавета Клыкова, БКЛ181\n",
    "### Запись лемм в базу данных\n",
    "1. Считать леммы из pickle-файлов\n",
    "2. Объединить словари\n",
    "3. Записать в базу\n",
    "4. Проиндексировать коллекцию лемм"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd19ae2",
   "metadata": {},
   "source": [
    "#### Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "136602c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext pycodestyle_magic\n",
    "%pycodestyle_on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87500667",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import gridfs\n",
    "import pymongo\n",
    "from tqdm.auto import tqdm\n",
    "from collections import Counter\n",
    "from pymongo.errors import DocumentTooLarge"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d59ee4d5",
   "metadata": {},
   "source": [
    "#### Открываем и считываем файлы с леммами"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8ddf5ad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('presov_lemmas.pickle', 'rb') as fl1:\n",
    "    presov_lemmas = pickle.load(fl1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c108fa44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "429760"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(presov_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3a2eda79",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sov_lemmas.pickle', 'rb') as fl2:\n",
    "    sov_lemmas = pickle.load(fl2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bc8e2820",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "475193"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sov_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2dd7a5e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('postsov_lemmas.pickle', 'rb') as fl3:\n",
    "    postsov_lemmas = pickle.load(fl3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "bda0fc85",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "471930"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(postsov_lemmas)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3c3f96dd",
   "metadata": {},
   "source": [
    "#### Объединяем словари"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "388fc279",
   "metadata": {},
   "outputs": [],
   "source": [
    "all_lemmas = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "e914f343",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "902e35b6991b4421bb065276fcbdfef6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/429760 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 1 + 2 и 3\n",
    "for lem_key, lemma in tqdm(list(presov_lemmas.items())):\n",
    "\n",
    "    if lem_key not in sov_lemmas and lem_key not in postsov_lemmas:\n",
    "        all_lemmas[lem_key] = lemma\n",
    "\n",
    "    else:\n",
    "        combined_freq = lemma['freq']\n",
    "        combined_freq0 = lemma['freq_0']\n",
    "        combined_freq1 = lemma['freq_1']\n",
    "        combined_freq2 = lemma['freq_2']\n",
    "        combined_docs = lemma['docs']\n",
    "\n",
    "        if lem_key in sov_lemmas:\n",
    "            combined_freq += sov_lemmas[lem_key]['freq']\n",
    "            combined_freq0 += sov_lemmas[lem_key]['freq_0']\n",
    "            combined_freq1 += sov_lemmas[lem_key]['freq_1']\n",
    "            combined_freq2 += sov_lemmas[lem_key]['freq_2']\n",
    "            combined_docs.update(sov_lemmas[lem_key]['docs'])\n",
    "            del sov_lemmas[lem_key]\n",
    "\n",
    "        if lem_key in postsov_lemmas:\n",
    "            combined_freq += postsov_lemmas[lem_key]['freq']\n",
    "            combined_freq0 += postsov_lemmas[lem_key]['freq_0']\n",
    "            combined_freq1 += postsov_lemmas[lem_key]['freq_1']\n",
    "            combined_freq2 += postsov_lemmas[lem_key]['freq_2']\n",
    "            combined_docs.update(postsov_lemmas[lem_key]['docs'])\n",
    "            del postsov_lemmas[lem_key]\n",
    "\n",
    "        all_lemmas[lem_key] = {'lemma': lemma['lemma'],\n",
    "                               'pos': lemma['pos'],\n",
    "                               'freq': combined_freq,\n",
    "                               'freq_0': combined_freq0,\n",
    "                               'freq_1': combined_freq1,\n",
    "                               'freq_2': combined_freq2,\n",
    "                               'docs': combined_docs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "2e4ed05d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a32b06b7fae4d93a997bb8e25d03074",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/294726 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 2 + 3\n",
    "for lem_key, lemma in tqdm(list(sov_lemmas.items())):\n",
    "\n",
    "    if lem_key not in postsov_lemmas:\n",
    "        all_lemmas[lem_key] = lemma\n",
    "\n",
    "    else:\n",
    "        combined_freq = lemma['freq'] + postsov_lemmas[lem_key]['freq']\n",
    "        combined_freq0 = lemma['freq_0'] + postsov_lemmas[lem_key]['freq_0']\n",
    "        combined_freq1 = lemma['freq_1'] + postsov_lemmas[lem_key]['freq_1']\n",
    "        combined_freq2 = lemma['freq_2'] + postsov_lemmas[lem_key]['freq_2']\n",
    "        combined_docs = lemma['docs']\n",
    "        combined_docs.update(postsov_lemmas[lem_key]['docs'])\n",
    "        del postsov_lemmas[lem_key]\n",
    "\n",
    "        all_lemmas[lem_key] = {'lemma': lemma['lemma'],\n",
    "                               'pos': lemma['pos'],\n",
    "                               'freq': combined_freq,\n",
    "                               'freq_0': combined_freq0,\n",
    "                               'freq_1': combined_freq1,\n",
    "                               'freq_2': combined_freq2,\n",
    "                               'docs': combined_docs}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7b235a15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28c810c6019c4d048b20c2b43a1b2ed8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/253995 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# то, что есть только в 3\n",
    "for lem_key, lemma in tqdm(list(postsov_lemmas.items())):\n",
    "    all_lemmas[lem_key] = lemma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c63c714b",
   "metadata": {},
   "outputs": [],
   "source": [
    "del presov_lemmas\n",
    "del sov_lemmas\n",
    "del postsov_lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "253534a3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "978481"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "07a298a4",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('all_lemmas.pickle', 'wb') as flem:\n",
    "    pickle.dump(all_lemmas, flem)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3e0a8ace",
   "metadata": {},
   "source": [
    "#### Подключаемся к базе данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "14c76844",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'thesis')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "db = client['thesis']\n",
    "fs = gridfs.GridFS(db)\n",
    "db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "292b831a",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = db.lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6c169dc7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmas.delete_many({})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91df1a12",
   "metadata": {},
   "source": [
    "#### Записываем все леммы в базу данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "94889a48",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('all_lemmas.pickle', 'rb') as flem:\n",
    "    all_lemmas = pickle.load(flem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbe25dc3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_lemmas(lemma_dict, lemma_collection):\n",
    "    lemmas_to_insert = list(lemma_dict.values())\n",
    "    lemma_collection.insert_many(lemmas_to_insert)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea91e006",
   "metadata": {},
   "outputs": [],
   "source": [
    "# insert_lemmas(all_lemmas, lemmas)  # здесь ошибка DocumentTooLarge"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a12d5ceb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "930935814e2e4c159b002690d14dff58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/978481 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# множества не записываются в базу\n",
    "for lem_key in tqdm(all_lemmas):\n",
    "    set_to_list = list(all_lemmas[lem_key]['docs'])\n",
    "    all_lemmas[lem_key]['docs'] = set_to_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2f26d585",
   "metadata": {},
   "outputs": [],
   "source": [
    "def insert_lemmas_one_by_one(lemma_dict, lemma_collection):\n",
    "    big_lemmas = []\n",
    "    for lem_key in tqdm(lemma_dict):\n",
    "        try:\n",
    "            lemma_collection.insert_one(lemma_dict[lem_key])\n",
    "        except DocumentTooLarge:\n",
    "            big_lemmas.append(lemma_dict[lem_key])\n",
    "    return big_lemmas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "7ace3593",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a86d5fa6ac1f44b5bed4f9527a6be79f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/978481 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "big_lemmas = insert_lemmas_one_by_one(all_lemmas, lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "1e9f597b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2cf7f91158b44939b6b60d11bb2104f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, lemma in enumerate(tqdm(big_lemmas)):\n",
    "    with open(f'lemma_{i}.pickle', 'wb') as lem:\n",
    "        pickle.dump(lemma, lem)\n",
    "    with open(f'lemma_{i}.pickle', 'rb') as lem:\n",
    "        lem_id = fs.put(lem)\n",
    "    lemma['file_id'] = lem_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3bc17f3b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffa31579876b42d797ebf06bb9c00235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/26 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for lemma in tqdm(big_lemmas):\n",
    "    info_to_insert = lemma\n",
    "    info_to_insert['docs'] = lemma['docs'][0:100000]\n",
    "    lemmas.insert_one(info_to_insert)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec5b5d33",
   "metadata": {},
   "source": [
    "#### Индексируем"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "4f99f549",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'lemma_1_pos_1'"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# индексируем\n",
    "lemmas.create_index([('lemma', pymongo.ASCENDING),\n",
    "                     ('pos', pymongo.ASCENDING)], unique=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}