16_elmo_sentence_extraction_for_clustering.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "71d43126",
   "metadata": {},
   "source": [
    "# Микродиахроническое исследование русских приставок методами дистрибутивной семантики\n",
    "## Автор: Елизавета Клыкова, БКЛ181\n",
    "### Получение предложений с наиболее изменившимся глаголами для кластеризации ELMo"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec35d73c",
   "metadata": {},
   "source": [
    "#### Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "9ef9b41f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext pycodestyle_magic\n",
    "# %pycodestyle_on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "118f0b43",
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "import pickle\n",
    "import pymongo\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "68a774a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "random.seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3541127c",
   "metadata": {},
   "source": [
    "#### Подключение к базе данных"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e1093910",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'thesis')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "db = client['thesis']\n",
    "db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "31458c80",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = db.sentences\n",
    "lemmas = db.lemmas\n",
    "tokens = db.tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9c3dff4",
   "metadata": {},
   "source": [
    "#### Получение списка глаголов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "84267c67",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "1:80: E501 line too long (157 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "verbs_to_search = ['заметать', 'постановлять', 'изготавливаться', 'наследовать', 'претерпеть', 'взводить', 'засылать', 'дозволять', 'устрашать', 'округлять']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eaa3f9c8",
   "metadata": {},
   "source": [
    "#### Поиск предложений с глаголами в базе"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0ccd4add",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae673520c40d40749e3ff90b4abad9b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# для каждого глагола получаем список предложений, в которых он встречается\n",
    "verbs_with_sents = {}\n",
    "for verb in tqdm(verbs_to_search):\n",
    "    sents_with_verb = lemmas.find_one({'lemma': verb, 'pos': 'V'},\n",
    "                                      {'docs': True, '_id': False})\n",
    "    verbs_with_sents[verb] = sents_with_verb['docs']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "551671f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "presov_sents = {verb: [] for verb in verbs_with_sents}\n",
    "sov_sents = {verb: [] for verb in verbs_with_sents}\n",
    "postsov_sents = {verb: [] for verb in verbs_with_sents}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ff8ba700",
   "metadata": {},
   "outputs": [],
   "source": [
    "ids_to_search = {verb: random.sample(verbs_with_sents[verb],\n",
    "                                     min(len(verbs_with_sents[verb]), 500))\n",
    "                 for verb in verbs_with_sents}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c43c703a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7480f76374f64301a69eedb9d7b14579",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2:80: E501 line too long (86 > 79 characters)\n",
      "3:80: E501 line too long (80 > 79 characters)\n",
      "5:80: E501 line too long (81 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "for verb in tqdm(verbs_with_sents):\n",
    "    for sentence in sentences.find({'_id': {'$in': ids_to_search[verb]}, 'period': 0},\n",
    "                                   {'token_info': True, '_id': True}).limit(50):\n",
    "        tok_list = [tok['token'] for tok in sentence['token_info']]\n",
    "        target_tok = [tok['lemma'] for tok in sentence['token_info']].index(verb)\n",
    "        presov_sents[verb].append((tok_list, target_tok))\n",
    "        ids_to_search[verb].remove(sentence['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "74be479b",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('presov_sents_for_clust.pickle', 'wb') as f:\n",
    "    pickle.dump(presov_sents, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "4661e741",
   "metadata": {},
   "outputs": [],
   "source": [
    "del presov_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "addf92eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6717a796fbd34cb7b62e9f514e21f5a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2:80: E501 line too long (86 > 79 characters)\n",
      "3:80: E501 line too long (80 > 79 characters)\n",
      "5:80: E501 line too long (81 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "for verb in tqdm(verbs_with_sents):\n",
    "    for sentence in sentences.find({'_id': {'$in': ids_to_search[verb]}, 'period': 1},\n",
    "                                   {'token_info': True, '_id': True}).limit(50):\n",
    "        tok_list = [tok['token'] for tok in sentence['token_info']]\n",
    "        target_tok = [tok['lemma'] for tok in sentence['token_info']].index(verb)\n",
    "        sov_sents[verb].append((tok_list, target_tok))\n",
    "        ids_to_search[verb].remove(sentence['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6cfc873c",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sov_sents_for_clust.pickle', 'wb') as f:\n",
    "    pickle.dump(sov_sents, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "904acc90",
   "metadata": {},
   "outputs": [],
   "source": [
    "del sov_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2f0f6602",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4b74677929341d2946a6376d5391a4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2:80: E501 line too long (86 > 79 characters)\n",
      "3:80: E501 line too long (80 > 79 characters)\n",
      "5:80: E501 line too long (81 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "for verb in tqdm(verbs_with_sents):\n",
    "    for sentence in sentences.find({'_id': {'$in': ids_to_search[verb]}, 'period': 2},\n",
    "                                   {'token_info': True, '_id': True}).limit(50):\n",
    "        tok_list = [tok['token'] for tok in sentence['token_info']]\n",
    "        target_tok = [tok['lemma'] for tok in sentence['token_info']].index(verb)\n",
    "        postsov_sents[verb].append((tok_list, target_tok))\n",
    "        ids_to_search[verb].remove(sentence['_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2393fcb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('postsov_sents_for_clust.pickle', 'wb') as f:\n",
    "    pickle.dump(postsov_sents, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6db848a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "del postsov_sents"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}