11_grammatical_profiles_extracting_sentences.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a133453",
   "metadata": {},
   "source": [
    "# Микродиахроническое исследование русских приставок методами дистрибутивной семантики\n",
    "## Автор: Елизавета Клыкова, БКЛ181\n",
    "### Извлечение всех вхождений нужных глаголов\n",
    "1. Выбрать из базы предложения нужных периодов\n",
    "2. Для каждого исследуемого глагола получить набор токенов с грамматическими характеристиками\n",
    "3. Сохранить в pickle-файлы"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd19ae2",
   "metadata": {},
   "source": [
    "#### Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "136602c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext pycodestyle_magic\n",
    "%pycodestyle_on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87500667",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pymongo\n",
    "import pandas as pd\n",
    "from tqdm.auto import tqdm\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5f69938",
   "metadata": {},
   "source": [
    "**Подключение к базе**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "26ec1049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'thesis')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "db = client['thesis']\n",
    "db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "712edd64",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = db.sentences\n",
    "lemmas = db.lemmas\n",
    "tokens = db.tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33f23850",
   "metadata": {},
   "source": [
    "#### Получение списка глаголов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "9c1926c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>prefix</th>\n",
       "      <th>lemma</th>\n",
       "      <th>abs_freq</th>\n",
       "      <th>abs_freq0</th>\n",
       "      <th>abs_freq1</th>\n",
       "      <th>abs_freq2</th>\n",
       "      <th>freq</th>\n",
       "      <th>freq0</th>\n",
       "      <th>freq1</th>\n",
       "      <th>freq2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>бе[зс]</td>\n",
       "      <td>бездействовать</td>\n",
       "      <td>541</td>\n",
       "      <td>155</td>\n",
       "      <td>255</td>\n",
       "      <td>131</td>\n",
       "      <td>2.194</td>\n",
       "      <td>2.147</td>\n",
       "      <td>2.741</td>\n",
       "      <td>1.609</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>бе[зс]</td>\n",
       "      <td>бездельничать</td>\n",
       "      <td>308</td>\n",
       "      <td>41</td>\n",
       "      <td>169</td>\n",
       "      <td>98</td>\n",
       "      <td>1.249</td>\n",
       "      <td>0.568</td>\n",
       "      <td>1.817</td>\n",
       "      <td>1.204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>бе[зс]</td>\n",
       "      <td>безмолвствовать</td>\n",
       "      <td>726</td>\n",
       "      <td>361</td>\n",
       "      <td>191</td>\n",
       "      <td>174</td>\n",
       "      <td>2.944</td>\n",
       "      <td>5.000</td>\n",
       "      <td>2.053</td>\n",
       "      <td>2.138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>бе[зс]</td>\n",
       "      <td>безобразить</td>\n",
       "      <td>149</td>\n",
       "      <td>99</td>\n",
       "      <td>43</td>\n",
       "      <td>7</td>\n",
       "      <td>0.604</td>\n",
       "      <td>1.371</td>\n",
       "      <td>0.462</td>\n",
       "      <td>0.086</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>бе[зс]</td>\n",
       "      <td>безобразничать</td>\n",
       "      <td>306</td>\n",
       "      <td>110</td>\n",
       "      <td>136</td>\n",
       "      <td>60</td>\n",
       "      <td>1.241</td>\n",
       "      <td>1.524</td>\n",
       "      <td>1.462</td>\n",
       "      <td>0.737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8429</th>\n",
       "      <td>у</td>\n",
       "      <td>ущемляться</td>\n",
       "      <td>62</td>\n",
       "      <td>8</td>\n",
       "      <td>16</td>\n",
       "      <td>38</td>\n",
       "      <td>0.251</td>\n",
       "      <td>0.111</td>\n",
       "      <td>0.172</td>\n",
       "      <td>0.467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8430</th>\n",
       "      <td>у</td>\n",
       "      <td>ущипывать</td>\n",
       "      <td>510</td>\n",
       "      <td>147</td>\n",
       "      <td>222</td>\n",
       "      <td>141</td>\n",
       "      <td>2.068</td>\n",
       "      <td>2.036</td>\n",
       "      <td>2.386</td>\n",
       "      <td>1.732</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8431</th>\n",
       "      <td>у</td>\n",
       "      <td>уязвлять</td>\n",
       "      <td>1305</td>\n",
       "      <td>606</td>\n",
       "      <td>411</td>\n",
       "      <td>288</td>\n",
       "      <td>5.291</td>\n",
       "      <td>8.393</td>\n",
       "      <td>4.418</td>\n",
       "      <td>3.538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8432</th>\n",
       "      <td>у</td>\n",
       "      <td>уяснять</td>\n",
       "      <td>1803</td>\n",
       "      <td>677</td>\n",
       "      <td>740</td>\n",
       "      <td>386</td>\n",
       "      <td>7.311</td>\n",
       "      <td>9.377</td>\n",
       "      <td>7.954</td>\n",
       "      <td>4.742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8433</th>\n",
       "      <td>у</td>\n",
       "      <td>уясняться</td>\n",
       "      <td>206</td>\n",
       "      <td>108</td>\n",
       "      <td>87</td>\n",
       "      <td>11</td>\n",
       "      <td>0.835</td>\n",
       "      <td>1.496</td>\n",
       "      <td>0.935</td>\n",
       "      <td>0.135</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8434 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      prefix            lemma  abs_freq  abs_freq0  abs_freq1  abs_freq2  \\\n",
       "0     бе[зс]   бездействовать       541        155        255        131   \n",
       "1     бе[зс]    бездельничать       308         41        169         98   \n",
       "2     бе[зс]  безмолвствовать       726        361        191        174   \n",
       "3     бе[зс]      безобразить       149         99         43          7   \n",
       "4     бе[зс]   безобразничать       306        110        136         60   \n",
       "...      ...              ...       ...        ...        ...        ...   \n",
       "8429       у       ущемляться        62          8         16         38   \n",
       "8430       у        ущипывать       510        147        222        141   \n",
       "8431       у         уязвлять      1305        606        411        288   \n",
       "8432       у          уяснять      1803        677        740        386   \n",
       "8433       у        уясняться       206        108         87         11   \n",
       "\n",
       "       freq  freq0  freq1  freq2  \n",
       "0     2.194  2.147  2.741  1.609  \n",
       "1     1.249  0.568  1.817  1.204  \n",
       "2     2.944  5.000  2.053  2.138  \n",
       "3     0.604  1.371  0.462  0.086  \n",
       "4     1.241  1.524  1.462  0.737  \n",
       "...     ...    ...    ...    ...  \n",
       "8429  0.251  0.111  0.172  0.467  \n",
       "8430  2.068  2.036  2.386  1.732  \n",
       "8431  5.291  8.393  4.418  3.538  \n",
       "8432  7.311  9.377  7.954  4.742  \n",
       "8433  0.835  1.496  0.935  0.135  \n",
       "\n",
       "[8434 rows x 10 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pref_df = pd.read_csv('prefixes_and_lemmas_dropna.tsv', sep='\\t')\n",
    "pref_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3426df6c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# list of dicts\n",
    "pref_dict = pref_df.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "63411b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "verbs_to_search = sorted(set([v['lemma'] for v in pref_dict]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "296f82bd",
   "metadata": {},
   "source": [
    "#### Поиск глагольных употреблений в базе"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f53ef635",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14df7d3f24d849c284d25b17f189fa9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:80: E501 line too long (88 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "# досоветский период\n",
    "\n",
    "presov_verb_dict = {verb: [] for verb in verbs_to_search}\n",
    "\n",
    "for sentence in tqdm(sentences.find({'period': 0},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    verb_list = [token for token in sentence['token_info']\n",
    "                 if token['pos'] == 'V']\n",
    "    for verb in verb_list:\n",
    "        if verb['lemma'] in verbs_to_search:\n",
    "            presov_verb_dict[verb['lemma']].append({'token': verb['token'],\n",
    "                                                    'gram': verb['gram'].split('|')[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8c570c05",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('presov_verb_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(presov_verb_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c770137d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a897f409723b4caa9e06a7c578c1f1e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:80: E501 line too long (85 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "# советский период\n",
    "\n",
    "sov_verb_dict = {verb: [] for verb in verbs_to_search}\n",
    "\n",
    "for sentence in tqdm(sentences.find({'period': 1},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    verb_list = [token for token in sentence['token_info']\n",
    "                 if token['pos'] == 'V']\n",
    "    for verb in verb_list:\n",
    "        if verb['lemma'] in verbs_to_search:\n",
    "            sov_verb_dict[verb['lemma']].append({'token': verb['token'],\n",
    "                                                 'gram': verb['gram'].split('|')[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ec9f225f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('sov_verb_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(sov_verb_dict, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9619d03f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd036fa3f8774616b0b1a87e38b7196a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "12:80: E501 line too long (89 > 79 characters)\n"
     ]
    }
   ],
   "source": [
    "# постсоветский период\n",
    "\n",
    "postsov_verb_dict = {verb: [] for verb in verbs_to_search}\n",
    "\n",
    "for sentence in tqdm(sentences.find({'period': 2},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    verb_list = [token for token in sentence['token_info']\n",
    "                 if token['pos'] == 'V']\n",
    "    for verb in verb_list:\n",
    "        if verb['lemma'] in verbs_to_search:\n",
    "            postsov_verb_dict[verb['lemma']].append({'token': verb['token'],\n",
    "                                                     'gram': verb['gram'].split('|')[0]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9b8a238f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('postsov_verb_dict.pickle', 'wb') as f:\n",
    "    pickle.dump(postsov_verb_dict, f)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}