05_datasets_statistics.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a133453",
   "metadata": {},
   "source": [
    "# Микродиахроническое исследование русских приставок методами дистрибутивной семантики\n",
    "## Автор: Елизавета Клыкова, БКЛ181\n",
    "### Подсчет статистики по корпусу\n",
    "1. Выбрать из базы данных предложения разных периодов\n",
    "2. Посчитать их длину в токенах (с учетом пунктуации и без нее)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd19ae2",
   "metadata": {},
   "source": [
    "#### Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "136602c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext pycodestyle_magic\n",
    "%pycodestyle_on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87500667",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import pymongo\n",
    "from tqdm.auto import tqdm\n",
    "from collections import Counter"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5f69938",
   "metadata": {},
   "source": [
    "**Подключение к базе**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "26ec1049",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Database(MongoClient(host=['localhost:27017'], document_class=dict, tz_aware=False, connect=True), 'thesis')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "client = pymongo.MongoClient('localhost', 27017)\n",
    "db = client['thesis']\n",
    "db"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "712edd64",
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = db.sentences\n",
    "lemmas = db.lemmas\n",
    "tokens = db.tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "296f82bd",
   "metadata": {},
   "source": [
    "#### Подсчет токенов в предложениях разных периодов\n",
    "Досоветский период (4,4 млн)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4a501709",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e0da31a2dd4fa7ba0f7abc5e416cca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "presov_len = 0\n",
    "presov_len_punct = 0\n",
    "for sentence in tqdm(sentences.find({'period': 0},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    for token in sentence['token_info']:\n",
    "        if token['pos'] != 'PUNCT':\n",
    "            presov_len += 1\n",
    "        if ' ' not in token['token']:\n",
    "            presov_len_punct += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "21ddfd45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "72199757"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presov_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5f1930da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "92360656"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "presov_len_punct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "50e5fba3",
   "metadata": {},
   "source": [
    "Советский период (7 млн)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9e3ce936",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0da6cbd0dd8a49c8a024bb10204d84ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sov_len = 0\n",
    "sov_len_punct = 0\n",
    "for sentence in tqdm(sentences.find({'period': 1},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    for token in sentence['token_info']:\n",
    "        if token['pos'] != 'PUNCT':\n",
    "            sov_len += 1\n",
    "        if ' ' not in token['token']:\n",
    "            sov_len_punct += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2891dafe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "93031557"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sov_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "67991021",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "120980059"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sov_len_punct"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4214bd15",
   "metadata": {},
   "source": [
    "Постсоветский период (5,8 млн)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "57b3cd4e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b8f455625e044b382cdead8118f6be7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "0it [00:00, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "postsov_len = 0\n",
    "postsov_len_punct = 0\n",
    "for sentence in tqdm(sentences.find({'period': 2},\n",
    "                                    {'token_info': True, '_id': False})):\n",
    "    for token in sentence['token_info']:\n",
    "        if token['pos'] != 'PUNCT':\n",
    "            postsov_len += 1\n",
    "        if ' ' not in token['token']:\n",
    "            postsov_len_punct += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "d3356abc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81392744"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "postsov_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b7376d49",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "107124670"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "postsov_len_punct"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}