20_results_rushifteval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "3a133453",
   "metadata": {},
   "source": [
    "# Микродиахроническое исследование русских приставок методами дистрибутивной семантики\n",
    "## Автор: Елизавета Клыкова, БКЛ181\n",
    "### Оценка на материале [RuShiftEval](https://github.com/akutuzov/rushifteval_public)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd19ae2",
   "metadata": {},
   "source": [
    "#### Импорт модулей"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "136602c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# %load_ext pycodestyle_magic\n",
    "# %pycodestyle_on"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "87500667",
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.stats import spearmanr\n",
    "from gensim.models.fasttext import load_facebook_model\n",
    "from tqdm.auto import tqdm\n",
    "from scipy.spatial.distance import cityblock\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83613dac",
   "metadata": {},
   "source": [
    "#### Подготовка датасета"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ff77d256",
   "metadata": {},
   "outputs": [],
   "source": [
    "!wget https://raw.githubusercontent.com/akutuzov/rushifteval_public/main/annotated_devset.tsv\n",
    "!wget https://raw.githubusercontent.com/akutuzov/rushifteval_public/main/annotated_testset.tsv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "31115a02",
   "metadata": {},
   "outputs": [],
   "source": [
    "colnames = ['lemma', 'presov_to_sov', 'sov_to_postsov', 'presov_to_postsov']\n",
    "part1 = pd.read_csv('annotated_devset.tsv', sep='\\t', names=colnames, header=None)\n",
    "part2 = pd.read_csv('annotated_testset.tsv', sep='\\t', names=colnames, header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c7790dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "rushifteval = pd.concat([part1, part2]).sort_values(by='lemma')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4808c634",
   "metadata": {},
   "outputs": [],
   "source": [
    "rushifteval.to_csv('RuShiftEval_full.tsv', sep='\\t', index=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d4f3d0e",
   "metadata": {},
   "source": [
    "#### Считываем заранее подготовленный датасет"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "ab24f367",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>lemma</th>\n",
       "      <th>presov_to_sov</th>\n",
       "      <th>sov_to_postsov</th>\n",
       "      <th>presov_to_postsov</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>авторитет</td>\n",
       "      <td>3.233333</td>\n",
       "      <td>2.955556</td>\n",
       "      <td>2.844444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>амбиция</td>\n",
       "      <td>3.111111</td>\n",
       "      <td>3.444444</td>\n",
       "      <td>3.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>апостол</td>\n",
       "      <td>3.494118</td>\n",
       "      <td>3.426966</td>\n",
       "      <td>3.423529</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>благодарность</td>\n",
       "      <td>3.233333</td>\n",
       "      <td>3.566667</td>\n",
       "      <td>3.655556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>блин</td>\n",
       "      <td>3.213483</td>\n",
       "      <td>1.662921</td>\n",
       "      <td>2.577778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>штаб</td>\n",
       "      <td>3.633333</td>\n",
       "      <td>3.388889</td>\n",
       "      <td>3.500000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>эшелон</td>\n",
       "      <td>2.922222</td>\n",
       "      <td>2.288889</td>\n",
       "      <td>2.333333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>юбилей</td>\n",
       "      <td>3.688889</td>\n",
       "      <td>3.700000</td>\n",
       "      <td>3.788889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>ядро</td>\n",
       "      <td>1.554348</td>\n",
       "      <td>1.911111</td>\n",
       "      <td>1.477778</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>ясли</td>\n",
       "      <td>2.280899</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.900000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             lemma  presov_to_sov  sov_to_postsov  presov_to_postsov\n",
       "0        авторитет       3.233333        2.955556           2.844444\n",
       "1          амбиция       3.111111        3.444444           3.333333\n",
       "2          апостол       3.494118        3.426966           3.423529\n",
       "3    благодарность       3.233333        3.566667           3.655556\n",
       "4             блин       3.213483        1.662921           2.577778\n",
       "..             ...            ...             ...                ...\n",
       "106           штаб       3.633333        3.388889           3.500000\n",
       "107         эшелон       2.922222        2.288889           2.333333\n",
       "108         юбилей       3.688889        3.700000           3.788889\n",
       "109           ядро       1.554348        1.911111           1.477778\n",
       "110           ясли       2.280899        3.000000           1.900000\n",
       "\n",
       "[111 rows x 4 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rse = pd.read_csv('RuShiftEval_full.tsv', sep='\\t')\n",
    "rse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "50768d0f",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_rows = rse.to_dict(orient='records')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "13eb6baf",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_pre_to_sov = [row['lemma'] for row in sorted(gold_rows,\n",
    "                                                  key=lambda x: x['presov_to_sov'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e6827204",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_sov_to_post = [row['lemma'] for row in sorted(gold_rows,\n",
    "                                                   key=lambda x: x['sov_to_postsov'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "df88c6f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_pre_to_post = [row['lemma'] for row in sorted(gold_rows,\n",
    "                                                   key=lambda x: x['presov_to_postsov'])]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6dd854fb",
   "metadata": {},
   "source": [
    "#### Загружаем нужные модели"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1bf7cbb0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_word2vec_model(file_with_model):\n",
    "    model = gensim.models.KeyedVectors.load_word2vec_format(file_with_model,\n",
    "                                                            binary=True)\n",
    "    print('Размер словаря:', len(model.wv.vocab))\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "8ed58a2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 127656\n"
     ]
    }
   ],
   "source": [
    "wv_presov = load_word2vec_model('word2vec/word2vec_presov_lemmas.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "dec8193d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 151890\n"
     ]
    }
   ],
   "source": [
    "wv_sov = load_word2vec_model('word2vec/word2vec_sov_lemmas.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "576dd8a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 149617\n"
     ]
    }
   ],
   "source": [
    "wv_postsov = load_word2vec_model('word2vec/word2vec_postsov_lemmas.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "e919931e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 123804\n"
     ]
    }
   ],
   "source": [
    "wv_pos_presov = load_word2vec_model('word2vec/word2vec_presov_lemmas_pos.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cdf472d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 147146\n"
     ]
    }
   ],
   "source": [
    "wv_pos_sov = load_word2vec_model('word2vec/word2vec_sov_lemmas_pos.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5dd909f7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 144650\n"
     ]
    }
   ],
   "source": [
    "wv_pos_postsov = load_word2vec_model('word2vec/word2vec_postsov_lemmas_pos.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0701beca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 127657\n"
     ]
    }
   ],
   "source": [
    "ft_presov = load_facebook_model('fasttext/fasttext_presov_lemmas.bin')\n",
    "ft_presov.init_sims(replace=True)\n",
    "print('Размер словаря:', len(ft_presov.wv.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5b147275",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 151891\n"
     ]
    }
   ],
   "source": [
    "ft_sov = load_facebook_model('fasttext/fasttext_sov_lemmas.bin')\n",
    "ft_sov.init_sims(replace=True)\n",
    "print('Размер словаря:', len(ft_sov.wv.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "949a09d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Размер словаря: 149618\n"
     ]
    }
   ],
   "source": [
    "ft_postsov = load_facebook_model('fasttext/fasttext_postsov_lemmas.bin')\n",
    "ft_postsov.init_sims(replace=True)\n",
    "print('Размер словаря:', len(ft_postsov.wv.vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e093ce7e",
   "metadata": {},
   "source": [
    "#### Выравнивание"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6f1f15dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection_align_gensim(m1, m2, words=None):\n",
    "    \"\"\"\n",
    "    Intersect two gensim word2vec models, m1 and m2.\n",
    "    Only the shared vocabulary between them is kept.\n",
    "    If 'words' is set (as list or set), then the vocabulary is intersected with this list as well.\n",
    "    Indices are re-organized from 0..N in order of descending frequency (=sum of counts from both m1 and m2).\n",
    "    These indices correspond to the new syn0 and syn0norm objects in both gensim models:\n",
    "        -- so that Row 0 of m1.syn0 will be for the same word as Row 0 of m2.syn0\n",
    "        -- you can find the index of any word on the .index2word list: model.index2word.index(word) => 2\n",
    "    The .vocab dictionary is also updated for each model, preserving the count but updating the index.\n",
    "    \"\"\"\n",
    "\n",
    "    # Get the vocab for each model\n",
    "    vocab_m1 = set(m1.wv.vocab.keys())\n",
    "    vocab_m2 = set(m2.wv.vocab.keys())\n",
    "\n",
    "    # Find the common vocabulary\n",
    "    common_vocab = vocab_m1 & vocab_m2\n",
    "    if words:\n",
    "        common_vocab &= set(words)\n",
    "\n",
    "    # If no alignment necessary because vocab is identical...\n",
    "    if not vocab_m1-common_vocab and not vocab_m2-common_vocab:\n",
    "        return (m1, m2)\n",
    "\n",
    "    # Otherwise sort by frequency (summed for both)\n",
    "    common_vocab = list(common_vocab)\n",
    "    common_vocab.sort(key=lambda w: m1.wv.vocab[w].count + m2.wv.vocab[w].count, reverse=True)\n",
    "\n",
    "    # Then for each model...\n",
    "    for m in [m1, m2]:\n",
    "        # Replace old syn0norm array with new one (with common vocab)\n",
    "        indices = [m.wv.vocab[w].index for w in common_vocab]\n",
    "        old_arr = m.wv.syn0norm\n",
    "        new_arr = np.array([old_arr[index] for index in indices])\n",
    "        m.wv.syn0norm = m.wv.syn0 = new_arr\n",
    "\n",
    "        # Replace old vocab dictionary with new one (with common vocab)\n",
    "        # and old index2word with new one\n",
    "        m.wv.index2word = common_vocab\n",
    "        old_vocab = m.wv.vocab\n",
    "        new_vocab = {}\n",
    "        for new_index, word in enumerate(common_vocab):\n",
    "            old_vocab_obj = old_vocab[word]\n",
    "            new_vocab[word] = gensim.models.word2vec.Vocab(index=new_index, count=old_vocab_obj.count)\n",
    "        m.wv.vocab = new_vocab\n",
    "\n",
    "    return (m1, m2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "326d6c77",
   "metadata": {},
   "outputs": [],
   "source": [
    "def smart_procrustes_align_gensim(base_embed, other_embed, words=None):\n",
    "    \"\"\"\n",
    "    Procrustes align two gensim word2vec models (to allow for comparison between same word across models).\n",
    "    Code ported from HistWords <https://github.com/williamleif/histwords> by William Hamilton <wleif@stanford.edu>.\n",
    "        (With help from William. Thank you!)\n",
    "\n",
    "    First, intersect the vocabularies (see `intersection_align_gensim` documentation).\n",
    "    Then do the alignment on the other_embed model.\n",
    "    Replace the other_embed model's syn0 and syn0norm numpy matrices with the aligned version.\n",
    "    Return other_embed.\n",
    "\n",
    "    If `words` is set, intersect the two models' vocabulary with the vocabulary in words (see `intersection_align_gensim` documentation).\n",
    "    \"\"\"\n",
    "\n",
    "    # patch by Richard So [https://twitter.com/richardjeanso] (thanks!) to update this code for new version of gensim\n",
    "    base_embed.init_sims()\n",
    "    other_embed.init_sims()\n",
    "\n",
    "    # make sure vocabulary and indices are aligned\n",
    "    in_base_embed, in_other_embed = intersection_align_gensim(base_embed, other_embed, words=words)\n",
    "\n",
    "    # get the embedding matrices\n",
    "    base_vecs = in_base_embed.wv.syn0norm\n",
    "    other_vecs = in_other_embed.wv.syn0norm\n",
    "\n",
    "    # just a matrix dot product with numpy\n",
    "    m = other_vecs.T.dot(base_vecs)\n",
    "    # SVD method from numpy\n",
    "    u, _, v = np.linalg.svd(m)\n",
    "    # another matrix operation\n",
    "    ortho = u.dot(v)\n",
    "    # Replace original array with modified one\n",
    "    # i.e. multiplying the embedding matrix (syn0norm)by \"ortho\"\n",
    "    other_embed.wv.syn0norm = other_embed.wv.syn0 = (other_embed.wv.syn0norm).dot(ortho)\n",
    "\n",
    "    return other_embed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "bd9ae823",
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_sov_to_presov = smart_procrustes_align_gensim(wv_presov, wv_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "06dc7624",
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_postsov_to_sov = smart_procrustes_align_gensim(wv_sov, wv_postsov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fc89db71",
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_pos_sov_to_presov = smart_procrustes_align_gensim(wv_pos_presov, wv_pos_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "886f2a04",
   "metadata": {},
   "outputs": [],
   "source": [
    "wv_pos_postsov_to_sov = smart_procrustes_align_gensim(wv_pos_sov, wv_pos_postsov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "aa64437c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_sov_to_presov = smart_procrustes_align_gensim(ft_presov, ft_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "bd8c0c82",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_postsov_to_sov = smart_procrustes_align_gensim(ft_sov, ft_postsov)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4415995a",
   "metadata": {},
   "source": [
    "#### Получаем нужные векторы"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "5a9b66ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "nouns = rse['lemma'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a9fcd4b0",
   "metadata": {},
   "outputs": [],
   "source": [
    "noun_dict = {}\n",
    "for noun in nouns:\n",
    "    noun_dict[noun] = {}\n",
    "    # word2vec на леммах\n",
    "    noun_dict[noun]['wv_presov'] = wv_presov[noun] if noun in wv_presov else None\n",
    "    noun_dict[noun]['wv_sov'] = wv_sov_to_presov[noun] if noun in wv_sov_to_presov else None\n",
    "    noun_dict[noun]['wv_postsov'] = wv_postsov_to_sov[noun] if noun in wv_postsov_to_sov else None\n",
    "    # word2vec с частями речи\n",
    "    lemma = noun + '_S'\n",
    "    noun_dict[noun]['wvpos_presov'] = wv_pos_presov[lemma] if lemma in wv_pos_presov else None\n",
    "    noun_dict[noun]['wvpos_sov'] = wv_pos_sov_to_presov[lemma] if lemma in wv_pos_sov_to_presov else None\n",
    "    noun_dict[noun]['wvpos_postsov'] = wv_pos_postsov_to_sov[lemma] if lemma in wv_pos_postsov_to_sov else None\n",
    "    # fasttext\n",
    "    noun_dict[noun]['ft_presov'] = ft_presov[noun] if noun in ft_presov else None\n",
    "    noun_dict[noun]['ft_sov'] = ft_sov_to_presov[noun] if noun in ft_sov_to_presov else None\n",
    "    noun_dict[noun]['ft_postsov'] = ft_postsov_to_sov[noun] if noun in ft_postsov_to_sov else None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1d7ce036",
   "metadata": {},
   "outputs": [],
   "source": [
    "noun_clean = {noun: noun_dict[noun] for noun in noun_dict\n",
    "              if not [item for item in list(noun_dict[noun].values()) if item is None]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "50be3209",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "111"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(noun_clean)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9473ed22",
   "metadata": {},
   "source": [
    "#### Рассчитываем изменения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "50a5e1ab",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2697333abc35423d8056f557bebbd0c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/111 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for idx, verb in tqdm(list(noun_clean.items())):\n",
    "    # word2vec на леммах\n",
    "    verb['wv_pre_to_sov'] = round(cityblock(verb['wv_presov'], verb['wv_sov']), 3)\n",
    "    verb['wv_sov_to_post'] = round(cityblock(verb['wv_sov'], verb['wv_postsov']), 3)\n",
    "    verb['wv_pre_to_post'] = round(cityblock(verb['wv_presov'], verb['wv_postsov']), 3)\n",
    "    # word2vec с частями речи\n",
    "    verb['wvpos_pre_to_sov'] = round(cityblock(verb['wvpos_presov'], verb['wvpos_sov']), 3)\n",
    "    verb['wvpos_sov_to_post'] = round(cityblock(verb['wvpos_sov'], verb['wvpos_postsov']), 3)\n",
    "    verb['wvpos_pre_to_post'] = round(cityblock(verb['wvpos_presov'], verb['wvpos_postsov']), 3)\n",
    "    # fasttext\n",
    "    verb['ft_pre_to_sov'] = round(cityblock(verb['ft_presov'], verb['ft_sov']), 3)\n",
    "    verb['ft_sov_to_post'] = round(cityblock(verb['ft_sov'], verb['ft_postsov']), 3)\n",
    "    verb['ft_pre_to_post'] = round(cityblock(verb['ft_presov'], verb['ft_postsov']), 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3b99bc08",
   "metadata": {},
   "source": [
    "#### Ранжируем"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "65eb8e47",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vlem_pre_to_sov = [noun for noun in sorted(noun_clean,\n",
    "                                             key=lambda x: noun_clean[x]['wv_pre_to_sov'],\n",
    "                                             reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "ce39c0f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vpos_pre_to_sov = [noun for noun in sorted(noun_clean,\n",
    "                                             key=lambda x: noun_clean[x]['wvpos_pre_to_sov'],\n",
    "                                             reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "31442d32",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_pre_to_sov = [noun for noun in sorted(noun_clean,\n",
    "                                         key=lambda x: noun_clean[x]['ft_pre_to_sov'],\n",
    "                                         reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4fecd371",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vlem_sov_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                              key=lambda x: noun_clean[x]['wv_sov_to_post'],\n",
    "                                              reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "9edc77cb",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vpos_sov_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                              key=lambda x: noun_clean[x]['wvpos_sov_to_post'],\n",
    "                                              reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6a401463",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_sov_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                          key=lambda x: noun_clean[x]['ft_sov_to_post'],\n",
    "                                          reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "82d8e47e",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vlem_pre_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                              key=lambda x: noun_clean[x]['wv_pre_to_post'],\n",
    "                                              reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "8a804a30",
   "metadata": {},
   "outputs": [],
   "source": [
    "w2vpos_pre_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                              key=lambda x: noun_clean[x]['wvpos_pre_to_post'],\n",
    "                                              reverse=True)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "fee5eaae",
   "metadata": {},
   "outputs": [],
   "source": [
    "ft_pre_to_post = [noun for noun in sorted(noun_clean,\n",
    "                                          key=lambda x: noun_clean[x]['ft_pre_to_post'],\n",
    "                                          reverse=True)]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6b2e1615",
   "metadata": {},
   "source": [
    "#### Собираем в датафрейм"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "37511d37",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>w2vlem_pre_to_sov</th>\n",
       "      <th>w2vpos_pre_to_sov</th>\n",
       "      <th>ft_pre_to_sov</th>\n",
       "      <th>gold_pre_to_sov</th>\n",
       "      <th>w2vlem_sov_to_post</th>\n",
       "      <th>w2vpos_sov_to_post</th>\n",
       "      <th>ft_sov_to_post</th>\n",
       "      <th>gold_sov_to_post</th>\n",
       "      <th>w2vlem_pre_to_post</th>\n",
       "      <th>w2vpos_pre_to_post</th>\n",
       "      <th>ft_pre_to_post</th>\n",
       "      <th>gold_pre_to_post</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>лох</td>\n",
       "      <td>лох</td>\n",
       "      <td>тупик</td>\n",
       "      <td>лох</td>\n",
       "      <td>лох</td>\n",
       "      <td>лох</td>\n",
       "      <td>дядя</td>\n",
       "      <td>линейка</td>\n",
       "      <td>лох</td>\n",
       "      <td>лох</td>\n",
       "      <td>фаворит</td>\n",
       "      <td>лох</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>монстр</td>\n",
       "      <td>монстр</td>\n",
       "      <td>хрен</td>\n",
       "      <td>радикал</td>\n",
       "      <td>маньяк</td>\n",
       "      <td>монстр</td>\n",
       "      <td>маньяк</td>\n",
       "      <td>полоса</td>\n",
       "      <td>роспись</td>\n",
       "      <td>амбиция</td>\n",
       "      <td>хрен</td>\n",
       "      <td>линейка</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>амбиция</td>\n",
       "      <td>амбиция</td>\n",
       "      <td>бригада</td>\n",
       "      <td>роспись</td>\n",
       "      <td>фаворит</td>\n",
       "      <td>фаворит</td>\n",
       "      <td>ясли</td>\n",
       "      <td>дух</td>\n",
       "      <td>дядька</td>\n",
       "      <td>монстр</td>\n",
       "      <td>маньяк</td>\n",
       "      <td>полоса</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>роспись</td>\n",
       "      <td>маньяк</td>\n",
       "      <td>роспись</td>\n",
       "      <td>ядро</td>\n",
       "      <td>линейка</td>\n",
       "      <td>маньяк</td>\n",
       "      <td>монстр</td>\n",
       "      <td>блин</td>\n",
       "      <td>амбиция</td>\n",
       "      <td>дядька</td>\n",
       "      <td>монстр</td>\n",
       "      <td>ядро</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>маньяк</td>\n",
       "      <td>роспись</td>\n",
       "      <td>эшелон</td>\n",
       "      <td>хрен</td>\n",
       "      <td>монстр</td>\n",
       "      <td>наложение</td>\n",
       "      <td>фаворит</td>\n",
       "      <td>головка</td>\n",
       "      <td>линейка</td>\n",
       "      <td>роспись</td>\n",
       "      <td>бригада</td>\n",
       "      <td>роспись</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>земля</td>\n",
       "      <td>земля</td>\n",
       "      <td>проникновение</td>\n",
       "      <td>обоснование</td>\n",
       "      <td>путь</td>\n",
       "      <td>путь</td>\n",
       "      <td>предательство</td>\n",
       "      <td>наволочка</td>\n",
       "      <td>облако</td>\n",
       "      <td>карман</td>\n",
       "      <td>лечение</td>\n",
       "      <td>сверстник</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>путь</td>\n",
       "      <td>путь</td>\n",
       "      <td>лишение</td>\n",
       "      <td>понедельник</td>\n",
       "      <td>стол</td>\n",
       "      <td>стена</td>\n",
       "      <td>стена</td>\n",
       "      <td>издательство</td>\n",
       "      <td>земля</td>\n",
       "      <td>любовь</td>\n",
       "      <td>установление</td>\n",
       "      <td>понедельник</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>любовь</td>\n",
       "      <td>любовь</td>\n",
       "      <td>наложение</td>\n",
       "      <td>стипендия</td>\n",
       "      <td>земля</td>\n",
       "      <td>земля</td>\n",
       "      <td>размышление</td>\n",
       "      <td>понедельник</td>\n",
       "      <td>любовь</td>\n",
       "      <td>стена</td>\n",
       "      <td>проникновение</td>\n",
       "      <td>верховье</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>стена</td>\n",
       "      <td>стена</td>\n",
       "      <td>лечение</td>\n",
       "      <td>сверстник</td>\n",
       "      <td>стена</td>\n",
       "      <td>стол</td>\n",
       "      <td>установление</td>\n",
       "      <td>сверстник</td>\n",
       "      <td>стена</td>\n",
       "      <td>земля</td>\n",
       "      <td>предательство</td>\n",
       "      <td>наволочка</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>стол</td>\n",
       "      <td>стол</td>\n",
       "      <td>любовь</td>\n",
       "      <td>блондин</td>\n",
       "      <td>любовь</td>\n",
       "      <td>любовь</td>\n",
       "      <td>лечение</td>\n",
       "      <td>блондин</td>\n",
       "      <td>стол</td>\n",
       "      <td>стол</td>\n",
       "      <td>жесть</td>\n",
       "      <td>блондин</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>111 rows × 12 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    w2vlem_pre_to_sov w2vpos_pre_to_sov  ft_pre_to_sov gold_pre_to_sov  \\\n",
       "0                 лох               лох          тупик             лох   \n",
       "1              монстр            монстр           хрен         радикал   \n",
       "2             амбиция           амбиция        бригада         роспись   \n",
       "3             роспись            маньяк        роспись            ядро   \n",
       "4              маньяк           роспись         эшелон            хрен   \n",
       "..                ...               ...            ...             ...   \n",
       "106             земля             земля  проникновение     обоснование   \n",
       "107              путь              путь        лишение     понедельник   \n",
       "108            любовь            любовь      наложение       стипендия   \n",
       "109             стена             стена        лечение       сверстник   \n",
       "110              стол              стол         любовь         блондин   \n",
       "\n",
       "    w2vlem_sov_to_post w2vpos_sov_to_post ft_sov_to_post gold_sov_to_post  \\\n",
       "0                  лох                лох           дядя          линейка   \n",
       "1               маньяк             монстр         маньяк           полоса   \n",
       "2              фаворит            фаворит           ясли              дух   \n",
       "3              линейка             маньяк         монстр             блин   \n",
       "4               монстр          наложение        фаворит          головка   \n",
       "..                 ...                ...            ...              ...   \n",
       "106               путь               путь  предательство        наволочка   \n",
       "107               стол              стена          стена     издательство   \n",
       "108              земля              земля    размышление      понедельник   \n",
       "109              стена               стол   установление        сверстник   \n",
       "110             любовь             любовь        лечение          блондин   \n",
       "\n",
       "    w2vlem_pre_to_post w2vpos_pre_to_post ft_pre_to_post gold_pre_to_post  \n",
       "0                  лох                лох        фаворит              лох  \n",
       "1              роспись            амбиция           хрен          линейка  \n",
       "2               дядька             монстр         маньяк           полоса  \n",
       "3              амбиция             дядька         монстр             ядро  \n",
       "4              линейка            роспись        бригада          роспись  \n",
       "..                 ...                ...            ...              ...  \n",
       "106             облако             карман        лечение        сверстник  \n",
       "107              земля             любовь   установление      понедельник  \n",
       "108             любовь              стена  проникновение         верховье  \n",
       "109              стена              земля  предательство        наволочка  \n",
       "110               стол               стол          жесть          блондин  \n",
       "\n",
       "[111 rows x 12 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ranked_nouns = pd.DataFrame({\n",
    "    'w2vlem_pre_to_sov': w2vlem_pre_to_sov,\n",
    "    'w2vpos_pre_to_sov': w2vpos_pre_to_sov,\n",
    "    'ft_pre_to_sov': ft_pre_to_sov,\n",
    "    'gold_pre_to_sov': gold_pre_to_sov,\n",
    "    'w2vlem_sov_to_post': w2vlem_sov_to_post,\n",
    "    'w2vpos_sov_to_post': w2vpos_sov_to_post,\n",
    "    'ft_sov_to_post': ft_sov_to_post,\n",
    "    'gold_sov_to_post': gold_sov_to_post,\n",
    "    'w2vlem_pre_to_post': w2vlem_pre_to_post,\n",
    "    'w2vpos_pre_to_post': w2vpos_pre_to_post,\n",
    "    'ft_pre_to_post': ft_pre_to_post,\n",
    "    'gold_pre_to_post': gold_pre_to_post\n",
    "})\n",
    "ranked_nouns"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b29649cd",
   "metadata": {},
   "source": [
    "#### Считаем корреляцию с золотым стандартом (RuShiftEval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5e96ba56",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.004308529308529308, pvalue=0.9642033377767343)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vlem_pre_to_sov, gold_pre_to_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "e0a90f8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.040400140400140394, pvalue=0.6737583232800901)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vpos_pre_to_sov, gold_pre_to_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "628bcd06",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.03484555984555985, pvalue=0.7165486877420728)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(ft_pre_to_sov, gold_pre_to_sov)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "4ce82d38",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.03747806247806248, pvalue=0.6961495864808469)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vlem_sov_to_post, gold_sov_to_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "64bb02e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.02733415233415233, pvalue=0.7758154547124656)"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vpos_sov_to_post, gold_sov_to_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b080740a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.07847490347490348, pvalue=0.4129640066020368)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(ft_sov_to_post, gold_sov_to_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "3be7523d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.13793436293436295, pvalue=0.14882659186922198)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vlem_pre_to_post, gold_pre_to_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d07dfe92",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=-0.1158037908037908, pvalue=0.22615067747649165)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(w2vpos_pre_to_post, gold_pre_to_post)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "387d9efe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SpearmanrResult(correlation=0.004852579852579852, pvalue=0.9596867712809428)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spearmanr(ft_pre_to_post, gold_pre_to_post)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}