ab_testing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fae3c5d6-7d6a-477d-b2bf-eb9c17a48174",
   "metadata": {},
   "source": [
    "## Download and read data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1266fe6c-fdf2-4131-a3c2-f9d9d56de486",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "import pandahouse\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ace2166a-099c-4742-b8c1-5c9a34189af5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def select(query):\n",
    "    return pandahouse.read_clickhouse(query, connection=connection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c6528117-cea8-4b78-a5c8-45ef28d287e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "connection = {\n",
    "    'database': 'simulator_20221120',\n",
    "    'host': 'https://clickhouse.lab.karpov.courses',\n",
    "    'user': 'student',\n",
    "    'password': 'dpo_python_2020'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "318e2c81-4a90-4ada-b33c-3fd070da738d",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = '''\n",
    "select exp_group, user_id,\n",
    "sum(action = 'like') as likes,\n",
    "sum(action = 'view') as views,\n",
    "likes / views as ctr\n",
    "from {db}.feed_actions \n",
    "where toDate(time) between '2022-11-02' and '2022-11-08' and exp_group in (2, 1)\n",
    "group by exp_group, user_id\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "ef3b4ce8-fd84-4bba-b9b2-4f031bb43917",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = select(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "d3d44683-0a11-4fbc-bfa9-4962351e5e50",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exp_group</th>\n",
       "      <th>user_id</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>109963</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>26117</td>\n",
       "      <td>32</td>\n",
       "      <td>141</td>\n",
       "      <td>0.226950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>138232</td>\n",
       "      <td>18</td>\n",
       "      <td>73</td>\n",
       "      <td>0.246575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>26295</td>\n",
       "      <td>33</td>\n",
       "      <td>122</td>\n",
       "      <td>0.270492</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>18392</td>\n",
       "      <td>7</td>\n",
       "      <td>32</td>\n",
       "      <td>0.218750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20026</th>\n",
       "      <td>1</td>\n",
       "      <td>132567</td>\n",
       "      <td>45</td>\n",
       "      <td>296</td>\n",
       "      <td>0.152027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20027</th>\n",
       "      <td>1</td>\n",
       "      <td>30872</td>\n",
       "      <td>11</td>\n",
       "      <td>59</td>\n",
       "      <td>0.186441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20028</th>\n",
       "      <td>2</td>\n",
       "      <td>25383</td>\n",
       "      <td>42</td>\n",
       "      <td>127</td>\n",
       "      <td>0.330709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20029</th>\n",
       "      <td>2</td>\n",
       "      <td>139020</td>\n",
       "      <td>24</td>\n",
       "      <td>114</td>\n",
       "      <td>0.210526</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20030</th>\n",
       "      <td>2</td>\n",
       "      <td>17310</td>\n",
       "      <td>9</td>\n",
       "      <td>82</td>\n",
       "      <td>0.109756</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20031 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       exp_group  user_id  likes  views       ctr\n",
       "0              1   109963      3     15  0.200000\n",
       "1              1    26117     32    141  0.226950\n",
       "2              1   138232     18     73  0.246575\n",
       "3              1    26295     33    122  0.270492\n",
       "4              1    18392      7     32  0.218750\n",
       "...          ...      ...    ...    ...       ...\n",
       "20026          1   132567     45    296  0.152027\n",
       "20027          1    30872     11     59  0.186441\n",
       "20028          2    25383     42    127  0.330709\n",
       "20029          2   139020     24    114  0.210526\n",
       "20030          2    17310      9     82  0.109756\n",
       "\n",
       "[20031 rows x 5 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3026e47-2138-4a83-a103-18cc5a6f71a7",
   "metadata": {},
   "source": [
    "## Histogram "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "2fe6e387-3cf4-45e3-b992-6aa86d52ddbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='ctr', ylabel='Count'>"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtEAAAH0CAYAAAD7QHEfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAvXklEQVR4nO3de5hddX0v/vfeM8mQSQghIYQJoBBADNBqIR48/rAqSMMlJNoaweC9mnpphfaIBC8JiNomeIEWEfj1PLRIxQNWSQkQsKWIeCwKVSgmWg1RqOQ6IQKJuczMOn9QIpHM7L2Smb1nMq/X8+R5Mvuz9lqf2es7M+9Z893fVSmKoggAAFC3arMbAACAoUaIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKKm12Q0005NPbkpPT+OWyZ4wYUw6O59p2PEYfIwBjIHhzfnHGBhaqtVK9t9/9C5rwzpE9/QUDQ3Rzx2T4c0YwBgY3px/jIG9g+kcAABQkhANAAAlDevpHAAAw0l3d1eefHJdurq2NbuVQaW1dWT2339iWlrqj8ZCNADAMPHkk+uyzz7tGT36oFQqlWa3MygURZFNm57Kk0+uywEHdNT9PNM5AACGia6ubRk9eqwA/TyVSiWjR48tfXVeiAYAGEYE6BfanddEiAYAgJKEaAAAKEmIBgCAkoRoAAAGpa6urma30CtL3AEADHPr16/LF76wKA899IOMGtWeN795TqZPPz3veMdb8r/+17ycdNLvZ/PmzXnXu+bkne98T04/fUY+/emLM3LkyPzyl/+VH/3okRx99Evz8Y9fkoMO6nuZuO9979/yhS8syoYNnTn11NPz858/munTz8hZZ70ht99+a2699RuZOvXYLF16e97whj/KnDlvzxe+sCj33/9/09a2T8466w15+9vfnWq1mv/9v6/JL3/5X5k//9IkyapVT2T27Jm5555/S2tra/70T+fmuON+Nw8++L384he/yPHHn5CPfnRBxo7db49fM1eiAQCGsZ6ennzkI3+eI498Sb7xjTty+eVX5aabbszy5cty0UXzs2jRp/PkkxvyN3/z+Rx11Ety+ukzdjz3rruW5p3vfE9uu+2fc+SRL8kll3y8z2Nt3LgxH//4hXnf+/40t932L3nRi16c//iPh3baZtmyH2Xy5IPzT/90Z97xjnfnC19YlE2bnslNNy3OlVdem6VLb8/tt/9T3Z/f0qW35aKL5mfx4qVpaWnJ5Zd/ttwL1AshGgBgGFu+fFk2bnwy73rXezNixIgcfPAhmTnzDfmXf7kr/+N/vDKve90pOe+89+e73/1OLrjgozs991Wv+v/y8pcfn5EjR2bu3A/kRz/6j6xZs7rXY333u/fl8MOn5DWvOTmtra2ZPfucTJhwwE7bTJhwQN70pnPS2tqa1tYR+Zd/uSt/8id/mvb20enomJxzzjk3S5feXvfnN336GZky5ciMGjUq73nP+3P33d9Md3d3uRdpF0znAAAYxlavXpXOzvU57bTX7nisu7snL3vZy5MkM2f+Yf7xH2/K2972ruy337idnnvggQft+H97e3vGjh2b9evXZ9Kkg7Ir69evz4EHTtrxcaVSycSJB+60zaRJv6n/6lcb09XVtdMUkYMO6sj69evq/vyef7yDDupIV1dXfvWrjRk/fkLd+9gVIRoAYBibNGlSOjom56tf/cYLat3d3Vm06NM57bQz841vfC1nnjkzhxxy6I762rW/ueq8efPmPPXUUznggANesJ/nHHDAAfnOd9bu+Lgoiqxbt/a3tvrNjU/2229cWltbs3r1qhx++JQkyZo1q3PAAROTJKNGjcqWLVt2bN/Z2fmCY65du2bH/9esWZ3W1tYX/DKwO0znAAAYxqZOPTbt7e254Ya/y9atW9Ld3Z1HH/1Zli//Ub785etSqVRy0UXzM2fO2/KpTy3YaSrEd7/7f/PQQz/M9u3b87d/+6Uce+xxvV6FTpL/+T9PyooVP8u9996Trq6ufP3rN6Wzc32v27e0tOTkk0/Ntddelc2bN2X16lX5P//nHzJ9+hlJkqOOekkeeujfs3r16jzzzDO54YbrXrCPu+66IytXPpotW7bkb//26rz2taekpaVlD16xZwnRAADDWEtLSxYtujw//el/ZvbsWTnzzNfnr/7qU3nwwe/nq1/9h3z845ekpaUl5577jlQqyQ03/N2O55566vRcd921OeOMk/OTn/w4n/jEpX0ea9y4cbn00r/KVVf9dc4885SsXLkyL33pMRkxYkSvzzn//AsyatSovPnNs/KBD7wnp556Ws48c2aS5BWveGVOPvkP8s53npM//uO35lWvevULnj99+hn5zGcuzsyZ07Nt27acf/4Fu/dC/ZZKURRFv+xpCOrsfCY9PY379CdO3Dfr1j3dsOMx+BgDGAPDm/NPs8fA6tW/yEEHvbhf9vXpT1+ciRMPzNy5H9jtffT09OSNbzwjCxZ8KscfP61f+nq+P/3TuTuWz6tlV69NtVrJhAljdrm9OdH0u/bRbc+fzvRCRbJ509aG9QMADB733//dHHPMcWlra8tXvnJ9kiLHHntcs9sqTYim/1WSr9yxvNfyW888Ju1j2vreh6ANAEPSQw/9IB/+8Id2WfvmN7+dRx55OJdc8vFs3749hx12eD7zmc+lrW2fBne550znMJ2j37WPaeszRJ97xjH5h9uX9bmPOadPzeZn9r4QPVzGAL0zBoY3559mj4H+nM6xtyk7ncMbCwEAoCQhGgAAShKiAQCgJCEaAABKsjoHAAB12X+/fdI6svcbo+yurm3b8+SvttTecBARogEAqEvryBFZ/rfX9/t+p77n7Ulqh+grr7w83/rW3Vm16olcf/1XM2XKkf3eS71M5wAAYEh49atfmyuvvDYHHdTR7FZciQYAYGh42cte3uwWdnAlGgAAShKiAQCgJCEaAABKEqIBAKAkbywEAKAuXdu2//dydP2/33pcfvll+da3/jUbNnTm/PM/mLFj98sNN9zU7/3UQ4gGAKAuz94QpXk3RTn//Aty/vkXNO34z2c6BwAAlCREAwBASUI0AACUJEQDAEBJQjQAAJQkRAMAQEmWuAMAoC77jWvPyBEt/b7fbdu786uNm2tu96tfbcyll87PL3/5XxkxYkQOOeRFueCCj2b//ffv955qEaIBAKjLyBEt+f+//lC/7/e9f/iyurarVCqZM+ftOf74aUmSL37xilx99d/koovm93tPtZjOAQDAkDB27H47AnSSHHvscVm9enVTehGiAQAYcnp6evKNb/xjTjrp95tyfCEaAIAh5wtfuCzt7aPyR3/05qYc35xoAACGlCuvvDz/9V+PZeHCL6Rabc41YSEaAIAh45prvpif/GR5LrvsiowcObJpfQjRAAAMCY8+uiJf/vJ1OfTQF+V973t3kqSjY3L+8i8/2/BeGhKiFy5cmDvvvDO//OUvc+utt+YlL3lJkmTlypWZN29eNm7cmHHjxmXhwoU57LDD9qgGAMDA2La9u+7l6Mrutx5TphyR++57oN+PvzsaEqJPOeWUvP3tb8+555670+MLFizInDlzMmvWrCxevDjz58/P9ddfv0c1AAAGRj03RBkuGjITe9q0aeno6Njpsc7OzixbtiwzZsxIksyYMSPLli3Lhg0bdrsGAACN0LQ50atWrcqkSZPS0vLsrSNbWlpy4IEHZtWqVSmKYrdq48ePb9anAwDAMDKs31g4YcKYhh9z4sR9G37MRtuytSujR7f1Wq+k0mc9SVqq1b32tdpbPy/qZwwMb84/zRwDa9dW09JSSaVSaVoPg1FRFKmWzB5NC9EdHR1Zs2ZNuru709LSku7u7qxduzYdHR0pimK3amV1dj6Tnp5iAD67XZs4cd+sW/d0w47XLO1j2rJp09Ze60WKPutJ0t3Ts1e+VsNlDNA7Y2B4c/5p9hioVlvzq19tzOjRYwXp/1YURTZteirVausLzk21Wun1omvTQvSECRMyderULFmyJLNmzcqSJUsyderUHVMydrcGAMCu7b//xDz55Lo888zGZrcyqLS2jsz++08s9ZxKURQDfin2U5/6VO66666sX78++++/f8aNG5fbbrstK1asyLx58/LUU09l7NixWbhwYaZMmZIku10rw5XogdE+pi1fuWN5r/Vzzzgm/3D7sj73Mef0qdn8TN9Xq4ei4TIG6J0xMLw5/xgDQ0tfV6IbEqIHKyF6YAjRvRsuY4DeGQPDm/OPMTC09BWim3OzcQAAGMKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAklqb3QBDT/votqTSe71S6aMIALAXEKIpr5J85Y7lvZbPPeOYBjYDANB4pnMAAEBJQjQAAJQkRAMAQElCNAAAlCREAwBASYMiRP/rv/5r3vCGN2TWrFmZOXNm7rrrriTJypUrc/bZZ2f69Ok5++yz8/Of/3zHc/qqAQDAQGp6iC6KIh/5yEeyaNGiLF68OIsWLcqFF16Ynp6eLFiwIHPmzMmdd96ZOXPmZP78+Tue11cNAAAGUtNDdJJUq9U8/fTTSZKnn346Bx54YJ588sksW7YsM2bMSJLMmDEjy5Yty4YNG9LZ2dlrDQAABlrTb7ZSqVRy+eWX5wMf+EDa29uzadOmXHvttVm1alUmTZqUlpaWJElLS0sOPPDArFq1KkVR9FobP3583ceeMGHMgHxOfZk4cd+GH7O/bdnaldGj23qtV1LZo3qStFSre8VrtSt76+dF/YyB4c35xxjYOzQ9RHd1deWaa67JVVddlRNOOCEPPvhgzj///CxatGjAj93Z+Ux6eooBP85zJk7cN+vWPd2w4w2U9jFt2bRpa6/1IsUe1ZOku6dnr3itftveMgbYfcbA8Ob8YwwMLdVqpdeLrk0P0cuXL8/atWtzwgknJElOOOGEjBo1Km1tbVmzZk26u7vT0tKS7u7urF27Nh0dHSmKotcaAAAMtKbPiT7ooIOyevXqPProo0mSFStWpLOzMy9+8YszderULFmyJEmyZMmSTJ06NePHj8+ECRN6rQEAwEBr+pXoiRMn5uKLL855552XSqWSJPnMZz6TcePG5eKLL868efNy1VVXZezYsVm4cOGO5/VVAwCAgdT0EJ0kM2fOzMyZM1/w+BFHHJGbb755l8/pqwYAAAOp6dM5AABgqBGiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQhGgAAShoU60TDUDS2vTWVFL3Wi1Ty1OauBnYEADSKEA27qZIiK278Wq/1I97ypgZ2AwA0kukcAABQkhANAAAlmc7BsFRrPnNiTjMA0DshmmGp1nzmxJxmAKB3pnMAAEBJQjQAAJQkRAMAQElCNAAAlOSNhTBAqtVq9mvf+bHuLVuyX3tLEqt/AMBQJkTDQCleuALI6NFt2bRpaxKrfwDAUGY6BwAAlCREAwBASUI0AACUJEQDAEBJQjQAAJQkRAMAQElCNAAAlCREAwBASUI0AACUJEQDAEBJQjQAAJQkRAMAQElCNAAAlNTa7AZguKpWq9mvve9tilTy1OauxjQEANRNiIZmKYqsuPFrfW5yxFve1KBmAIAyTOcAAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkqzOAb2otQRdtVJpXDMAwKAiRENvaixBd9Sc2Q1sBgAYTEznAACAkoRoAAAoSYgGAICShGgAACjJGwvZK41tb00lRa91K2sAAHtCiGavVImVNQCAgWM6BwAAlCREAwBASUI0AACUJEQDAEBJQjQAAJQkRAMAQElCNAAAlCREAwBASUI0AACU5I6FvED76Lakj7tiV9wyGwAY5oRoXqiSfOWO5b2Wzz3jmAY2AwAw+JjOAQAAJQnRAABQkhANAAAlCdEAAFCSEA0AACUJ0QAAUJIQDQAAJVknGgaxarWa/dp7rxep5KnNXY1rCABIIkTD4FYUWXHj13otH/GWNzWwGQDgOaZzAABASUI0AACUVHeIvuOOO3b5+NKlS/utGQAAGArqDtEf+9jHdvn4/Pnz97iJrVu3ZsGCBfmDP/iDnHXWWfnEJz6RJFm5cmXOPvvsTJ8+PWeffXZ+/vOf73hOXzUAABhINd9Y+PjjjydJiqLY8f/n10aOHLnHTVx22WVpa2vLnXfemUqlkvXr1ydJFixYkDlz5mTWrFlZvHhx5s+fn+uvv75mDQAABlLNEH3qqaemUqmkKIqceuqpO9UOOOCA/Nmf/dkeNbBp06bccsst+da3vpVKpbJjv52dnVm2bFmuu+66JMmMGTNy6aWXZsOGDSmKotfa+PHj96gfAACopWaI/vGPf5wkeetb35obbrih3xt4/PHHM27cuFx55ZW5//77M3r06Jx33nnZZ599MmnSpLS0tCRJWlpacuCBB2bVqlUpiqLXWpkQPWHCmH7/fGqZOHHfhh+zrC1buzJ6dFuv9UoqA1pPkpZqdY9eq+4tW/o+RiU1e6i5zW7WdzzWDz20tFQzcWIfC0kzKA2F7wMMHOcfY2DvUPc60QMRoJOku7s7jz/+eI455phceOGFeeihh/K+970vV1xxxYAc7/k6O59JT08x4Md5zsSJ+2bduqcbdrzd1T6mLZs2be21XqQY0HqSdPf07NFrtV97S9/HKFKzh5rb7EZ99Ojnvbb90EN3d082DIExxW8Mle8DDAznH2NgaKlWK71edK07RD/++OO5/PLLs3z58mzevHmn2j333LPbzXV0dKS1tTUzZsxIkrzsZS/L/vvvn3322Sdr1qxJd3d3Wlpa0t3dnbVr16ajoyNFUfRaAwCAgVZ3iP7whz+cQw89NBdeeGFGjRrVbw2MHz8+J554Yr7zne/kpJNOysqVK9PZ2ZnDDjssU6dOzZIlSzJr1qwsWbIkU6dO3TFdo68aAAAMpLpD9E9/+tPceOONqVb7//4sl1xyST760Y9m4cKFaW1tzaJFizJ27NhcfPHFmTdvXq666qqMHTs2Cxcu3PGcvmoAADCQ6g7Rr3jFK7Js2bIcd9xx/d7EoYcemi9/+csvePyII47IzTffvMvn9FUDAICBVHeIPvjgg/Oe97wnp556ag444ICdauedd16/NwYAAINV3SH617/+dV73utelq6srq1evHsieAABgUKs7RP/lX/7lQPYBAABDRqkl7npz6KGH9kszAAAwFNQdop9/++/nPHeb7uXLl/d/ZwAAMEjVHaKfu/33c9atW5crr7wy06ZN6/emAABgMNvtRZ8nTpyYj33sY/n85z/fn/0AAMCgt0d3Tnn00Ufz61//ur96AQCAIaHu6Rxz5szZMQc6eXbJu5/97Gf54Ac/OCCNAQDAYFV3iJ49e/ZOH48aNSovfelLc9hhh/V3TwAAMKjVHaLf+MY3DmQfsJNqtZL2MW29b1AkmzdtbVxDAADPU3eI3r59e770pS9l8eLFWbt2bQ488MDMmjUr73vf+zJy5MiB7JFhqCiSr9zR+9KJc06f2sBuAAB2VneIvuyyy/Lwww/nkksuyeTJk/PEE0/kqquuyjPPPJOPfvSjA9kjAAAMKnWH6KVLl2bx4sXZf//9kyRTpkzJMccck1mzZgnRAAAMK3Uvcff8OxXW8zgAAOyt6g7Rp512Wt7//vfn29/+dlasWJF77703H/zgB3PaaacNZH8AADDo1D2d44ILLsiXvvSlfPKTn8zatWszadKknHnmmXn/+98/kP0BAMCgU/NK9IMPPpjLLrssI0eOzHnnnZdvfvObeeihh3LXXXdl27ZtWbZsWSP6BACAQaNmiL7mmmvyile8Ype1E088MVdffXW/NwUAAINZzRC9fPnyvPrVr95l7VWvelUeeeSRfm8KAAAGs5oh+plnnsn27dt3Wevq6sqmTZv6vSkAABjMaoboKVOm5L777ttl7b777suUKVP6vSkAABjMaobod77znVmwYEHuuuuu9PT0JEl6enpy11135eKLL8673vWuAW8SAAAGk5pL3J111llZv359Lrzwwmzfvj3jxo3Lxo0bM2LEiHzoQx/KjBkzGtEnAAAMGnWtE/2ud70rs2fPzg9+8INs3Lgx48aNy+/93u9lzJgxA90f0IdqtZr92nuvF6nkqc1djWsIAIaJum+2MmbMmF5X6QCapCiy4sav9Vo+4i1vamAzADB81H3bbwAA4FlCNAAAlCREAwBASUI0AACUJEQDAEBJda/OAQw9lsADgIEhRMPezBJ4ADAgTOcAAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICSWpvdAOyOkZUiI9pbeq1XK5UGdgMADDdCNENT0ZMVN36t1/JRc2Y3sBkAYLgxnQMAAEoSogEAoCQhGgAAShKiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQ3WwEGtfbRbUmtG1AWyeZNWxvSDwAkQjQw2FWSr9yxvM9N5pw+tUHNAMCzTOcAAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkgZViL7yyitz9NFH5z//8z+TJD/84Q8zc+bMTJ8+Pe9+97vT2dm5Y9u+agAAMJAGTYj+0Y9+lB/+8Ic5+OCDkyQ9PT254IILMn/+/Nx5552ZNm1aPvvZz9asAQDAQBsUIXrbtm355Cc/mYsvvnjHY4888kja2toybdq0JMk555yTpUuX1qwBAMBAGxQ3W7niiisyc+bMHHLIITseW7VqVSZPnrzj4/Hjx6enpycbN27sszZu3LhGtg7UUPOOg+42CMAQ1PQQ/YMf/CCPPPJIPvzhDzf82BMmjGn4MSdO3Lfhxyxry9aujB7d1mu9ksqA1uvbR/reR2UP6/2xj17qOx5rYg/PaWmpZuLE9r572ENbtnbl6/f8rNf6H772yD6/LmqNxyRpqVaHxNfWc4ZSr/Q/5x9jYO/Q9BD9/e9/PytWrMgpp5ySJFm9enX++I//OG9729vyxBNP7Nhuw4YNqVarGTduXDo6OnqtldHZ+Ux6eop++TzqMXHivlm37umGHW93tY9py6Y+rgwWKQa0Xt8+0vc+ij2s98c+dlEfPfp5r22TetipXCTbNm3u4+mVPLW5q+8ea6g1nrp7evr8uqj1/Hr2MZgMle8DDAznH2NgaKlWK71edG16iJ47d27mzp274+OTTz45V199dY488sjcdNNNeeCBBzJt2rR89atfzWmnnZYkOe6447Jly5Zd1oASiiIrbvxar+Uj3vKmBjYDAENH00N0b6rVahYtWpQFCxZk69atOfjgg3PZZZfVrAEAwEAbdCH67rvv3vH/448/Prfeeusut+urBgAAA2lQLHEHAABDiRANAAAlCdEAAFCSEA0AACUJ0QAAUJIQDQAAJQnRAABQkhANAAAlDbqbrQCDR7VazX7tfW9TpJKnNnc1piEAGCSEaAalEenJ7Ncc3mu90sBehrWiyIobv9bnJke85U0NagYABg8hmkGpUvTkniuu67U+46K5DeyGgVStVtI+pq3XeqXiVyYABh8hGmiqoki+csfyXuvnnnFMA7sBgPp4YyEAAJQkRAMAQElCNAAAlGRONLBHKt4YCMAwJEQDe8wbAwEYbkznAACAkoRoAAAoSYgGAICShGgAAChJiAYAgJKEaAAAKEmIBgCAkoRoAAAoSYgGAICShGgAACjJbb+HmfbRbUml720qlRobAAAMc0L0cFNJvnLH8j43OfeMYxrUDHuDSqWS2a85vNf6iPQ0sBsAaAwhGtgzRZF7rriu1/KMi+Y2sBkAaAxzogEAoCQhGgAAShKiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQhGgAAShKiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQhGgAAShKiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQhGgAAShKiAQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoKTWZjcA7N0qlUpmv+bwXusj0tPAbgCgfwjRwIAqiiL3XHFdr/UZF80d8B7aR7cllT42KJLNm7YOeB8A7D2EaGDvV0m+csfyXstzTp/awGYA2BuYEw0AACW5Eg0MaiPS0+ec6iQZWSmyeQB7qDkdJDElBGCYEaKBQa1S9PQ5pzpJzhzoedU1poMkpoQADDemcwAAQEmuRNNw9fx5HgBgMBOiabh6/jzfiGXPAAB2l+kcAABQkhANAAAlCdEAAFCSEA0AACUJ0QAAUJLVOYBhr1qtpH1MW6/1SqXW7QoBGG6aHqKffPLJfOQjH8ljjz2WkSNH5sUvfnE++clPZvz48fnhD3+Y+fPnZ+vWrTn44INz2WWXZcKECUnSZw2gjKLo+46E555xTAO7AWAoaPp0jkqlkve85z258847c+utt+bQQw/NZz/72fT09OSCCy7I/Pnzc+edd2batGn57Gc/myR91gAAYKA1PUSPGzcuJ5544o6PX/7yl+eJJ57II488kra2tkybNi1Jcs4552Tp0qVJ0meN4WPEyNZe/8Vf3wGAAdT06RzP19PTkxtvvDEnn3xyVq1alcmTJ++ojR8/Pj09Pdm4cWOftXHjxjWhc5rhkZ+t67V2WOPaAACGoUEVoi+99NK0t7fnrW99a775zW8O+PEmTBgz4Mf4bRMn7tvwYz7flq1dGT269zdQJUkllT632dN6kowYUXvo1dqmVr2tbUTvxUpq9lhzm92s73isiT30W/2/7em52tPxUknfX1u1xn1/jOmWarXur+9mfx+guZx/jIG9w6AJ0QsXLswvfvGLXH311alWq+no6MgTTzyxo75hw4ZUq9WMGzeuz1oZnZ3PpKen6K9PoaaJE/fNunVPN+x4u9I+pi2bNm3tc5siRZ/b7Gk9SbZv7+q70Tq2qVV/+Kdre629KKnZY4oa2+xGffTo573+tZ4/QD30a/2/7em52tPxUiR9fm3VGvf9Maa7e3rq+voeDN8HaB7nH2NgaKlWK71edB0UIfrzn/98HnnkkVx77bUZOXJkkuS4447Lli1b8sADD2TatGn56le/mtNOO61mDRhaKpVKZr/m8Ga3MeDaR7cllWeviu9yOb0i2VzHLywADA5ND9E//elPc8011+Swww7LOeeckyQ55JBD8sUvfjGLFi3KggULdlrGLkmq1WqvNWBoKYoi91xxXa/1GRfNbWA3A6jy7DJ6O/014nnmnD61CU0BsLuaHqKPOuqo/OQnP9ll7fjjj8+tt95augb0k8qzq6AAADvz0xHoU1+roCSDYyWUSuKOgwA0lBAN7BXccRCARmr6zVYAAGCoEaIBAKAkIRoAAEoSogEAoCQhGgAAShKiAQCgJEvcAfSDarVirWqAYUSIBugHRWGtaoDhxHQOAAAoSYgGAICSTOeg341IT2a/5vBmtwEAMGCEaPpdpejJPVdc12t9xkVzG9gNAED/M50DAABKEqIBAKAkIRoAAEoSogEAoCRvLIThrJKMGOnbAACU5acnDHOP/Gxdr7XDGtfGsFfrtuEpks2btjauIQD6JEQDDAK1bhs+5/SpDewGgFrMiQYAgJJcid7LtI9uSyq91yuVPooAANRFiN7bVPr+k/C5ZxzTwGYAAPZOpnMAAEBJrkQDe70R6cns1xzeZx0AyhCiGbZqro9s+vheo1L05J4rruu1PuOiuQ3sBoC9gRDNsNXX+siJNZIBgN6ZEw0AACUJ0QAAUJIQDQAAJZkTDTAEVKuVtI9p63ujItm8aWtjGgIY5oRogCGgKPq+kVKSzDl9aoO6AUCIprRaa+4CAOzthGheoFZItubuEFKpYz1sAKA0P115ASH5N/oMoEPkZix9rYd9WOPaGNLq+euLux4CDC9CNPRBACWp/YtlMrx+uQRAiIaBs4upFF1dPb95rJ6pFrW2GSJXwwdapVLZo3n6e/r8oaJ9dFvfY8bqHgB1E6JhAP32lewRI1qzfXtXkmevZNdz63FXw2srimKPpiDt6fOHjErfK3xY3QOgfkI0wF6i1lrSlYo/XQD0FyEaYC9Ray3pc884ZsB7MGUEGC6EaNgDlo+D32LKCDBMSACwBwbDfGVBHgAaz09fGOIGQ5AHgOGm2uwGAABgqBGiAQCgJNM5AEhSe4k8K2sA/IYQDUCS2kvkWVkD4DdM5wAAgJJciW6grq1bM25MjZe8Us3Gp7c1piEAAHaLEN1IPT257S+v7XOTMy+a26BmAADYXUI0QD+oVCqZ/ZrDe62PSE8DuwFgoAnRAP2gKIrcc8V1vdbP+uifZPZrDk+lWknRU7ygLmQDDC1CNEADPBeyR4xozfbtXS+ozzCVC2BIEaIBBgHTQQCGFiEaYBCoNR3ElWqAwUWIHmZGpKfPq13A0FXr63tPr2bXvKNhnr2iDjAcCNFDTPvotqSPn1G1foBVip4+r3YlrnjBUFXr63tPv7Zr3dEwSc4945g9OgbAUCFEDzWVvn+I+QEGADDw3PYbAABKciUaYAiotXpHf+yjP1YAqTUve2SlyOY9PgpA8wnRAENArdU7ktpznhuxAkitedln7uExar0vJElSJJs3bd3tfVSrlfTs4oY49e4fGB6EaACGjhrvC0mSOadP3aN9nHvGMX3Wa+4fGBaEaAAGjT1dgSipvRSfZfiA/iBEAzB49MMKRLWW4rOKEdAfhGgAKKGem87s6bxs865h8BOi9zLuSAgwsOq56cxbzzxml0F7y9autI9pS6VSyT/cvqzX55t3DYOfEL2XGeg7lgHsiUoyLOYr9xa0R49uy6ZNW2tOKal5tbsfrlS7Gg57RogGIElj1pFO9my+cj1/beuvPpup1tXufrlSXWP+uavh0DcheogZWSlM1wAGRCPWkd5Ttf7algyOPoG935AO0StXrsy8efOycePGjBs3LgsXLsxhhx3W7LYGlukaQJP0x10T6R/98ebGocCUEwazIR2iFyxYkDlz5mTWrFlZvHhx5s+fn+uvv77ZbQHslfrjrom1NOLN0bWOUWs6yGCYUrInb258zp7OP2/E3SMb8QZMQZ3dNWRDdGdnZ5YtW5brrnv2G/qMGTNy6aWXZsOGDRk/fnxd+6hWG/wGlkol7eP2rbFJJaP37eObXnr63ketY9TRwx7vQw+91ltbWzOiq6upPfRrXQ+l97HTGGhSDwNWr2ObSrWas183pfd6inzv77/Wa/3kD8yp/X20H44xpn3Ebj8/SU754Lm77qFaSXqKjKz09NnjiEpP3z1U0mc9SVrTk19v2dZrvb19ZJ89jKwW2dLHz8lKNVl8z8/67OGNrzuq759plUpu+def9lp/w+uOqvk69PWzfFT7yJpBv1YPs157ZL/nhYbnj0GgnnORIvn15t7HbDP0da4qRVEUDeyl3zzyyCO58MILc9ttt+147Iwzzshll12WY489tomdAQCwt6s2uwEAABhqhmyI7ujoyJo1a9Ld3Z0k6e7uztq1a9PR0dHkzgAA2NsN2RA9YcKETJ06NUuWLEmSLFmyJFOnTq17PjQAAOyuITsnOklWrFiRefPm5amnnsrYsWOzcOHCTJnS+5skAACgPwzpEA0AAM0wZKdzAABAswjRAABQkhANAAAlCdEAAFCSEN3PVq5cmbPPPjvTp0/P2WefnZ///Ocv2Ka7uzuXXHJJXv/61+fUU0/NzTff3PhGGTD1jIEvfvGLOfPMM3PWWWflD//wD/Ptb3+78Y0yYOoZA8959NFH87KXvSwLFy5sXIMMqHrP/+23356zzjorM2bMyFlnnZX169c3tlEGTD1joLOzM3Pnzs1ZZ52V008/PRdffHG6uroa3yy7r6Bfve1tbytuueWWoiiK4pZbbine9ra3vWCbb3zjG8W73/3uoru7u+js7Cxe/epXF48//nijW2WA1DMG7r333mLz5s1FURTF8uXLixNOOKH49a9/3dA+GTj1jIGiKIqurq7irW99a/EXf/EXxV/91V81skUGUD3n/+GHHy5OP/30Yu3atUVRFMVTTz1VbNmypaF9MnDqGQOf+tSndnzdb9u2rXjTm95U3HbbbQ3tkz3jSnQ/6uzszLJlyzJjxowkyYwZM7Js2bJs2LBhp+1uv/32zJ49O9VqNePHj8/rX//6LF26tBkt08/qHQOvfvWrM2rUqCTJ0UcfnaIosnHjxka3ywCodwwkybXXXpvXvva1OeywwxrcJQOl3vP/d3/3d3n3u9+diRMnJkn23XfftLW1Nbxf+l+9Y6BSqWTTpk3p6enJtm3bsn379kyaNKkZLbObhOh+tGrVqkyaNCktLS1JkpaWlhx44IFZtWrVC7abPHnyjo87OjqyevXqhvbKwKh3DDzfLbfckhe96EU56KCDGtUmA6jeMfDjH/849913X975znc2oUsGSr3nf8WKFXn88cdz7rnn5o1vfGOuuuqqFG7bsFeodwx84AMfyMqVK3PSSSft+HfCCSc0o2V2kxANTfS9730vV1xxRT73uc81uxUaaPv27fnEJz6RSy65ZMcPWoaX7u7u/OQnP8l1112XL3/5y7n33nuzePHiZrdFAy1dujRHH3107rvvvtx777154IEH/FV6iBGi+1FHR0fWrFmT7u7uJM9+k1y7dm06OjpesN0TTzyx4+NVq1a5CrmXqHcMJMkPfvCDXHDBBfniF7/odvV7kXrGwLp16/LYY49l7ty5Ofnkk/P3f//3uemmm/KJT3yiWW3TT+r9HjB58uScdtppGTlyZMaMGZNTTjklDz/8cDNapp/VOwZuuOGGzJw5M9VqNfvuu29OPvnk3H///c1omd0kRPejCRMmZOrUqVmyZEmSZMmSJZk6dWrGjx+/03annXZabr755vT09GTDhg3553/+50yfPr0ZLdPP6h0DDz/8cP78z/88f/3Xf51jjz22Ga0yQOoZA5MnT87999+fu+++O3fffXfe8Y535M1vfnMuvfTSZrVNP6n3e8CMGTNy3333pSiKbN++Pf/2b/+Wl770pc1omX5W7xg45JBDcu+99yZJtm3blu9+97s56qijGt4vu69SmITVr1asWJF58+blqaeeytixY7Nw4cJMmTIl733ve/OhD30ov/M7v5Pu7u588pOfzHe+850kyXvf+96cffbZTe6c/lLPGPijP/qj/PKXv9zpTSSLFi3K0Ucf3cTO6S/1jIHn+5u/+Zts3rw5F154YZM6pj/Vc/57enqycOHC3HvvvalWqznppJNy4YUXplp1bWtvUM8YeOyxx7JgwYKsX78+3d3dOfHEE/Oxj30sra2tzW6fOgnRAABQkl95AQCgJCEaAABKEqIBAKAkIRoAAEoSogEAoCQhGgAAShKiAcjXv/71vOUtb2l2GwBDhhANQF2eu40xAG62AjDsrFq1Kp/+9KfzwAMPpCiKvPSlL82///u/p6urK/vss09aWlrywAMPZN68eWlra8sTTzyR73//+7nqqqvyqle9qtntAwwK7i0JMIx0d3fnT/7kT/LKV74yd999d1paWvIf//Efeeyxx3LzzTfnxhtv3Gn7JUuW5Nprr80111yT7du3N6lrgMHHdA6AYeThhx/O2rVr85GPfCTt7e1pa2vLtGnTet3+lFNOyQknnJBqtZq2trYGdgowuAnRAMPIqlWrMnny5LS21veHyI6OjgHuCGBoEqIBhpGOjo6sWrUqXV1dOz1eqVSa1BHA0CREAwwjv/u7v5uJEyfmc5/7XDZv3pytW7fmwQcfzIQJE7JmzZps27at2S0CDAlCNMAw0tLSkquvvjq/+MUv8rrXvS6///u/nzvuuCOvfOUrc+SRR+akk07KiSee2Ow2AQY9S9wBAEBJrkQDAEBJQjQAAJQkRAMAQElCNAAAlCREAwBASUI0AACUJEQDAEBJQjQAAJQkRAMAQEn/D/th2efY4wAvAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 842.4x595.44 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set(rc={'figure.figsize': (11.7, 8.27)})\n",
    "sns.histplot(data=df, x='ctr', hue='exp_group', kde = False, palette=['r', 'b'], alpha=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "970c42aa-0cd9-418c-9a2d-e3b50681fc2d",
   "metadata": {},
   "source": [
    "Анализ \"на глаз\" и гистограмма утверждает, что отличие между группами есть. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "628ce219-cd8f-40fa-915e-db57874c7f45",
   "metadata": {},
   "source": [
    "## Student's t-test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "bb6bff5e-0c60-4da4-acc9-ab98a551ce67",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_A = df[df['exp_group'] == 2].copy() \n",
    "group_B = df[df['exp_group'] == 1].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "951589df-fcca-4898-ba3e-d71e4605e6b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=-0.7094392041270485, pvalue=0.4780623130874935)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.ttest_ind(group_A['ctr'], group_B['ctr'], equal_var=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2ea85d64-2073-4733-912b-bbbf3141c5d2",
   "metadata": {},
   "source": [
    "T-test утверждает, что отличия между группами нет."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1cdf37b9-0b8b-4480-ba37-b735a822abc4",
   "metadata": {},
   "source": [
    "## Smothed CTR"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "23fa0051-87b2-49ca-bab2-c19d459d9749",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_smothed_ctr(user_likes, user_views, global_ctr, alpha):\n",
    "    smothed_ctr = (user_likes + alpha * global_ctr) / (user_views + alpha)\n",
    "    return smothed_ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8c2c4bc3-0abd-437e-bb58-39eabbcdeace",
   "metadata": {},
   "outputs": [],
   "source": [
    "global_ctr_A = df[df['exp_group'] == 2]['likes'].sum() / df[df['exp_group'] == 2]['views'].sum() \n",
    "global_ctr_B = df[df['exp_group'] == 1]['likes'].sum() / df[df['exp_group'] == 1]['views'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "d1c49ac0-59f9-48b9-b22f-00bb56f3c771",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_A['smothed_ctr'] = df.apply(lambda x: get_smothed_ctr(x['likes'], x['views'], global_ctr_A, 5), axis=1) \n",
    "group_B['smothed_ctr'] = df.apply(lambda x: get_smothed_ctr(x['likes'], x['views'], global_ctr_B, 5), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "80d81fe6-cddc-4e4a-880a-a70ebb5ed65c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ttest_indResult(statistic=-2.2841320431616983, pvalue=0.0223769815558559)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.ttest_ind(group_A['smothed_ctr'], group_B['smothed_ctr'], equal_var=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d921668d-7999-43cf-b5d3-f955642451d8",
   "metadata": {},
   "source": [
    "Smothed CTR утверждает, что отличие между группами есть. Так как теоритичекой гарантии нет, это может быть основанием не утверждать, что отличие между группами есть. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df523cdb-f0ee-49e4-a1ac-98788ca0cee3",
   "metadata": {},
   "source": [
    "По какой-то причине t-test утверждает, что отличия между группами нет, возможно, это связано с тем, что разные метрики имеют разную чувствительность и мощности t-test не достаточно, чтобы выявить отличие."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c62a8af3-ceff-452e-aa91-5dacec1e1773",
   "metadata": {},
   "source": [
    "## Poisson bootstrap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b449e2cd-74c1-432c-8435-e2fde07210a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def bootstrap(likes1, views1, likes2, views2, n_bootstrap=2000):\n",
    "\n",
    "    poisson_bootstraps1 = stats.poisson(1).rvs(\n",
    "        (n_bootstrap, len(likes1))).astype(np.int64)\n",
    "\n",
    "    poisson_bootstraps2 = stats.poisson(1).rvs(\n",
    "            (n_bootstrap, len(likes2))).astype(np.int64)\n",
    "    \n",
    "    globalCTR1 = (poisson_bootstraps1*likes1).sum(axis=1)/(poisson_bootstraps1*views1).sum(axis=1)\n",
    "    \n",
    "    globalCTR2 = (poisson_bootstraps2*likes2).sum(axis=1)/(poisson_bootstraps2*views2).sum(axis=1)\n",
    "\n",
    "    return globalCTR1, globalCTR2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "bb39e556-6b59-4f08-81e5-c4f79723870e",
   "metadata": {},
   "outputs": [],
   "source": [
    "likes1 = df[df['exp_group'] == 2]['likes'].to_numpy()\n",
    "views1 = df[df['exp_group'] == 2]['views'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d46920c8-8456-45ff-ab25-6b95f21c200d",
   "metadata": {},
   "outputs": [],
   "source": [
    "likes2 = df[df['exp_group'] == 1]['likes'].to_numpy()\n",
    "views2 = df[df['exp_group'] == 1]['views'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "02db2475-9ba0-41c5-9423-e3887e9bec2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "ctr1, ctr2 = bootstrap(likes1, views1, likes2, views2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "0c99bae5-7e1d-4919-8f80-c27d73d6537f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Count'>"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsoAAAHjCAYAAAA63TkjAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAArvUlEQVR4nO3dfZCV5Xk/8GsPe87+MIDIZkNXtLGaQlbJjEbSNOkgDdQJrSBtHEeGqqPRztQ4hOiwylQGIpFYXsZEEhL9g9ixYbTT2ICAEZOamKSTmcTEvFBMYo3mRVBhVdzNAvtyzu8Phx3WvVkOsOdlz/l8/tG9n/s8e51rn/Pw3Wfv85yGQqFQCAAAYJBMpQsAAIBqJCgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmNlS6g1F5//Y+Rz7tV9Ilobh4XHR1dlS6j5uhraejryNPT0tDX0tDXkVdvPc1kGuKMM96R3FbzQTmfLwjKJ0HPSkNfS0NfR56eloa+loa+jjw9fYulFwAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAQmOlCwBO3YTTx0ZT7vgv58M9ffHmgYNlqAgARj9BGWpAU64x2u996rjz1i2ZVYZqAKA2WHoBAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkFC2+yivWbMmdu7cGS+99FJs27Ytpk6dGn/4wx/i5ptvHpjT2dkZXV1d8cMf/jAiImbPnh25XC6ampoiImLp0qUxc+bMcpUMAEAdK1tQnjNnTlx77bXxj//4jwNjZ511VmzdunXg69WrV0d/f/+gx23YsCGmTp1arjIBgDrUky9EZ3fvoLHxp2Ujl2moUEVUg7IF5RkzZgy7vaenJ7Zt2xabNm0qU0UAAG/p7O6NTY/uGjR2w+XTo3lcrkIVUQ2q5iOsn3zyyZg8eXJccMEFg8aXLl0ahUIhLr744rj11ltjwoQJJ7Tf5uZxI1lm3WhpGV/pEmpSKfuazRb3cq7Fn20tPqdK09PS0NfSGIm+dvUcGHIebco11u3PrF6f99tVTVB+5JFH4oorrhg0tnnz5mhtbY2enp5YvXp1rFq1KtavX39C++3o6Ip8vjCSpda8lpbxsW9fZ6XLqDml7GtLy/jo7e0ram6t/WwdryNPT0tDX0tjpPp6uKdvyHn0cE9fXf7M6u1YzWQajnlhtSruevHKK6/Ej370o5g/f/6g8dbW1oiIyOVysWjRovjJT35SifIAAKhDVRGUv/71r8esWbPijDPOGBjr7u6Ozs63fpspFArx2GOPRVtbW6VKBACgzpRt6cVdd90VTzzxROzfvz+uv/76mDhxYuzYsSMi3grKd9xxx6D5HR0dsXjx4ujv7498Ph/nnXderFy5slzlAgBQ58oWlJcvXx7Lly9Pbtu5c+eQsbPPPju2bNlS4qoAACCtKpZeAABAtRGUAQAgQVAGAIAEQRkAABIEZQAASBCUAQAgQVAGAIAEQRkAABIEZQAASBCUAQAgQVAGAIAEQRkAABIEZQAASGisdAEAACOhJ1+Izu7eQWPjT8tGLtNQoYoY7QRlAKAmdHb3xqZHdw0au+Hy6dE8LlehihjtBGUAoK6krjznK1QL1U1QBgDqSurK87WXnV+haqhm3swHAAAJgjIAACQIygAAkGCNMpTZhNPHRlPu+C+9wz198eaBg2WoCABIEZShzJpyjdF+71PHnbduyawyVAMAHIulFwAAkCAoAwBAgqAMAAAJgjIAACQIygAAkOCuF1Clevvy0dIyvmL7dHs6AOqdoAxVKtuYKeo2chHF30quFPsEgFpl6QUAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACQ0lusbrVmzJnbu3BkvvfRSbNu2LaZOnRoREbNnz45cLhdNTU0REbF06dKYOXNmRET89Kc/jRUrVsThw4djypQpsW7dumhubi5XyQAA1LGyXVGeM2dObN68OaZMmTJk24YNG2Lr1q2xdevWgZCcz+ejvb09VqxYETt37owZM2bE+vXry1UuAAB1rmxBecaMGdHa2lr0/F27dkVTU1PMmDEjIiIWLlwYjz/+eKnKAwCAQcq29GI4S5cujUKhEBdffHHceuutMWHChNi7d2+ceeaZA3MmTZoU+Xw+3njjjZg4cWLligUAoC5UPChv3rw5Wltbo6enJ1avXh2rVq0a0SUWzc3jRmxf9aSlZXylS6hJR/qazRb30it2Xqn2OVqOg9FS52iip6Whr6VxpK9dPQeGnOOaco1D+p6al8k0FPXYelGvz/vtKh6UjyzHyOVysWjRorjpppsGxvfs2TMw77XXXotMJnPCV5M7Oroiny+MWL31oKVlfOzb11npMmrOkb62tIyP3t6+oh5T7LwTmXsi+xwNx4HjdeTpaWnoa2kc3dfDPX1DznGHe/qG9D01L58vFPXYelBvx2om03DMC6sVvT1cd3d3dHa+9YMoFArx2GOPRVtbW0RETJ8+PQ4dOhRPP/10REQ8/PDDMXfu3IrVCgBAfSnbFeW77rornnjiidi/f39cf/31MXHixLjvvvti8eLF0d/fH/l8Ps4777xYuXJlRERkMplYu3ZtrFy5ctDt4QAAoBzKFpSXL18ey5cvHzK+ZcuWYz7m/e9/f2zbtq2EVQEAQJpP5gMAgARBGQAAEgRlAABIEJQBACCh4vdRBqpTb1++qBvOH+7pizcPHCxDRQBQXoIykJRtzET7vU8dd966JbPKUA0AlJ+lFwAAkCAoAwBAgqAMAAAJgjIAACQIygAAkOCuFzACJpw+Nppyx385FXO7tdGm2NvIRbiVHACji6AMI6Ap13jcW6lls43R29tXc7dTK/Y2chFuJQfA6GLpBQAAJLiiDADUrGx2THR09Qway5/C/nryhejs7h0yPv60bOQyDaewZ6qRoAwA1KzuQ73x4I7dg8auvez8k95fZ3dvbHp015DxGy6fHs3jcie9X6qTpRcAAJAgKAMAQIKlFwAACSO9vpnRR1AGAEgY6fXNjD6WXgAAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQEJjpQuAajbh9LHRlPMyAYB6JAHAMJpyjdF+71PHnbduyawyVAMAlJOlFwAAkCAoAwBAgqAMAAAJgjIAACQIygAAkCAoAwBAgqAMAAAJgjIAACT4wBEAYNTpyReis7s3unoOxOGevoiIyFe4JmqPoAwAjDqd3b2x6dFdkc02Rm/vW0H52svOr3BV1BpLLwAAIKFsV5TXrFkTO3fujJdeeim2bdsWU6dOjddffz1uu+22+N3vfhe5XC7e/e53x6pVq2LSpEkRETFt2rSYOnVqZDJv5fm1a9fGtGnTylUyAAB1rGxXlOfMmRObN2+OKVOmDIw1NDTEjTfeGDt37oxt27bF2WefHevXrx/0uIcffji2bt0aW7duFZIBACibsgXlGTNmRGtr66CxiRMnxgc/+MGBry+88MLYs2dPuUoCAIBjqpo38+Xz+XjooYdi9uzZg8avueaa6O/vj0suuSQWL14cuVyuQhUCAFBPqiYof+Yzn4nTTjstrr766oGx73znO9Ha2hpdXV3R3t4eGzdujFtuueWE9tvcPG6kS60LLS3jK11C1chmi3uZFDPvyJyR3OeJzq30PktxbDleR56eloa+jpyungNDzqmZTMOQ81E5xiIimnKNNfXzraXnciqqIiivWbMmfvvb38Z999038Ma9iBhYqjFu3Li48sor44EHHjjhfXd0dEU+XxixWutBS8v42Levs9JlVIWWlvEDtx06nuPNO/oWRiO1z5OZW+l9jvSx5XgdeXpaGvo6sg739EVvb9+gc2s+XxhyPirH2JF6auXnW2/HaibTcMwLqxW/Pdw999wTu3btio0bNw5aVnHgwIE4dOhQRET09fXFzp07o62trVJlAgBQZ8p2Rfmuu+6KJ554Ivbv3x/XX399TJw4MT7/+c/H/fffH+ecc04sXLgwIiLOOuus2LhxY/zmN7+JFStWRENDQ/T19cVFF10US5YsKVe5AADUubIF5eXLl8fy5cuHjP/qV79Kzr/oooti27ZtpS4LAACSKr70AgAAqpGgDAAACYIyAAAkCMoAAJBQFfdRBgAYzbLZMdHR1TNobPxp2chlGipUESNBUAYAOEXdh3rjwR27B43dcPn0aB6XO8YjGA0svQAAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgITGShcAAFCLstkx0dHVM2hs/GnZyGUaKlQRJ0pQBgAoge5DvfHgjt2Dxm64fHo0j8tVqCJOlKUXAACQICgDAECCoAwAAAnWKAMAVasnX4jO7t4h4/kK1EL9EZQBgKrV2d0bmx7dNWT82svOr0A11BtBmboz4fSx0ZRz6AMAw5MWqDtNucZov/epouauWzKrxNUAANXKm/kAACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEhorXQBQP3r78tHSMv648w739MWbBw6WoSIAODZBGSibbGMm2u996rjz1i2ZVYZqAGB4ll4AAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAllCcpr1qyJ2bNnx7Rp0+LXv/71wPgLL7wQV111VXz0ox+Nq666Kl588cWitgEAQKkVHZS/8Y1vJMcff/zx4z52zpw5sXnz5pgyZcqg8ZUrV8aiRYti586dsWjRolixYkVR2wAAoNSKDsp33HFHcryYADtjxoxobW0dNNbR0RG7d++OefPmRUTEvHnzYvfu3fHaa68Nuw0AAMrhuJ/M9/vf/z4iIgqFwsD/H70tl8ud1Dfeu3dvTJ48OcaMGRMREWPGjIl3vetdsXfv3igUCsfcNmnSpBP6Ps3N406qvnpXzMcMj2bZbPEfSlns3GLmHZkzkvs80bmjZZ8ncgzW+vFaCXpaGvp64rp6DiTPG5lMw5Bz6tFjqXmlGjuRuU25xlFxHIyGGsvhuP9iXXrppdHQ0BCFQiEuvfTSQdve+c53xuLFi0tW3Ejo6OiKfL5Q6TJGlZaW8bFvX2elyyiZlpbx0dvbV/T8Yuceb1422zgwZ6T2eTJzR8s+iz0Ga/14rQQ9LQ19PTmHe/qS5418vhC9vX2Dzq1HxlLzSjl2InMP9/RV/XFQb8dqJtNwzAurxw3Kv/zlLyMi4uqrr46vfvWrI1ZUa2trvPLKK9Hf3x9jxoyJ/v7+ePXVV6O1tTUKhcIxtwEAQDkUvUZ5JENyRERzc3O0tbXF9u3bIyJi+/bt0dbWFpMmTRp2GwAAlEPRCwt///vfx+c///l49tlno7u7e9C273znO8M+9q677oonnngi9u/fH9dff31MnDgxduzYEZ/+9Kdj2bJl8aUvfSkmTJgQa9asGXjMcNsAAKDUig7KS5cujbPPPjtuv/32GDt27Al9k+XLl8fy5cuHjJ933nnxn//5n8nHDLcNAABKreig/Nxzz8VDDz0UmYwP8wMAoPYVnXo/8IEPxO7du0tZCwAAVI2iryhPmTIlbrzxxrj00kvjne9856BtS5YsGfHCAACgkooOygcPHoyPfOQj0dfXFy+//HIpawIAgIorOijffffdpawDAACqygndHu5Yzj777BEpBgAAqkXRQfnoj7I+oqGhISIinn322ZGvDAAAKqjooHzko6yP2LdvX3zxi1+MGTNmjHhRAABQaSd9U+SWlpa444474p577hnJegAAoCqc0qeH/OY3v4mDBw+OVC0AAFA1il56sWjRooE1yRFv3S7u//7v/+Lmm28uSWEAAFBJRQflK6+8ctDXY8eOjfe+971xzjnnjHRNcFImnD42mnJFH9IAAMMqOlX8wz/8QynrgFPWlGuM9nufOu68dUtmlaEaAGC0K3qNcm9vb2zYsCHmzJkT73vf+2LOnDmxYcOG6OnpKWV9AABQEUVfUV63bl38/Oc/jzvvvDPOPPPM2LNnT3zpS1+Krq6u+Jd/+ZdS1ggAAGVXdFB+/PHHY+vWrXHGGWdERMS5554b559/fixYsEBQBgCg5hS99OLoT+QrZhwAAEazooPy3Llz46abborvfe978fzzz8d3v/vduPnmm2Pu3LmlrA8AACqi6KUX7e3t8eUvfzlWrVoVr776akyePDkuu+yyuOmmm0pZHwBQJ3ryhejs7h00lq9QLRBRRFD+8Y9/HE8++WS0t7fHkiVLYsmSJQPb1q1bF7t3744LL7ywlDUCAHWgs7s3Nj26a9DYtZedX6FqoIilF/fff3984AMfSG774Ac/GPfdd9+IFwUAAJV23KD87LPPxsyZM5PbPvzhD8euXbuS2wAAYDQ7blDu6uqK3t7e5La+vr744x//OOJFAQBApR03KJ977rnx/e9/P7nt+9//fpx77rkjXhQAAFTacYPyddddFytXrownnngi8vm33nuaz+fjiSeeiE9/+tNx/fXXl7xIAAAot+Pe9WL+/Pmxf//+uP3226O3tzcmTpwYb7zxRmSz2fjkJz8Z8+bNK0edAABQVkXdR/n666+PK6+8Mp555pl44403YuLEiXHRRRfFuHHjSl0fAABURNEfODJu3Lhj3v0CAABqTdEfYQ0AAPVEUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIaKx0ATCcCaePjaacwxQAKD8JhKrWlGuM9nufKmruuiWzSlwNAFBPLL0AAIAEV5QBAMokmx0THV09g8bGn5aNXKahQhUxHEEZAKBMug/1xoM7dg8au+Hy6dE8LlehihiOpRcAAJBQ8SvKf/jDH+Lmm28e+LqzszO6urrihz/8YcyePTtyuVw0NTVFRMTSpUtj5syZlSoVAIA6UvGgfNZZZ8XWrVsHvl69enX09/cPfL1hw4aYOnVqJUoDAKCOVdXSi56enti2bVtcccUVlS4FAIA6V/Erykd78sknY/LkyXHBBRcMjC1dujQKhUJcfPHFceutt8aECRMqWCEAAPWiqoLyI488Muhq8ubNm6O1tTV6enpi9erVsWrVqli/fv0J7bO5edxIl1kXWlrGV7qEAdls8YdpsXMrtc8jc6q9zmrY54kcg9V0vNYKPS0NfR1eV8+BIeeITKYhed44evzIf1NzyzF2qo9vyjVW3bFRbfVUStUE5VdeeSV+9KMfxdq1awfGWltbIyIil8vFokWL4qabbjrh/XZ0dEU+XxixOutBS8v42Levs9JlRMRbtfT29hU9v9i5ldhnNts4MKea66yWfRZ7DFbT8Vor9LQ09PX4Dvf0DTlH5POF5HnjyPjR59bU3HKMnerjD/f0VdWxUW/HaibTcMwLq1WzRvnrX/96zJo1K84444yIiOju7o7Ozrd+SIVCIR577LFoa2urZIkAANSRqrmi/PWvfz3uuOOOga87Ojpi8eLF0d/fH/l8Ps4777xYuXJlBSsEAKCeVE1Q3rlz56Cvzz777NiyZUtligEAoO5VTVAGOKK3L1/0G0l6+/IlrgaAeiUoA1Un25iJ9nufKmruuiWzSlwNAPWqat7MBwAA1URQBgCABEEZAAASBGUAAEgQlAEAIMFdLwCAsuvJF6Kzu3fQmJs9Um0EZQCg7Dq7e2PTo7sGjV172fkVqgbSLL0AAIAEQRkAABIEZQAASLBGGQCggrLZMdHR1TNobPxp2chlGipUEUcIygAAFdR9qDce3LF70NgNl0+P5nG5ClXEEYIyMKr19uWjpWX8cecd7umLNw8cLENFANQKQRkY1bKNmWi/96njzlu3ZFYZqgGglngzHwAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmNlS4AAKhtPflCdHb3DhrLV6gWOBGCMgBQUp3dvbHp0V2Dxq697PwKVQPFs/QCAAASBGUAAEgQlAEAIEFQBgCABEEZAAASBGUAAEgQlAEAIEFQBgCABEEZAAASquKT+WbPnh25XC6ampoiImLp0qUxc+bM+OlPfxorVqyIw4cPx5QpU2LdunXR3Nxc4WoBAKgHVRGUIyI2bNgQU6dOHfg6n89He3t73H333TFjxoz40pe+FOvXr4+77767glUCAFAvqnbpxa5du6KpqSlmzJgRERELFy6Mxx9/vMJVAQBQL6rmivLSpUujUCjExRdfHLfeemvs3bs3zjzzzIHtkyZNinw+H2+88UZMnDix6P02N48rQbW1r6VlfKVLGJDNFn+YFju3Uvs8Mqfa66zVfVbTcV3t9Ko06rWvXT0HhrxOM5mGkx57+/iR/57KPkeynpEYa8o1VvR4qddj9e2qIihv3rw5Wltbo6enJ1avXh2rVq2KSy+9dET23dHRFfl8YUT2VS9aWsbHvn2dlS4jIt6qpbe3r+j5xc6txD6z2caBOdVcZy3vs1qO62pXTeeAWlLPfT3c0zfkdZrPF0567Ojxo8+tp7LPkapnpMYO9/RV7Hipt2M1k2k45oXVqlh60draGhERuVwuFi1aFD/5yU+itbU19uzZMzDntddei0wmc0JXkwEA4GRVPCh3d3dHZ+dbv7UUCoV47LHHoq2tLaZPnx6HDh2Kp59+OiIiHn744Zg7d24lSwUAoI5UfOlFR0dHLF68OPr7+yOfz8d5550XK1eujEwmE2vXro2VK1cOuj0cAACUQ8WD8tlnnx1btmxJbnv/+98f27ZtK29BAAAQVbD0AgAAqpGgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkCMoAAJAgKAMAQIKgDAAACYIyAAAkNFa6AACgdvTkC9HZ3TtoLF+hWuBUCcoAwIjp7O6NTY/uGjR27WXnV6gaODWCMgDAKJC6Wj/+tGzkMg0Vqqj2CcoAAKNA6mr9DZdPj+ZxuQpVVPu8mQ8AABIEZQAASBCUAQAgwRplRsyE08dGU664Q6q3rz+yjWNKXBEAjE7Z7Jjo6OoZNOY2e+UnKDNimnKN0X7vU0XNXbdkVlFz1y2ZdaplAcCo032oNx7csXvQmNvslZ+lFwAAkCAoAwBAgqAMAAAJgjIAACQIygAAkOCuFwBvU+ytDg/39MWbBw6WoSIAKkFQBnibYm916PaFALXN0gsAAEgQlAEAIEFQBgCABEEZAAASBGUAAEgQlAEAIEFQBgCABEEZAAASfOAIx1Xsp5RBNevty0dLy/hKlwHAKCL9cFw+pYxakG3MFHUcRziWAXiLpRcAAJBQ8SvKr7/+etx2223xu9/9LnK5XLz73e+OVatWxaRJk2LatGkxderUyGTeyvNr166NadOmVbhiAADqQcWDckNDQ9x4443xwQ9+MCIi1qxZE+vXr4/PfvazERHx8MMPxzve8Y5KlggAQB2q+NKLiRMnDoTkiIgLL7ww9uzZU8GKAACgCq4oHy2fz8dDDz0Us2fPHhi75ppror+/Py655JJYvHhx5HK5ClYIAEC9qKqg/JnPfCZOO+20uPrqqyMi4jvf+U60trZGV1dXtLe3x8aNG+OWW245oX02N48rRak17+230cpmiztUip1Xr/s8Mqfa67TP4ufW6i3navV5VVqt9fW1Nw/Fgc7Dg8YaxmSGvH4ymYYRHXv7+JH/jvT3Odl6yjnWlGssyXFVa8fqyaqaoLxmzZr47W9/G/fdd9/Am/daW1sjImLcuHFx5ZVXxgMPPHDC++3o6Ip8vjCitda6lpbxsW9f56Cve3v7inpssfNOZG6t7DObbRyYU8112ueJzT36tVIr3n4OYGTUYl87unpi06O7Bo1de9n5Q14/+XxhRMeOHj/63DrS3+dk6in32OGevhE/rmrxWB1OJtNwzAurFV+jHBFxzz33xK5du2Ljxo0DSysOHDgQhw4dioiIvr6+2LlzZ7S1tVWyTAAA6kjFryg/99xzcf/998c555wTCxcujIiIs846K2688cZYsWJFNDQ0RF9fX1x00UWxZMmSClcLAEC9qHhQ/vM///P41a9+ldy2bdu2MlcDAABvqXhQBhitevvyRb/h5XBPX7x54GCJKwLqTTY7Jjq6eoaMjz8tG7lMQwUqqi2CMsBJyjZmov3ep4qau27JrBJXA9Sj7kO98eCO3UPGb7h8ejSPc0vdU1UVb+YDAIBqIygDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQICgDAECCoAwAAAmCMgAAJAjKAACQ0FjpAhhZE04fG0254/9YD/f0xZsHDpahIiAiorcvHy0t4487z2sToHoIyjWmKdcY7fc+ddx565bMKkM1wBHZxozXJsAoY+kFAAAkuKJcp473Z+Bi/kQMQO3pyReis7t3yPj407KRyzRUoCKoHEG5Tg33Z+BstjF6e/sGvvanYID60dndG5se3TVk/IbLp0fzuFwFKoLKsfQCAAASBGUAAEgQlAEAIEFQBgCABEEZAAAS3PUCADiubHZMdHT1DBrLV6gWji/183rH2Gz88eDgW/+57d/wBGUA4Li6D/XGgzt2Dxq79rLzK1QNx3Osn9fbx9z2b3iWXgAAQIKgDAAACYIyAAAkWKM8Ckw4fWw05fyoAIDSe+3NQ0PeCFivb/qTvkaBplxjtN/7VFFz1y2ZVeJqAIBadqDzcGx6dNegsXp905+lFwAAkCAoAwBAgqUXAAB1KvXBJA1jXEc9QlAGAKhTqQ8muW7+BRWqpvr4lQEAABJcUQYYpYq9deThnr5488DBMlTEaNOTL0Rnd++gsXyFamH0SR0/tXYbOUEZYJQq9taRbhvJsXR29w65Ddi1l51foWoYbVLHT63dRk5QriAfJAK8XW9fPlpaxle6DGqQq8dw4qS0CnI1CHi7bGPGBwxREq4ew4nzZj4AAEhwRXmEWU4BVJsTWc5xuKevxNUw0lJLKt4xNht/PGiZBZyqqk90L7zwQixbtizeeOONmDhxYqxZsybOOeecSpd1TMUup4jwZ1OgPCznqG3HWlLx9nvjWmYBJ67qg/LKlStj0aJFsWDBgti6dWusWLEiHnzwwUqXBVCTevvykW3MHPcKtFvOQX1JfYJf6q8UqXkR6b9ypMaq7fZyVR2UOzo6Yvfu3fHAAw9ERMS8efPiM5/5TLz22msxadKkovaRqUCzzxjfNOJzy7nPxmxj9PWOGdF9nsrcWtnn0X2t5jrts773mW3MxNp/fzr6eodfgvEvH//LipxfR7tS9CzbmBny8x3psXJ9n5Op5+hzazXUUw1jpdhnb19//Ne3nxs09rHZf17UvCNzU49/+9hVl06L/3dadsjjS2m412VDoVAolLGWE7Jr1664/fbbY8eOHQNjf/d3fxfr1q2LCy7w8YoAAJSOu14AAEBCVQfl1tbWeOWVV6K/vz8iIvr7++PVV1+N1tbWClcGAECtq+qg3NzcHG1tbbF9+/aIiNi+fXu0tbUVvT4ZAABOVlWvUY6IeP7552PZsmXx5ptvxoQJE2LNmjVx7rnnVrosAABqXNUHZQAAqISqXnoBAACVIigDAECCoAwAAAmCMgAAJAjKNeaFF16Iq666Kj760Y/GVVddFS+++OKQOd///vfjYx/7WEyfPj3WrFkzaNu+ffvipptuivnz58ff/u3fxtatWwe2feELX4gPfehDsWDBgliwYEHceeedA9sOHjwYn/rUp+LSSy+NuXPnxre//e2SPcdKKGVfb7vttoGeLliwIN773vfGf//3f0fE8D0f7Yrp6caNG+Oyyy6L+fPnx8c+9rH43ve+N7BtuGPuZLfVglL29c4774y5c+fG5ZdfHgsXLoxf/OIXA9uuueaamDNnzsCx+sgjj5T0eZZbKfu6bNmyuOSSSwZ69+Uvf3lg2/79++PjH/94fPSjH43LL788fvazn5X0eZZTKXt63XXXDfRz3rx5MW3atPjlL38ZEcP3uxacal+3bt0a8+fPj/PPPz+++tWvDnpcXZ5bC9SUa665prBly5ZCoVAobNmypXDNNdcMmfPiiy8Wdu/eXbjnnnsK//qv/zpo26233lr44he/WCgUCoWOjo7CrFmzCnv27CkUCoXChg0bhsw/4gtf+ELhjjvuKBQKhcILL7xQ+PCHP1zo6uoasedVaaXs69GeffbZwl/8xV8UDh8+XCgUhu/5aFdMT7/73e8Wuru7C4XCW725+OKLCwcPHiwUCsMfcye7rRaUsq9PPvlkoaenZ+D/58yZM7DPq6++uvDkk0+W7olVWCn7evvttxf+/d//Pfl9ly1bVti4cWOhUCgUfvSjHxUuvfTSQj6fH9knVyGl7OnRvvnNbxYuu+yyga+H63ctONW+/upXvyo899xzhfb29iF9qsdzqyvKNaSjoyN2794d8+bNi4iIefPmxe7du+O1114bNO/d7353tLW1RWNj45B9/PKXv4yZM2dGRMSkSZPive99b3zjG9847vf+xje+EVdddVVERJxzzjkxffr0+O53v3uqT6kqlLOvX/va12L+/PmRy+VK8EyqR7E9nTlzZowdOzYiIqZNmxaFQiHeeOONiBj+mDvZbaNdqfv6kY98JLLZbEREXHjhhfHyyy9HPp8vx1OrqFL3dTiPP/54LFy4MCIiZsyYEblcbtCV/NGqnD392te+FldccUUJn031GIm+Tp06Nd7znvdEJjM0ItbjuVVQriF79+6NyZMnx5gxYyIiYsyYMfGud70r9u7dW/Q+LrjggnjssceiUCjE73//+3jmmWdiz549A9t37NgR8+fPj49//OPxzDPPDIzv2bMnpkyZMvB1a2trvPzyyyPwrCqvHH2NiOjp6Ylt27YNOaEfq+ej2cn0dMuWLfGnf/qn8Sd/8icRMfwxd7LbRrtS9/Vomzdvjr/+678e9I/p2rVrY/78+bF06dJ45ZVXRuppVVw5+vrAAw/E/Pnz4xOf+EQ8//zzERHx+uuvR6FQGPRptLVyvJbrWN23b1/84Ac/iAULFgwaT/W7FoxEX4dTj+fWoZe+qGvLli2Lz372s7FgwYI488wz40Mf+tDAC27hwoXxz//8z5HNZuN//ud/4hOf+EQ89thjccYZZ1S46uo3XF+P+Na3vhVnnnlmtLW1DYzp+Vt++MMfxr333htf+cpXKl1KTTnZvu7YsSO2bdsWmzdvHhhbu3ZttLa2Rn9/f9x///3xqU99Kh566KGRLnlUONG+3nLLLdHS0hKZTCa2bNkSN954Y3zrW98qcZWjy8keq1u2bImZM2cO+mXjWP1++zm5Hji3Hp8ryjWktbU1Xnnllejv74+IiP7+/nj11VejtbW16H1MmjQp1q9fH48++mjcd9998cc//jHe8573RERES0vLwJ9d/+qv/ipaW1vjueeei4iIM888M1566aWB/ezdu7eo305Hg1L39YhHHnlkyNXk4Xo+mp1IT5955plob2+PjRs3Dvr4+uGOuZPdNtqVuq8REd/85jfjc5/7XGzatCne+c53DvreEW9dwbr22mvjZz/7Wc0syyh1XydPnjxwZf7v//7vo7u7O15++eWBX4iP/rN5rRyv5ThWIyL+67/+a8h59Vj9rgUj0dfh1OO5VVCuIc3NzdHW1hbbt2+PiIjt27dHW1vboN+kj+f111+Pvr6+iIj4wQ9+EL/+9a8H1jod/afUZ599Nl566aX4sz/7s4iImDt3bvzHf/xHRES8+OKL8Ytf/GJgTe5oV+q+RkS8/PLL8eMf/zjmz58/6HHD9Xw0K7anP//5z+OWW26JDRs2xAUXXDBo23DH3MluG+1K3ddvf/vbcffdd8emTZvirLPOGnhMX19f7N+/f+DrHTt2xNSpU5NrHEejUvf16Nf59773vchkMjF58uSBxz388MMREfH000/HoUOHYvr06aV5omVU6p5GRPzkJz+Jzs7OuOSSSwY9brh+j3Yj0dfh1OO5taFQKBQqXQQj5/nnn49ly5bFm2++GRMmTIg1a9bEueeeG//0T/8Un/zkJ+N973tfPP3003HrrbdGV1dXFAqFGD9+fKxevTpmzpwZTz31VKxevToymUycccYZsWLFioGlALfffnv87//+b2Qymchms/HJT34yZs2aFRER3d3dsWzZsnj22Wcjk8lEe3t7/M3f/E0lWzGiStnXiIgvf/nL8etf/zo+97nPDfq+w/V8tCump1dccUW89NJLg/4RW7t2bUybNm3YY+5kt9WCUvb1L//yLyObzQ76R/ff/u3foqmpKa6++uro7e2NiIh3vetdcccddxR9lWo0KGVfr7vuuujo6IiGhoYYN25c3HbbbXHhhRdGxFtrbNvb22PPnj3R1NQUd955Z7z//e+vRAtGXCl7GhGxfPnymDhxYixdunTQ9x2u37XgVPu6ffv2WLt2bbz55puRzWZj7Nix8ZWvfCXe85731OW5VVAGAICE2vi7GAAAjDBBGQAAEgRlAABIEJQBACBBUAYAgARBGQAAEgRlAABIEJQBACDh/wOf4w7gJzmvHwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 842.4x595.44 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(ctr1)\n",
    "sns.histplot(ctr2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "be3de2fc-c277-4619-a840-f7e2c6e8dfe8",
   "metadata": {},
   "source": [
    "Poisson bootstrap утверждает, что экспериментальный алгоритм статистически значимо хуже контрольного алгоритма. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e562caec-f04b-4e0b-aa6e-1fdcf267f826",
   "metadata": {},
   "source": [
    "## Bucket evaluations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "c0b36f65-5a20-49dc-9c4b-8a566d22fe8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = '''\n",
    "select exp_group, xxHash64(user_id)%50 as bucket,\n",
    "sum(action = 'like') as likes,\n",
    "sum(action = 'view') as views,\n",
    "likes / views as bucket_ctr\n",
    "from {db}.feed_actions \n",
    "where toDate(time) between '2022-11-02' and '2022-11-08' and exp_group in (2, 1)\n",
    "group by exp_group, bucket\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c183ad5c-8626-4e6e-94a2-9dff10b1ac89",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = select(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "9d534589-07a0-4c85-8ead-88e10e3d3713",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>exp_group</th>\n",
       "      <th>bucket</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>bucket_ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2624</td>\n",
       "      <td>13355</td>\n",
       "      <td>0.196481</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>2634</td>\n",
       "      <td>13559</td>\n",
       "      <td>0.194262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2681</td>\n",
       "      <td>12653</td>\n",
       "      <td>0.211887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2686</td>\n",
       "      <td>12474</td>\n",
       "      <td>0.215328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3046</td>\n",
       "      <td>14767</td>\n",
       "      <td>0.206271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>2</td>\n",
       "      <td>47</td>\n",
       "      <td>2510</td>\n",
       "      <td>12814</td>\n",
       "      <td>0.195880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>1</td>\n",
       "      <td>48</td>\n",
       "      <td>2839</td>\n",
       "      <td>13342</td>\n",
       "      <td>0.212787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>2554</td>\n",
       "      <td>13131</td>\n",
       "      <td>0.194502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>1</td>\n",
       "      <td>49</td>\n",
       "      <td>2949</td>\n",
       "      <td>14515</td>\n",
       "      <td>0.203169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>2</td>\n",
       "      <td>49</td>\n",
       "      <td>2726</td>\n",
       "      <td>13870</td>\n",
       "      <td>0.196539</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    exp_group  bucket  likes  views  bucket_ctr\n",
       "0           1       0   2624  13355    0.196481\n",
       "1           2       0   2634  13559    0.194262\n",
       "2           1       1   2681  12653    0.211887\n",
       "3           2       1   2686  12474    0.215328\n",
       "4           1       2   3046  14767    0.206271\n",
       "..        ...     ...    ...    ...         ...\n",
       "95          2      47   2510  12814    0.195880\n",
       "96          1      48   2839  13342    0.212787\n",
       "97          2      48   2554  13131    0.194502\n",
       "98          1      49   2949  14515    0.203169\n",
       "99          2      49   2726  13870    0.196539\n",
       "\n",
       "[100 rows x 5 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "d0e65c17-ef1a-4cf5-baba-9c940b98b9fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MannwhitneyuResult(statistic=505.0, pvalue=2.859963936492646e-07)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats.mannwhitneyu(df[df['exp_group'] == 2]['bucket_ctr'], df[df['exp_group'] == 1]['bucket_ctr'], alternative='two-sided')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "212c39c5-ee7a-475b-bdb9-521fbc2534b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:xlabel='bucket_ctr', ylabel='Count'>"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAs4AAAH0CAYAAAAt5at6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzoklEQVR4nO3deXhU5cH+8XsmC5IETAJDGBZBVCACVRZLa0vVQJoIhFAVQVAqChTEBRcKlhQkgBqgSEEUQcRWXNBXIQ07VCugvnWjyJIqVRbNRgggJCwhmfn9wc+pebM9wMycSeb7uS6va2bOmXPuOc9McnN8csbmdrvdAgAAAFAju9UBAAAAgLqA4gwAAAAYoDgDAAAABijOAAAAgAGKMwAAAGCA4gwAAAAYoDgDAAAABkKtDuBPR4+WyOXistW+0qRJlIqKiq2OgYvEONYfjGX9wDjWD4xj3WC32xQTE1nt8qAqzi6Xm+LsYxzf+oFxrD8Yy/qBcawfGMe6j6kaAAAAgAGKMwAAAGAgqKZqAAAABJPy8jIdPVqosrJSq6MElNDQcMXEOBQScn5VmOIMAABQTx09WqhLLolQZGRz2Ww2q+MEBLfbrZKS4zp6tFBNmzrP67lM1QAAAKinyspKFRnZmNL8IzabTZGRjS/oLDzFGQAAoB6jNFd2oceE4gwAAAAYoDgDAAAABijOAAAAgAGKMwAAAAJCWVmZ1RFqxOXoAAAAgszhw4V65plZ2rFjuxo2jNDttw9VUtLN+u1v79Cjj07SL3/5K508eVIjRgzV3XeP1M0399fMmU8oPDxcOTnfaffuXerQoaPS0qapefOaL+n28cf/q2eemaUjR4qUmHiz9u//RklJfZWSMlBr12YpK2ul4uM7af36tRo48FYNHTpczzwzS//854dq0OASpaQM1PDh98hut2vp0heUk/OdpkyZLknKy8vVoEED9I9//K9CQ0N1//2j1bnzT/TZZx/rwIED6tatu/7wh6lq3PhSrxw3zjgDAAAEEZfLpd///mFdeWV7rVy5TvPmPac333xd2dl79PjjUzRr1kwdPXpECxbM1VVXtdfNN/f3PHfjxvW6++6RWrNms668sr2mTUurcV/Hjh1TWtpEjRlzv9as+bsuu6yNdu7cUWGdPXt2q0WLlvrb3zbot7+9R888M0slJcV6881MPfvsYq1fv1Zr1/7N+PWtX79Gjz8+RZmZ6xUSEqJ58+ac3wGqAcUZAAAgiGRn79GxY0c1YsQohYWFqWXLVhowYKD+/veN+ulPf6abbuqthx4aq48++kATJvyhwnOvv/4XuvbabgoPD9fo0fdp9+6dKijIr3ZfH320TZdf3k433JCg0NBQDRo0RE2aNK2wTpMmTXXbbUMUGhqq0NAw/f3vG/W7392viIhIOZ0tNGTIMK1fv9b49SUl9VW7dleqYcOGGjlyrN59d5PKy8vP7yBVg6kaAAAAQSQ/P09FRYeVnHyj57HycpeuueZaSdKAAbfo7bff1F13jdCll0ZXeG6zZs09tyMiItS4cWMdPnxYcXHNVZXDhw+rWbM4z32bzSaHo1mFdeLi/rv8+++PqaysrML0j+bNnTp8uND49f14f82bO1VWVqbvvz+m2NgmxtuoDsUZAAAgiMTFxcnpbKE33lhZaVl5eblmzZqp5OR+Wrnyf9Sv3wC1atXas/zQof+eXT558qSOHz+upk2bVtrOD5o2baoPPjjkue92u1VYeOj/rPXfLyO59NJohYaGKj8/T5df3k6SVFCQr6ZNHZKkhg0b6vTp0571i4qKKu3z0KECz+2CgnyFhoZW+gfAhWKqBgAAQBCJj++kiIgILV/+ss6cOa3y8nJ9881/lJ29W6+8skw2m02PPz5FQ4fepRkzplaY5vDRRx9qx45/6ezZs3rxxefVqVPnas82S9LPf/5Lff31f7Rlyz9UVlamd955U0VFh6tdPyQkRAkJiVq8+DmdPFmi/Pw8rVjxqpKS+kqSrrqqvXbs+Fz5+fkqLi7W8uXLKm1j48Z12rfvG50+fVovvrhIN97YWyEhIRdxxP6L4gwAABBEQkJCNGvWPO3d+5UGDUpVv3599PTTM/TZZ5/ojTdeVVraNIWEhGjYsN/KZpOWL3/Z89zExCQtW7ZYffsm6Msv/60//nF6jfuKjo7W9OlP67nn5qtfv97at2+fOna8WmFhYdU+Z/z4CWrYsKFuvz1V9903UomJyerXb4Ak6brrfqaEhF/r7ruH6N5779T11/eq9PykpL568sknNGBAkkpLSzV+/IQLO1BVsLndbrfXthbgioqK5XIFzcv1O4ejkQoLT1gdAxeJcaw/GMv6gXGsH6wax/z8A2revI1XtjVz5hNyOJpp9Oj7LngbLpdLv/lNX02dOkPduvXwSq4fu//+0Z5L3dWmqmNjt9vUpElUtc9hjjMAwDIxl16i0PDqzzwFi7LSszr6/enaVwTqoH/+8yNdfXVnNWjQQK+99ldJbnXq1NnqWBeE4gwAsExoeJiyX/yr1TEsFz9yuCSKM+qmHTu267HHHqxy2aZNW7Vr1xeaNi1NZ8+eVdu2l+vJJ/+kBg0u8XNK72CqBryG/51YPzCO9UddGEuHoxHFWeeKc3VjVRfGEbWrD1M16psLmarBHwcCAAAABijOAAAAgAGKMwAAAGCA4gwAAAAY4KoaAAAAqJavLhtZFy/DSHEGAABAtXx12UjTyzA+++w8vf/+u8rLy9Vf//qG2rW70utZTDFVAwAAAAGrV68b9eyzi9W8udPqKJxxBgAAQOC65pprrY7gwRlnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAA/xxIAAAAKpVVnr2/186zvvbNTFv3my9//57OnKkSOPHj1Pjxpdq+fI3vZ7HBMUZAAAA1Tr3JSXWfVHJ+PETNH78BMv2/2N+mapx9OhRjRo1SklJSUpJSdH999+vI0eOSJL+9a9/acCAAUpKStI999yjoqKiKrdx6tQpjR8/XomJiUpOTtZ7773nj+gAAACAJD8VZ5vNppEjR2rDhg3KyspS69atNWfOHLlcLk2YMEFTpkzRhg0b1KNHD82ZM6fKbSxdulRRUVHatGmTFi1apLS0NJWUlPgjPgAAAOCf4hwdHa2ePXt67l977bXKzc3Vrl271KBBA/Xo0UOSNGTIEK1fv77Kbaxbt06DBw+WJLVt21adO3fWli1bfB8eAAAAkAVX1XC5XHr99deVkJCgvLw8tWjRwrMsNjZWLpdLx44dq/S83NxctWzZ0nPf6XQqPz/fH5EBAAAA//9x4PTp0xUREaE777xTmzZt8uu+mzSJ8uv+gpHD0cjqCPACxrH+qAtjGRnZwOoIAaGmsaoL44jaWTGOhw7ZFRrK1YerYrfbz3tM/FqcMzIydODAAS1atEh2u11Op1O5ubme5UeOHJHdbld0dHSl57Zo0UI5OTmKjY2VJOXl5VWY/mGiqKhYLpf7ol4DqudwNFJh4QmrY+AiMY71R10YS4ejkUpKzlgdIyBUN1Z1YRxRO6vG0eVyqazM5ff91gUul6vSmNjtthpPtPqtOM+dO1e7du3S4sWLFR4eLknq3LmzTp8+rU8//VQ9evTQG2+8oeTk5Cqfn5ycrBUrVqhLly7av3+/du7cqT/96U/+ig8AABCULo2OUHhYiNe3W3q2XN8fO1njOt9/f0zTp09RTs53CgsLU6tWl2nChD8oJibG63lM+KU47927Vy+88ILatm2rIUOGSJJatWqlhQsXatasWZo6darOnDmjli1bavbs2Z7npaamavHixYqLi9O9996rSZMmKTExUXa7Xenp6YqKYuoFAACAL4WHhWjJOzu8vt1Rt1xT6zo2m01Dhw5Xt27nLiSxcOGftWjRAj3++BSv5zHhl+J81VVX6csvv6xyWbdu3ZSVlVXlsszMTM/tiIgIzZ8/3yf5AAAAEHgaN77UU5olqVOnzlq58m3L8jBbHAAAAAHP5XJp5cq39ctf/sqyDBRnAAAABLxnnpmtiIiGuvXW2y3L4PfL0QEAAADn49ln5+m77w4qI+MZ2e3WnfelOAMAACBgvfDCQn35ZbZmz/6z58psVqE4AwAAICB9883XeuWVZWrd+jKNGXOPJMnpbKGnnppjSR6KMwAAAKpVerbc6NJxF7Ld2rRrd4W2bfvU6/u+UBRnAAAAVKu2LykJJlxVAwAAADBAcQYAAAAMUJwBAADqMbfbbXWEgHOhx4TiDAAAUE+FhoarpOQ45flH3G63SkqOKzT0/C9txx8HAgAA1FMxMQ4dPVqo4uJjVkcJKKGh4YqJcZz/83yQBQAAAAEgJCRUTZs6rY5RbzBVAwAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMhFodAACAYOcuL5fD0aja5TUtq0/KSs/q6PenrY4BVIviDACAxWwhIcp+8a9VLouMbKCSkjN+TmSN+JHDJVGcEbiYqgEAAAAYoDgDAAAABijOAAAAgAGKMwAAAGCA4gwAAAAY8NtVNTIyMrRhwwbl5OQoKytL7du313fffadx48Z51jlx4oSKi4v18ccfV3r+ggUL9Nprr6lZs2aSpG7dumnq1Kn+ig8AAIAg57fi3Lt3bw0fPlzDhg3zPNaqVStlZmZ67s+cOVPl5eXVbmPgwIGaOHGiT3MCAAAAVfFbce7Ro0eNy0tLS5WVlaWlS5f6KREAAABgLmDmOL/77ruKi4tTp06dql1nzZo1SklJ0T333KPt27f7MR0AAACCXcB8c+Dbb7+tW2+9tdrlQ4YM0ZgxYxQWFqYPPvhA9913n9auXauYmBjjfTRpEuWNqKhBsHwtbH3HONYfdWEsIyMbWB0hINR0HILpGNWF9+yFqs+vLVgERHEuKCjQJ598olmzZlW7jsPh8Nz+xS9+IafTqb179+qnP/2p8X6KiorlcrkvKiuq53A0UmHhCatj4CIxjvVHXRhLh6NR0HyddG2qOw7B9JXbkgL+PXuh6sLnEZLdbqvxRGtATNVYuXKlbrjhhhrPHhcUFHhuZ2dnKycnR5dffrk/4gEAAAD+O+M8Y8YMbdy4UYcPH9aIESMUHR2tNWvWSDpXnCdPnlzpOaNGjdKDDz6oLl26aO7cudq9e7fsdrvCwsI0a9asCmehAQAAAF/yW3FOS0tTWlpalcs2bNhQ5eNLlizx3M7IyPBJLgAAAMBEQEzVAAAAAAIdxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMBAqNUBAASfS6MjFB4WYnUMy5WeLdf3x05aHQMAYIjiDMDvwsNCtOSdHVbHsNyoW66xOgIA4DwwVQMAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAN+K84ZGRlKSEhQhw4d9NVXX3keT0hIUHJyslJTU5WamqqtW7dW+fxTp05p/PjxSkxMVHJyst577z1/RQcAAAAU6q8d9e7dW8OHD9ewYcMqLZs/f77at29f4/OXLl2qqKgobdq0Sfv379ewYcO0ceNGRUZG+ioyAAAA4OG3M849evSQ0+m84OevW7dOgwcPliS1bdtWnTt31pYtW7wVDwAAAKiR38441+Sxxx6T2+1W9+7d9cgjj6hx48aV1snNzVXLli09951Op/Lz889rP02aRF10VtTM4WhkdQR4gT/GMTKygc/3URf4+ljXhc8k74VzajoOwXSM6sJ79kLV59cWLCwvzq+++qqcTqdKS0s1c+ZMpaena86cOT7ZV1FRsVwut0+2jXM/EAoLT1gdAxfJH+PocDRSSckZn+6jrvDlsa4Ln0neC/9V3XGIjGwQVMco0N+zF6oufB4h2e22Gk+0Wn5VjR+mb4SHh2vo0KH6/PPPq1yvRYsWysnJ8dzPy8tT8+bN/ZIRAAAAsLQ4nzx5UidOnPvXl9vt1tq1axUfH1/lusnJyVqxYoUkaf/+/dq5c6d69erlt6wAAAAIbn4rzjNmzNCvfvUr5efna8SIEerXr5+Kiop01113KSUlRf3799e+ffs0depUz3NSU1NVUFAgSbr33nt1/PhxJSYm6ne/+53S09MVFcWcZQAAAPiH3+Y4p6WlKS0trdLjq1atqvY5mZmZntsRERGaP3++L6IBAAAAtbJ8jjMAAABQF1CcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAOh/tpRRkaGNmzYoJycHGVlZal9+/Y6evSofv/73+vgwYMKDw9XmzZtlJ6ertjY2ErPnzRpkj788EPFxMRIkpKTkzV27Fh/xQcAAECQ89sZ5969e+vVV19Vy5YtPY/ZbDaNHDlSGzZsUFZWllq3bq05c+ZUu43Ro0crMzNTmZmZlGYAAAD4ld+Kc48ePeR0Ois8Fh0drZ49e3ruX3vttcrNzfVXJAAAAMCY36Zq1Mblcun1119XQkJCtessW7ZMK1asUOvWrfXoo4/qiiuuOK99NGkSdbExUQuHo5HVEeAF/hjHyMgGPt9HXeDrY10XPpO8F86p6TgE0zGqC+/ZC1WfX1uwCJjiPH36dEVEROjOO++scvnDDz8sh8Mhu92uVatWaeTIkdq8ebNCQkKM91FUVCyXy+2tyPg/HI5GKiw8YXUMXCR/jKPD0UglJWd8uo+6wpfHui58Jnkv/Fd1xyEyskFQHaNAf89eqLrweYRkt9tqPNEaEFfVyMjI0IEDBzRv3jzZ7VVHiouL8ywbOHCgTp48qfz8fH/GBAAAQBCzvDjPnTtXu3bt0sKFCxUeHl7tegUFBZ7bW7duld1uV1xcnD8iAgAAAP6bqjFjxgxt3LhRhw8f1ogRIxQdHa158+bphRdeUNu2bTVkyBBJUqtWrbRw4UJJUmpqqhYvXqy4uDhNnDhRRUVFstlsioqK0vPPP6/Q0ICZaQIAAIB6zm/NMy0tTWlpaZUe//LLL6t9TmZmpuf2yy+/7ItYAAAAgBHLp2oAAAAAdQHFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwADFGQAAADBAcQYAAAAMUJwBAAAAAxRnAAAAwIBxcV63bl2Vj69fv95rYQAAAIBAZVycJ0+eXOXjU6ZM8VoYAAAAIFCF1rbCt99+K0lyu92e2z9eFh4e7ptkAAAAQACptTgnJibKZrPJ7XYrMTGxwrKmTZvqgQce8Fk4AAAAIFDUWpz//e9/S5LuvPNOLV++3OeBAAAAgEBkPMeZ0gwAAIBgVusZ5x98++23mjdvnrKzs3Xy5MkKy/7xj394OxcAAAAQUIyL82OPPabWrVtr4sSJatiwoS8zAQAAAAHHuDjv3btXr7/+uux2vjMFAAAAwce4BV933XXas2ePL7MAAAAAAcv4jHPLli01cuRIJSYmqmnTphWWPfTQQ14PBgAAAAQS4+J86tQp3XTTTSorK1N+fr4vMwEAAAABx7g4P/XUU77MAQAAAAS087ocXXVat27tlTAAAABAoDIuzj/+6u0f2Gw2SVJ2drb3kwEAAAABxLg4//DV2z8oLCzUs88+qx49eng9FAAAABBoLviizA6HQ5MnT9bcuXO9mQcAAAAISBf1bSbffPONTp065a0sAAAAQMAynqoxdOhQz5xm6dzl6f7zn/9o3LhxPgkGAAAABBLj4jxo0KAK9xs2bKiOHTuqbdu23s4EAAAABBzj4vyb3/zGlzkAAACAgGY8x/ns2bOaP3++evfurS5duqh3796aP3++SktLfZkPAAAACAjGZ5xnz56tL774QtOmTVOLFi2Um5ur5557TsXFxfrDH/7gy4wAAACA5YyL8/r165WZmamYmBhJUrt27XT11VcrNTWV4gwAAIB6z3iqxo+/MdDkcQAAAKA+MS7OycnJGjt2rLZu3aqvv/5aW7Zs0bhx45ScnOzLfAAAAEBAMJ6qMWHCBD3//PNKT0/XoUOHFBcXp379+mns2LG+zAcAAAAEhFrPOH/22WeaPXu2wsPD9dBDD2nTpk3asWOHNm7cqNLSUu3Zs8cfOQEAAABL1VqcX3jhBV133XVVLuvZs6cWLVrk9VAAAABAoKm1OGdnZ6tXr15VLrv++uu1a9cur4cCAAAAAk2txbm4uFhnz56tcllZWZlKSkq8HgoAAAAINLUW53bt2mnbtm1VLtu2bZvatWvn9VAAAABAoKm1ON99992aOnWqNm7cKJfLJUlyuVzauHGjnnjiCY0YMcLnIQEAAACr1Xo5upSUFB0+fFgTJ07U2bNnFR0drWPHjiksLEwPPvig+vfvX+tOMjIytGHDBuXk5CgrK0vt27eXJO3bt0+TJk3SsWPHFB0drYyMDLVt27bS88vLyzVjxgxt3bpVNptNo0eP1qBBg87/1QIAAAAXyOg6ziNGjNCgQYO0fft2T8nt2rWroqKijHbSu3dvDR8+XMOGDavw+NSpUzV06FClpqYqMzNTU6ZM0V//+tdKz8/KytLBgwe1ceNGHTt2TAMHDtTPf/5ztWrVymj/AAAAwMUy/ubAqKgo9erVSykpKerVq5dxaZakHj16yOl0VnisqKhIe/bs8Zyx7t+/v/bs2aMjR45Uev7atWs1aNAg2e12xcbGqk+fPlq/fr3x/gEAAICLZfzNgd6Wl5enuLg4hYSESJJCQkLUrFkz5eXlKTY2ttK6LVq08Nx3Op3Kz88/7302aWJe9nFhHI5GVkeAF/hjHCMjG/h8H3WBr491XfhM8l44p6bjEEzHqC68Zy9UfX5twcKy4myFoqJiuVxuq2PUWw5HIxUWnrA6Bi6SP8bR4WikkpIzPt1HXeHLY10XPpO8F/6ruuMQGdkgqI5RoL9nL1Rd+DxCstttNZ5oNZ6q4W1Op1MFBQUqLy+XdO4PAA8dOlRpSscP6+bm5nru5+XlqXnz5n7LCgAAAFhWnJs0aaL4+HitXr1akrR69WrFx8dXmqYhScnJyXrrrbfkcrl05MgRbd68WUlJSf6ODAAAgCDml+I8Y8YM/epXv1J+fr5GjBihfv36SZKeeOIJLV++XElJSVq+fLmmTZvmec6oUaO0c+dOSVJqaqpatWqlX//617r99ts1btw4tW7d2h/RAQAAAEl+muOclpamtLS0So9fccUVeuutt6p8zpIlSzy3Q0JCKpRqAAAAwN8sm6oBAAAA1CUUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwEGp1ACCYXBodofCwEKtj1MrhaGR1BAAAAg7FGfCj8LAQLXlnh9UxahQZ2UAlJWd8uo9Rt1zj0+0DAOALTNUAAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFh+VY3vvvtO48aN89w/ceKEiouL9fHHH1dYb8GCBXrttdfUrFkzSVK3bt00depUv2YFAABA8LK8OLdq1UqZmZme+zNnzlR5eXmV6w4cOFATJ070VzQAAADAI6CmapSWliorK0u33nqr1VEAAACACiw/4/xj7777ruLi4tSpU6cql69Zs0bbtm2Tw+HQAw88oK5du57X9ps0ifJGTNSAb5yrXWRkA6sj1MofGevCcfAHX39m6sJnkvfCOTUdh2A6RnXhPXuh6vNrCxYBVZzffvvtas82DxkyRGPGjFFYWJg++OAD3XfffVq7dq1iYmKMt19UVCyXy+2tuPg/HI5GKiw8YXWMgOZwNPL5t/JdLH98c6CkgD8O/uLLz0xd+EzWhc+Ev1R3HPz1mQwUgf6evVB14fMIyW631XiiNWCmahQUFOiTTz5RSkpKlcsdDofCwsIkSb/4xS/kdDq1d+9ef0YEAABAEAuY4rxy5UrdcMMN1Z5BLigo8NzOzs5WTk6OLr/8cn/FAwAAQJALmKkaK1eu1OTJkys8NmrUKD344IPq0qWL5s6dq927d8tutyssLEyzZs2Sw+GwKC0AAACCTcAU5w0bNlR6bMmSJZ7bGRkZ/owDAAAAVBAwUzUAAACAQEZxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMBAwV9UAgGDjKivnK7cBoA6hOAOAReyhIVo15c8+235YWKjOni3z2fa9YWD6Q1ZHAABjTNUAAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMAAxRkAAAAwQHEGAAAADFCcAQAAAAMUZwAAAMBAqNUBJCkhIUHh4eFq0KCBJOmxxx5Tr169Kqxz6tQpPf7449q9e7dCQkI0ceJE3XTTTVbEBQAAQBAKiOIsSfPnz1f79u2rXb506VJFRUVp06ZN2r9/v4YNG6aNGzcqMjLSjykBAAAQrOrMVI1169Zp8ODBkqS2bduqc+fO2rJli8WpAAAAECwC5ozzY489Jrfbre7du+uRRx5R48aNKyzPzc1Vy5YtPfedTqfy8/PPax9NmkR5JSuq53A0sjpCwIuMbGB1hFr5I2NdOA7+EBbm2x/Dvt6+N/BeOKem4xBMx6g+/x6pz68tWATET9RXX31VTqdTpaWlmjlzptLT0zVnzhyv76eoqFgul9vr28U5DkcjFRaesDpGQHM4Gqmk5IzVMWoUGdnALxkD/Tj4y9mzZT7bdlhYqE+37y2fZ5/fSZD66DJV/5nw12cyUNTX3yP8jqwb7HZbjSdaA2KqhtPplCSFh4dr6NCh+vzzzyut06JFC+Xk5Hju5+XlqXnz5n7LCAAAgOBmeXE+efKkTpw49y8wt9uttWvXKj4+vtJ6ycnJWrFihSRp//792rlzZ6UrbwAAAAC+YvlUjaKiIj3wwAMqLy+Xy+XSFVdcoalTp0qSUlNTtXjxYsXFxenee+/VpEmTlJiYKLvdrvT0dEVFMWcZAAAA/mF5cW7durVWrVpV5bLMzEzP7YiICM2fP99PqQAAAICKLJ+qAQAAANQFFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMBBqdQAAACBFRja4oGUA/IfiDABAAPg8O7/Kx8PCQnX2bJmf01jjMqsDALVgqgYAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGAi1OgCCw6XREQoPC7E6BgAggLnLy+VwNLI6hs+YvjZXWZmKjp7ycRpcCIoz/CI8LERL3tlhdQzLjbrlGqsjAEDAsoWEaNWUP1sdwyfCwkJ19myZ0boD0x/ycRpcKKZqAAAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYsv6rG0aNH9fvf/14HDx5UeHi42rRpo/T0dMXGxlZYb9KkSfrwww8VExMjSUpOTtbYsWOtiAwAAIAgZHlxttlsGjlypHr27ClJysjI0Jw5c/Tkk09WWnf06NG68847/R0RAAAAsH6qRnR0tKc0S9K1116r3NxcCxMBAAAAlVl+xvnHXC6XXn/9dSUkJFS5fNmyZVqxYoVat26tRx99VFdcccV5bb9JkyhvxEQNavpWpMjIBn5MErjqwnHwR8a6cBz8ISzMtz+Gfb19b6gLGf2hpuMQTMeoPr/W83lt9fkbFOuygHp3Tp8+XREREVVOx3j44YflcDhkt9u1atUqjRw5Ups3b1ZIiPnXOBcVFcvlcnszMn7E4WikwsIT1S4rKTnj50SBKdCPQ2RkA79kDPTj4C+m3yR2Ic7nm8qsVBcy+kN1x6GujKO31NfXer7jWN3vU/iW3W6r8USr5VM1fpCRkaEDBw5o3rx5stsrx4qLi/M8PnDgQJ08eVL5+fn+jgkAAIAgFRDFee7cudq1a5cWLlyo8PDwKtcpKCjw3N66davsdrvi4uL8FREAAABBzvKpGnv37tULL7ygtm3basiQIZKkVq1aaeHChUpNTdXixYsVFxeniRMnqqioSDabTVFRUXr++ecVGmp5fAAAAAQJy5vnVVddpS+//LLKZZmZmZ7bL7/8sp8SAQAAAJUFxFQNAAAAINBRnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxYflWN+u7S6AiFh5l/u2Fdx1eEAgBw8fh9KpWeLdf3x05aHaMCirOPhYeFaMk7O6yO4Rc1fVXzqFuu8XMaAADqrmDpDjUJxO7AVA0AAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMEBxBgAAAAxQnAEAAAADFGcAAADAAMUZAAAAMBBqdQAAwcddXq5Rt1xjdQwAAM4LxRmA39lCQrRqyp+tjmG5gekPWR0BAHAemKoBAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAYozgAAAICBgCjO+/bt0+DBg5WUlKTBgwdr//79ldYpLy/XtGnT1KdPHyUmJuqtt97yf1AAAAAErYAozlOnTtXQoUO1YcMGDR06VFOmTKm0TlZWlg4ePKiNGzdqxYoVWrBggb777jsL0gIAACAYhVodoKioSHv27NGyZcskSf3799f06dN15MgRxcbGetZbu3atBg0aJLvdrtjYWPXp00fr16/XyJEjjfdlt9u8nt9EVESYJfv1t4hLwmRzu6pdHizHoTaBfhxqG0ev7Se6kc/3URf48jiEhoYqrKzMZ9v3Ft4L51R3HOrKOHpLfX0/nO84BvrvCn/xd3erbX82t9vt9lOWKu3atUsTJ07UmjVrPI/17dtXs2fPVqdOnTyPpaSkaObMmfrJT34iSVqyZIkKCgqUlpbm98wAAAAIPgExVQMAAAAIdJYXZ6fTqYKCApWXl0s690eAhw4dktPprLRebm6u535eXp6aN2/u16wAAAAIXpYX5yZNmig+Pl6rV6+WJK1evVrx8fEV5jdLUnJyst566y25XC4dOXJEmzdvVlJSkhWRAQAAEIQsn+MsSV9//bUmTZqk48ePq3HjxsrIyFC7du00atQoPfjgg+rSpYvKy8uVnp6uDz74QJI0atQoDR482OLkAAAACBYBUZwBAACAQGf5VA0AAACgLqA4AwAAAAYozgAAAIABijMAAABggOKMWu3bt0+DBw9WUlKSBg8erP3791daZ9u2bbrlllvUuXNnZWRkVFhWVFSk0aNHKyUlRTfffLOeeOIJlQXR18cGiosdx8LCQo0dO9YzjpmZmX5Kjh8zGceFCxeqX79+SklJ0S233KKtW7d6lp06dUrjx49XYmKikpOT9d577/kxPX7sYscyMzNTKSkpuvrqq7V8+XI/JsePXew4Tps2TcnJyRowYICGDBminTt3+jE9zpsbqMVdd93lXrVqldvtdrtXrVrlvuuuuyqts3//fveePXvcc+fOdT/99NMVls2YMcPzWGlpqfu2225zr1mzxvfBUcHFjuMjjzzifvbZZ91ut9tdVFTkvuGGG9y5ubm+D44KTMZxy5Yt7pMnT7rdbrc7Ozvb3b17d/epU6fcbrfbvWDBAvfkyZPdbrfbvW/fPvf111/vLi4u9lN6/NjFjuWXX37p3rt3r3vChAnuV155xX/BUcHFjuO7777rLi0t9dzu3bu3n5LjQnDGGTUqKirSnj171L9/f0lS//79tWfPHh05cqTCem3atFF8fLxCQ0MrbcNms6mkpEQul0ulpaU6e/as4uLi/JIf53hjHP/973+rV69ekqTY2Fh17NhR69at8314eJiOY69evdSwYUNJUocOHeR2u3Xs2DFJ0rp16zzXwG/btq06d+6sLVu2+O9FQJJ3xrJ9+/a68sorZbfzq9wq3hjHm266SWFhYZKka6+9Vvn5+XK5XP57ETgvfNpQo7y8PMXFxSkkJESSFBISombNmikvL894G/fdd5/27dunX/7yl57/unfv7qvIqII3xrFTp05au3at3G63vv32W23fvl25ubm+iowqXMg4rlq1SpdddpmaN28uScrNzVXLli09y51Op/Lz830bHJV4YyxhPW+P46uvvqobb7yRfwwFMEYGPrd+/Xp16NBB27Zt05YtW/Tpp59q/fr1VsfCeZo0aZIOHz6s1NRUzZw5Uz//+c89vywQmD7++GP9+c9/1p/+9Cero+AiMZb1Q03juGbNGmVlZemJJ57wfzAYozijRk6nUwUFBSovL5cklZeX69ChQ3I6ncbbWL58uQYMGCC73a5GjRopISFB//znP30VGVXwxjjGxsZqzpw5+tvf/qZFixappKREV155pa8iowrnM47bt2/XhAkTtHDhQrVr187zeIsWLZSTk+O5n5eXxxlMC3hjLGE9b43jpk2b9Mwzz2jp0qVq2rSpX7LjwlCcUaMmTZooPj5eq1evliStXr1a8fHxio2NNd5Gq1atPHMoS0tL9dFHH+mqq67ySV5UzRvjePToUc/VUD766CN99dVXnnl98A/Tcfziiy/08MMPa/78+erUqVOFZcnJyVqxYoUkaf/+/dq5c6dn7jr8xxtjCet5Yxzfe+89PfXUU1q6dKlatWrlt+y4MDa32+22OgQC29dff61Jkybp+PHjaty4sTIyMtSuXTuNGjVKDz74oLp06aJPP/1UjzzyiIqLi+V2u9WoUSPNnDlTvXr10sGDBzV16lQdPnxY5eXl6tmzpyZPnlzlH6DBdy52HN9//33NnDlTdrtdMTExmjJliuLj461+WUHHZBxvvfVW5eTkVPgj3FmzZqlDhw46efKkJk2apOzsbNntdk2YMEF9+vSx8BUFr4sdy9WrV2vWrFk6fvy4wsLC1LBhQ7300kv8nyA/u9hx/NnPfqawsLAKZfvll19WTEyMFS8HtaA4AwAAAAaYqgEAAAAYoDgDAAAABijOAAAAgAGKMwAAAGCA4gwAAAAYoDgDAAAABijOAGCxhIQEffjhh17b3nfffacOHTp4vrAGAOAdFGcAgCTvFW5v/0MAAAIFxRkA4FecCQdQV1GcASAA7Ny5U3379tV1112nxx9/XGfOnNE777yjO+64o8J6HTp00IEDByRJp0+f1tNPP62bbrpJ3bt31x133KHTp09X2vaGDRuUkJCgr776Si6XS4sXL1afPn3Us2dPPfTQQzp27Jgk6c4775QkXXfdderatau2b99eY+Y333xTN998s7p27aq+fftq9+7dmjBhgnJzczVmzBh17dpVS5Ys8ZzJfuutt3TjjTfqt7/9rReOGAD4X6jVAQAAUlZWlpYuXaqGDRtqzJgxeu6559SmTZsan5ORkaH//Oc/euONN9S0aVPt2LFDdnvF8yFvv/22Fi1apGXLlqlNmzb6y1/+os2bN2v58uWKjY3VjBkzlJ6errlz52r58uXq3bu3PvnkE4WG1vzrYd26dVqwYIEWLlyoLl266ODBgwoNDdXs2bP12WefacaMGbr++uslnZsCIkmffPKJ1q5dWykjANQV/PQCgAAwbNgwOZ1ORUdHa+zYsVqzZk2N67tcLr399tuaPHmy4uLiFBISom7duik8PNyzzl/+8hctXbpUr7zyiqeEv/HGG3r44YfVvHlzhYeH6/7779eGDRvOe/rE//zP/2jkyJH6yU9+IpvNpjZt2qhly5Y1PueBBx5QRESELrnkkvPaFwAECs44A0AAcDqdntstWrTQoUOHalz/6NGjOnPmjFq3bl3tOkuXLtW4cePUvHlzz2O5ubkaN25chbO+drtdRUVF55U3Ly9Pl1122Xk958c5AKAuojgDQADIy8vz3M7NzVWzZs3UsGHDCnOWCwsLPbdjYmLUoEEDffvtt+rYsWOV23zppZc0cuRINW3aVElJSZLOldcnn3xS3bt3r7R+Tk6OcV6n06mDBw8ary9JNpvtvNYHgEDDVA0ACACvvfaa8vPzdezYMS1atEh9+/ZVx44dtXfvXmVnZ+vMmTNasGCBZ3273a5bb71VTz31lAoKClReXq7t27ertLTUs86VV16pF198Uenp6fr73/8uSbrjjjs0b948T0k+cuSINm/eLEmKjY2V3W7Xt99+W2ve2267TS+99JJ27dolt9utAwcOeLbZtGlTo20AQF1DcQaAANC/f3/dc8896tOnjy677DKNHTtWl19+ucaNG6e7775bv/71ryudJZ44caLat2+v2267TT/96U81Z84cuVyuCut07NhRixYt0h//+Ee9//77Gj58uBISEnTPPfeoa9euuv322/XFF19IkucPE++44w716NFD//rXv6rNe/PNN2vMmDF69NFH1a1bN40bN07ff/+9JGn06NF6/vnn1aNHDy1dutS7BwoALGRzu91uq0MAAAAAgY4zzgAAAIAB/jgQAFClKVOmKCsrq9LjKSkpSk9PtyARAFiLqRoAAACAAaZqAAAAAAYozgAAAIABijMAAABggOIMAAAAGKA4AwAAAAb+H+t+1u9GiOhpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 842.4x595.44 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set(rc={'figure.figsize': (11.7, 8.27)})\n",
    "sns.histplot(data=df, x='bucket_ctr', hue='exp_group', kde=False, palette=['r', 'b'], alpha=0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3086a066-76cd-4e4b-ac67-49ba810742ae",
   "metadata": {},
   "source": [
    "Mann–Whitney U test и бакетное распределение показывают явное, статистически значимое отличие."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd8d4300-b31b-4e1c-a850-df7c50bfaba7",
   "metadata": {},
   "source": [
    "## Summary"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d724aa44-d12b-4e28-b72f-d13c30ffaeb4",
   "metadata": {},
   "source": [
    "Основываясь на данных проведенных тестов, рекомендация не использовать новый алгоритм на всех пользователях. \n",
    "Возможно, команде ML имеет смысл проводить дальнейшее исследование алгоритма, потому что характер распределения показывает, что алгоритм, условно разделил пользователей на две группы.\n",
    "У одной группы CTR стал меньше, у другой группы CTR стал больше. Возможно, понимание причин такого поведения алгоритма сможет помочь команде ML его улучшить. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}