Final Question2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "\n",
    "import math\n",
    "from collections import defaultdict\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################\n",
    "class ClassCounter:\n",
    "    def __init__(self, subject):\n",
    "        #initialize the subject name\n",
    "        self.subject = subject\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        #initialize the array which is to count how many elements are there in each class\n",
    "        #initialize the array which store the percentage of numbers of each class in the total numbers of instance\n",
    "        self.__classes_count = [0] * class_num\n",
    "        self.__classes_prob = [0] * class_num\n",
    "        \n",
    "    def add(self, class_index):\n",
    "        #count how many elements are there in each class\n",
    "        self.__classes_count[class_index] += 1\n",
    "    \n",
    "    def run(self):\n",
    "        #calculate how many elements are there in total \n",
    "        total = sum(self.__classes_count)\n",
    "        #calculate the percentage of numbers of each class in the total numbers of instance\n",
    "        for index, val in enumerate(self.__classes_count):\n",
    "            self.__classes_prob[index] = val/total\n",
    "    \n",
    "    def get(self):\n",
    "        #return the percentage of numbers of each class in the total numbers of instance\n",
    "        return tuple(self.__classes_prob)\n",
    "    \n",
    "    def get_classes_count(self):\n",
    "        #return how many elements are there in each class\n",
    "        return self.__classes_count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Father class to count the elements of attribute\n",
    "class AttrCounter:\n",
    "    def __init__(self, name):\n",
    "        #store the name of the attribute \n",
    "        self.name = name\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        #initialize the container to store the elements\n",
    "        return\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        #precess the dataset and instore the elements in to the container that initialized in thre prepare function\n",
    "        return \n",
    "    \n",
    "    def run(self):\n",
    "        #calculate the precentage\n",
    "        return \n",
    "    \n",
    "    def get(self, attr):\n",
    "        #returm the precentage\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#son class to counter the elemnts of nominal attibute\n",
    "class NomCounter(AttrCounter):\n",
    "    __smooth_alpha = 1\n",
    "    \n",
    "    def __init__(self, name):\n",
    "        #store the name of this attribute\n",
    "        super().__init__(name)\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        #store all the possible values of an attribute in a set\n",
    "        self.__attr_vals = set()\n",
    "        #for each class values, create a dictionary to store the corresponding numbers of this attribute values\n",
    "        self.__attr_count = [defaultdict(int) for i in range(class_num)]\n",
    "        #for each class values, create a dictionary to store the corresponding precentage of this attribute values(the target value)\n",
    "        self.__attr_prob = [defaultdict(float) for i in range(class_num)]\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        #add the attribute values\n",
    "        self.__attr_vals.add(attr)\n",
    "        #add the number of attribute value in corresponding class\n",
    "        self.__attr_count[class_index][attr] += 1\n",
    "    \n",
    "    def run(self):\n",
    "        #number of attribute in the dataset\n",
    "        dimension = len(self.__attr_vals)\n",
    "        #to calculate the precentage of each attribute value precentage in this particular class\n",
    "        for index, attrs in enumerate(self.__attr_count):\n",
    "            total = reduce(lambda x, y: x + y, attrs.values())\n",
    "            for val in self.__attr_vals:\n",
    "                self.__attr_prob[index][val] = self._laplace_smooth(attrs[val], total, dimension)\n",
    "    \n",
    "    def get(self, attr): \n",
    "        #return the precentage\n",
    "        return tuple(prob[attr] for prob in self.__attr_prob)\n",
    "    \n",
    "    #use Laplace smoothing to replace the zero value\n",
    "    def _laplace_smooth(self, num, total, dimension):\n",
    "        return (num+self.__smooth_alpha)/(total+dimension*self.__smooth_alpha)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#son class to counte the element of the numberical attributes\n",
    "class NumCounter(AttrCounter):\n",
    "    \n",
    "    def __init__(self, name):\n",
    "        #store the name of this attribute\n",
    "        super().__init__(name)\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        #for each class, create the container to store the corresponding attribute values\n",
    "        self.__attr_data = [[] for i in range(class_num)]\n",
    "        #for each class,calculate the mean of this attribute(the target value)\n",
    "        self.__mean = [0] * class_num\n",
    "        #for each class, calculate the deviation of this attribute(the target value)\n",
    "        self.__deviation = [0] * class_num\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        #to store all the values into the container\n",
    "        self.__attr_data[class_index].append(float(attr))\n",
    "    \n",
    "    def run(self):\n",
    "        #use the values that stored in the container to calculate the mean values adn deviation values \n",
    "        for index, data in enumerate(self.__attr_data):\n",
    "            #calculate the mean value\n",
    "            mean = sum(data)/len(data)\n",
    "            #calculate the deviation values\n",
    "            deviation = math.sqrt(sum([(x - mean)**2 for x in data])/(len(data)-1))\n",
    "            #set the mean inside the class\n",
    "            self.__mean[index] = mean\n",
    "            #set the deviation inside the class\n",
    "            self.__deviation[index] = deviation\n",
    "    \n",
    "    def get(self, attr):\n",
    "        #return the normal distribution\n",
    "        return tuple(self._normal_distribution(float(attr), self.__mean[i], self.__deviation[i]) for i in range(len(self.__mean)))\n",
    "    \n",
    "    def _normal_distribution(self, val, mean, deviation):\n",
    "        ###to calculate the normal distribution\n",
    "        return 1/(deviation*math.sqrt(2*math.pi))*np.exp(-((val-mean)/deviation)**2/2) if deviation > 0 else 0\n",
    "    \n",
    "    def draw_histogram(self):\n",
    "        ##################\n",
    "        bin_size = 20#####\n",
    "        ##################\n",
    "        plt.figure()\n",
    "        for i, values in enumerate(self.__attr_data):\n",
    "            plt.hist(values, bins = bin_size,color = 'cyan', edgecolor = 'k', label = 'historgram') \n",
    "            plt.title(self.name+' class'+str(i))\n",
    "            plt.show()\n",
    "        \n",
    "    def draw_box_plot(self):\n",
    "        figure,axes=plt.subplots() #得到画板、轴\n",
    "        axes.boxplot(self.__all_values,patch_artist=True) #描点上色\n",
    "        plt.show()\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Num2NomCounter(NomCounter):  \n",
    "    def __init__(self, name, bins):\n",
    "        assert(bins >= 2)\n",
    "        super().__init__(name)\n",
    "        self.__bins = bins\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        self.__split = [0] * (self.__bins-1)\n",
    "        self.__attr_data = []\n",
    "        self.__attr_count = [[0]*self.__bins for i in range(class_num)]\n",
    "        self.__attr_prob = [[0]*self.__bins for i in range(class_num)]\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self.__attr_data.append((float(attr), class_index))\n",
    "    \n",
    "    def run(self):\n",
    "        self.__attr_data.sort()\n",
    "        gaps = self.__split_integer(len(self.__attr_data),self.__bins)\n",
    "        cur = 0\n",
    "        for i in range(self.__bins-1):\n",
    "            npos = sum(gaps[:i+1])\n",
    "            self.__split[i] = self.__attr_data[npos][0]\n",
    "            for j in range(cur, npos):\n",
    "                self.__attr_count[self.__attr_data[j][1]][i] += 1\n",
    "            cur = npos\n",
    "        for j in range(cur, len(self.__attr_data)):\n",
    "            self.__attr_count[self.__attr_data[j][1]][i+1] += 1           \n",
    "        for index,attrs in enumerate(self.__attr_count):\n",
    "            total = reduce(lambda x, y: x + y, attrs)\n",
    "            for val, num in enumerate(attrs):\n",
    "                self.__attr_prob[index][val] = self._laplace_smooth(num, total, self.__bins)\n",
    "    \n",
    "    def get(self, attr):\n",
    "        attr = float(attr)\n",
    "        for i in range(self.__bins):\n",
    "            if i >= len(self.__split) or attr < self.__split[i]:\n",
    "                break\n",
    "        return tuple(prob[i] for prob in self.__attr_prob)\n",
    "    \n",
    "    def __split_integer(self, m, n):\n",
    "        quotient = int(m / n)\n",
    "        remainder = m % n\n",
    "        if remainder > 0:\n",
    "            return [quotient] * (n - remainder) + [quotient + 1] * remainder\n",
    "        if remainder < 0:\n",
    "            return [quotient - 1] * -remainder + [quotient] * (n + remainder)\n",
    "        return [quotient] * n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Nom2NumCounter(NumCounter):\n",
    "    def __init__(self, name, mapper):\n",
    "        super().__init__(name)\n",
    "        self.__mapper = mapper\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self._attr_data[class_index].append(self.__mapper(attr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Evaluation:\n",
    "#     num_of_class = 0\n",
    "#     num_of_inst = 0\n",
    "#     result = []\n",
    "#     multiclass_matrix=None\n",
    "#     correct_pre = []\n",
    "#     actual_num = []\n",
    "#     predict_num = []\n",
    "    \n",
    "    def __init__(self,result,num_of_class):\n",
    "        #the input(list of tuples, tuple has two value, first one is real class value, second is the predict one)\n",
    "        self.result = result\n",
    "        #number of class\n",
    "        self.num_of_class = num_of_class\n",
    "        #number of instances\n",
    "        self.num_of_inst = len(result)\n",
    "        #generate the muticlass matrix(that mentions in the lecture slide)\n",
    "        self.multiclass_matrix = self.generate_matrix()\n",
    "        #use the multiclass matrix to generte some value that we need to evaluate the model\n",
    "        self.precess_matrix()\n",
    "    \n",
    "    def generate_matrix(self):\n",
    "        matrix = np.zeros((self.num_of_class, self.num_of_class))\n",
    "        for tup in self.result:\n",
    "            matrix[tup[0]][tup[1]]+=1\n",
    "        return matrix\n",
    "    \n",
    "    def precess_matrix(self):\n",
    "        #fill in these array \n",
    "        #the values on the matrix diagonal, how many numbers of instance that the model correctly predict for this particular class\n",
    "        self.correct_pre = [self.multiclass_matrix[i][i] for i in range(self.num_of_class)]\n",
    "        #the actual numbers of instance that each class has in this dataset(TP+FN)\n",
    "        self.actual_num = [self.multiclass_matrix[i].sum() for i in range(self.num_of_class)]\n",
    "        #all numbers of instance that we predict for each class(TP+FP)\n",
    "        self.predict_num = [self.multiclass_matrix[:,i].sum() for i in range(self.num_of_class)]\n",
    "    \n",
    "    \n",
    "    def calculate_accuracy(self):\n",
    "        #calculate the accuracy\n",
    "        accuracy = sum(self.correct_pre)/sum(self.actual_num)\n",
    "        return accuracy\n",
    "    \n",
    "    def calculate_Macro_value(self):   \n",
    "        ###precision###\n",
    "        temp_arr1 = []\n",
    "        #calculate the precision by its formula\n",
    "        for i in range(self.num_of_class):\n",
    "            #if there is any value is 0, then store 0(otherwise, denominator will be zero)\n",
    "            if(self.correct_pre[i]==0 or self.predict_num[i]==0):\n",
    "                temp_arr1.append(0)\n",
    "            else:\n",
    "                this_pre = self.correct_pre[i]/self.predict_num[i]\n",
    "                temp_arr1.append(this_pre)\n",
    "        precision = sum(temp_arr1)/self.num_of_class\n",
    "            \n",
    "        ###recall###\n",
    "        temp_arr2 = []\n",
    "        #calculate the recall by its formula\n",
    "        for i in range(self.num_of_class):\n",
    "            #if there is any value is 0, then store 0(otherwise, denominator will be zero)\n",
    "            if(self.correct_pre[i]==0 or self.actual_num[i]==0):\n",
    "                temp_arr2.append(0)\n",
    "            else:\n",
    "                this_recall = self.correct_pre[i]/self.actual_num[i]\n",
    "                temp_arr2.append(this_recall)\n",
    "        recall = sum(temp_arr2)/self.num_of_class\n",
    "        \n",
    "        return [precision, recall]\n",
    "    \n",
    "    \n",
    "    def calculate_Micro_value(self):\n",
    "        ###precision###\n",
    "        precision = sum(self.correct_pre)/sum(self.predict_num)\n",
    "        ###recall###\n",
    "        recall = sum(self.correct_pre)/sum(self.actual_num)\n",
    "        return [precision, recall]\n",
    "    \n",
    "    def calculate_weight_value(self):\n",
    "        ###precision###\n",
    "        temp_arr1 = []\n",
    "        #calculate the precision by its formula\n",
    "        for i in range(self.num_of_class):\n",
    "            #if there is any value is 0, then store 0(otherwise, denominator will be zero)\n",
    "            if(self.correct_pre[i]==0 or self.predict_num[i]==0):\n",
    "                temp_arr1.append(0)\n",
    "            else:\n",
    "                this_pre = (self.actual_num[i]/self.num_of_inst)*(self.correct_pre[i]/self.predict_num[i])\n",
    "                temp_arr1.append(this_pre)\n",
    "        precision = sum(temp_arr1)     \n",
    "        ###recall###\n",
    "        temp_arr2 = []\n",
    "        #calculate the recall by its formula\n",
    "        for i in range(self.num_of_class):\n",
    "            #if there is any value is 0, then store 0(otherwise, denominator will be zero)\n",
    "            if(self.correct_pre[i]==0 or self.actual_num[i]==0):\n",
    "                temp_arr2.append(0)\n",
    "            else:\n",
    "                this_recall = (self.actual_num[i]/self.num_of_inst)*(self.correct_pre[i]/self.actual_num[i])\n",
    "                temp_arr2.append(this_recall)\n",
    "        recall = sum(temp_arr2)\n",
    "        \n",
    "        return [precision, recall]\n",
    "        \n",
    "        \n",
    "    def draw_heatmap_d(self):\n",
    "        #draw the heatmap with numbers of instance on it\n",
    "        sns.heatmap(self.multiclass_matrix, vmin=0, vmax=len(self.result), annot=True,cmap=\"RdPu_r\", fmt='.20g')\n",
    "        plt.show()\n",
    "        \n",
    "    def draw_heatmap_f(self):\n",
    "        #draw the heatmap with precentage on it\n",
    "        sns.heatmap(self.multiclass_matrix/len(self.result), vmin=0, vmax=1, annot=True,cmap=\"RdPu_r\", fmt='.2g')\n",
    "        plt.show()  \n",
    "        \n",
    "    def draw_bar_chart(self):\n",
    "        #draw the bar chart\n",
    "        #for each class\n",
    "        #show the numbers of predict value, actual value, correctly predict value\n",
    "        names = [\"calss\"+str(i) for i in range(self.num_of_class)]\n",
    "        plt.figure(figsize=(10,6), dpi=80)\n",
    "        x = range(len(names))\n",
    "        \n",
    "        plt.bar(x, self.actual_num, width=0.1)\n",
    "        plt.bar([i + 0.1 for i in x], self.predict_num, width=0.1)\n",
    "        plt.bar([i + 0.2 for i in x], self.correct_pre, width=0.1)\n",
    "\n",
    "        plt.xticks([i + 0.05 for i in x], names)\n",
    "        \n",
    "        plt.xlabel('Class Name', fontsize=14)\n",
    "        plt.ylabel('Number Of Each Class', fontsize=14)\n",
    "        plt.show()\n",
    "        \n",
    "        \n",
    "    def evaluate(self):\n",
    "        ################################Calculate Accuracy##################################\n",
    "        accuracy = self.calculate_accuracy()\n",
    "        #print(self.correct_pre,self.actual_num, self.predict_num)\n",
    "        print(\"accuracy is: %f\" %accuracy)\n",
    "        print(\"error rate is: %f\"%(1-accuracy))\n",
    "\n",
    "        ##macro averaging\n",
    "        precisionMa, recallMa = self.calculate_Macro_value()\n",
    "        print(\"\\n----------------the macro averaging way------------------\")\n",
    "        print(\"Marco averaging precision is: \",precisionMa)\n",
    "        print(\"Marco averaging recall is:\",recallMa)\n",
    "        \n",
    "        #micro averaging\n",
    "        precisionMi, recallMi = self.calculate_Micro_value()\n",
    "        print(\"----------------the micro averaging way------------------\")\n",
    "        print(\"Mirco averaging precision is:\", precisionMi)\n",
    "        print(\"Mirco averaging recall is:\", recallMi)\n",
    "        \n",
    "        #weight averaging\n",
    "        precisionWe, recallWe = self.calculate_weight_value()\n",
    "        print(\"----------------the weight averaging way------------------\")\n",
    "        print(\"weight averaging precision is:\" ,precisionWe)\n",
    "        print(\"weight averaging recall is:\" ,recallWe)\n",
    "        \n",
    "        ###################################Draw Graphs######################################\n",
    "        #self.draw_heatmap_d()\n",
    "        #self.draw_heatmap_f()\n",
    "        #self.draw_bar_chart()\n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Model:\n",
    "    def __init__(self, classes, *counters):\n",
    "        self.num = 10\n",
    "        #convert the class name(string) into the number(start from 0), like class name1 = 0, class name2 = 1, class name3 = 2, etc.\n",
    "        return_value = '''eg. {'unacc': 0, 'acc': 1, 'good': 2, 'vgood': 3}'''\n",
    "        self.__classes = {classname:index for index,classname in enumerate(classes)}\n",
    "        #it is a tuple to store all initialized particular attribute(in the particular type class)\n",
    "        self.__counters = counters\n",
    "        #initialize the filters\n",
    "        self.__filters = ()\n",
    "        #to judge whether we need to delete the first column (some time the first colum is the serial number)\n",
    "        self.__dump_firt_column = False\n",
    "        for index, counter in enumerate(counters):\n",
    "            #if the counter is a class type counter\n",
    "            if isinstance(counter, ClassCounter):\n",
    "                #set the position of the target class column in this Model class \n",
    "                self.__classpos = index\n",
    "            #use the function in the class to prepare the each attribute  (which is initialze all the attributes values container)\n",
    "            counter.prepare(len(classes))\n",
    "            \n",
    "    def set_filters(self, filters):\n",
    "        #set the particular filters\n",
    "        #because different datasets has different punctuation to represent the missing value\n",
    "        #so we need to set the particular filters to particular punctuation\n",
    "        self.__filters = filters\n",
    "        \n",
    "    def set_dump_firt_column(self, enabled):\n",
    "        #set whether we need to delete the first column (some time the first colum is the serial number)\n",
    "        self.__dump_firt_column = enabled\n",
    "            \n",
    "    def preprocess(self, lines):\n",
    "        #we will read the file line by line, so that we gonna preprocess the data line by line\n",
    "        for line in lines:\n",
    "            #split all the data in a line\n",
    "            inputs = line.strip().split(\",\")[1:] if self.__dump_firt_column else line.strip().split(\",\")\n",
    "            #if we successfully read the file lines into the program, continue running this preprocess function\n",
    "            if not inputs or not inputs[0]:\n",
    "                continue\n",
    "                \n",
    "            ###DEALING WITH CLASS VALUE###\n",
    "            #find out the class value of this instance\n",
    "            class_index = self.__classes[inputs[self.__classpos]]\n",
    "            ###DEALING WITH EACH ATTRIBUTE VALUE\n",
    "            #read in all the values and use their particular way to store into their value container \n",
    "            for i, val in enumerate(inputs):\n",
    "                ########!!!!!########\n",
    "                #if there is no missing value, continue store the instance inside the attribute class container, \n",
    "                #otherwise discard this attribute (we choose this way to deal with the missing value which will explain in the written example)\n",
    "                if val not in self.__filters:\n",
    "                    #if this is the value of the target class\n",
    "                    if i == self.__classpos:\n",
    "                        #add value in the ClassCounter\n",
    "                        self.__counters[i].add(class_index)\n",
    "                    else:\n",
    "                        #otherwise add value in their own AttriCounter\n",
    "                        self.__counters[i].add(val, class_index)\n",
    "                \n",
    "    def train(self):\n",
    "        #train the training set by running the function in their class\n",
    "        for counter in self.__counters:\n",
    "            counter.run()\n",
    "\n",
    "    def test(self, lines):\n",
    "        res = []\n",
    "        #we still need to read the file line by line, so that we gonna preprocess the test data line by line\n",
    "        for line in lines:\n",
    "            #split all the data in the line\n",
    "            inputs = line.strip().split(\",\")[1:] if self.__dump_firt_column else line.strip().split(\",\")\n",
    "            #if we successfully read the file lines into the program, continue running this preprocess function \n",
    "            if not inputs or not inputs[0]:\n",
    "                continue\n",
    "            \n",
    "            #real target class type of this instance\n",
    "            real = self.__classes[inputs[self.__classpos]]\n",
    "            #all the predict value \n",
    "            predict = []\n",
    "            \n",
    "            for i, val in enumerate(inputs):\n",
    "                #if there is no missing value, continue store the instance inside the attribute class container, \n",
    "                #otherwise discard this attribute (we choose this way to deal with the missing value which will explain in the written example)\n",
    "                if val not in self.__filters:\n",
    "                    #caluculate the presentage that each type that this instance might have\n",
    "                    if i == self.__classpos:\n",
    "                        predict.append(np.log2(np.array(self.__counters[i].get())))\n",
    "                    else:\n",
    "                        predict.append(np.log2(np.array(self.__counters[i].get(val))))\n",
    "            #store the real value and our predict value in a tuple(real value, predict value)\n",
    "            res.append((real, np.argmax(reduce(lambda x, y: x+y, predict))))\n",
    "        return res\n",
    "    \n",
    "    def train_zero_R_baseline(self):\n",
    "        for counter in self.__counters:\n",
    "            #if the counter is a class type counter\n",
    "            if isinstance(counter, ClassCounter):\n",
    "                #get the each number of times class happened at dataset(which store at the ClassCounter) eg.[3630(class0), 1152(class1), 207(class2), 195(class3)]\n",
    "                self.__class_count = counter.get_classes_count()\n",
    "                #find out the class that happened for most of time\n",
    "                maximum_class = self.__class_count[0]\n",
    "                #return its index(which is the index that represent the class)\n",
    "                self.___majority_class_index = 0\n",
    "                #start to find that class\n",
    "                for i in range(len(self.__class_count)):\n",
    "                    if(self.__class_count[i]>maximum_class):\n",
    "                        maximum_class = self.__class_count[i]\n",
    "                        self.___majority_class_index = i\n",
    "                        \n",
    "    def test_zero_R_baseline(self, lines):\n",
    "        res = []\n",
    "        #we still need to read the file line by line, so that we gonna preprocess the test data line by line\n",
    "        for line in lines:\n",
    "            #split all the data in the line\n",
    "            inputs = line.strip().split(\",\")[1:] if self.__dump_firt_column else line.strip().split(\",\")\n",
    "            #if we successfully read the file lines into the program, continue running this preprocess function \n",
    "            if not inputs or not inputs[0]:\n",
    "                continue\n",
    "            #real target class type of this instance\n",
    "            real = self.__classes[inputs[self.__classpos]]\n",
    "            #and predict all the class value of test instance as the class that occur for the most of time\n",
    "            res.append((real,self.___majority_class_index))\n",
    "        return res\n",
    "    \n",
    "    def graw_attr_distribution(self):\n",
    "        for counter in self.__counters:\n",
    "            if isinstance(counter, NumCounter):\n",
    "                counter.draw_histogram()\n",
    "                \n",
    "    def evaluate(self,result, classes):\n",
    "        evaluate_result = Evaluation(result, len(classes))\n",
    "        evaluate_result.evaluate()   \n",
    "        return evaluate_result\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_two_evaluation_class(evaluation1, evaluation2):\n",
    "        #generate the value throught the function inside the Evaluation class\n",
    "        accuracy1 = evaluation1.calculate_accuracy()\n",
    "        accuracy2 = evaluation2.calculate_accuracy()\n",
    "        precision1,recall1 = evaluation1.calculate_Macro_value()\n",
    "        precision2,recall2 = evaluation2.calculate_Macro_value()\n",
    "        \n",
    "        \n",
    "        #generate the x-axis\n",
    "        value_names = ['accuracy', 'precision','recall']\n",
    "        #generate the two y-axis\n",
    "        values1 = [accuracy1,precision1,recall1]\n",
    "        values2 = [accuracy2,precision2,recall2]\n",
    "        \n",
    "        \n",
    "        #draw the bar chart\n",
    "        plt.figure(figsize=(10,6), dpi=80)\n",
    "        x = range(len(value_names))\n",
    "\n",
    "        m1 = plt.bar(x, values1, width=0.1)\n",
    "        m2 = plt.bar([i + 0.1 for i in x], values2, width=0.1)\n",
    "\n",
    "        plt.xticks([i + 0.05 for i in x], value_names)\n",
    "        #draw the legend\n",
    "        plt.legend((m1,m2),(\"Navie Bayes Classifier\",\"Baseline\"), loc=\"upper right\")\n",
    "        \n",
    "        #lable the value\n",
    "        def autolabel(rects): \n",
    "            for rect in rects:\n",
    "                height = rect.get_height()\n",
    "                plt.text(rect.get_x()+rect.get_width()/2., 1.03*height, '%s' % float(height))\n",
    "\n",
    "        autolabel(m1)\n",
    "        autolabel(m2)\n",
    "        \n",
    "        #label x and y axis name\n",
    "        plt.xlabel('value name', fontsize=14)\n",
    "        plt.ylabel('precentage', fontsize=14)\n",
    "        #show the figure\n",
    "        plt.show()\n",
    "        \n",
    "#compare_two_evaluation_class(normal_method, baseline_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "###11. Wine Dataset - numeric only###\n",
    "####################################################################################################\n",
    "filename = \"datasets/wine.data\"\n",
    "classes = (\"1\" , \"2\", \"3\")\n",
    "thisClassCounter = ClassCounter(\"Alcohol\"),\n",
    "model = Model(\n",
    "    classes,\n",
    "    thisClassCounter,\n",
    "    NumCounter(\"age\"),\n",
    "    NumCounter(\"Malic acid\"),\n",
    "    NumCounter(\"Ash\"),\n",
    "    NumCounter(\"Alcalinity of ash\"),\n",
    "    NumCounter(\"Magnesium\"),\n",
    "    NumCounter(\"Total phenols\"),\n",
    "    NumCounter(\"Flavanoids\"),\n",
    "    NumCounter(\"Nonflavanoid phenols\"),\n",
    "    NumCounter(\"Proanthocyanins\"),\n",
    "    NumCounter(\"Color intensity\"),\n",
    "    NumCounter(\"Hue\"),\n",
    "    NumCounter(\"OD280/OD315 of diluted wines\"),\n",
    "    NumCounter(\"Proline\")\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy is: 0.988764\n",
      "error rate is: 0.011236\n",
      "\n",
      "----------------the macro averaging way------------------\n",
      "Marco averaging precision is:  0.9885024432308134\n",
      "Marco averaging recall is: 0.9896554468051245\n",
      "----------------the micro averaging way------------------\n",
      "Mirco averaging precision is: 0.9887640449438202\n",
      "Mirco averaging recall is: 0.9887640449438202\n",
      "----------------the weight averaging way------------------\n",
      "weight averaging precision is: 0.9888786975464341\n",
      "weight averaging recall is: 0.9887640449438202\n"
     ]
    }
   ],
   "source": [
    "#weight averaging precision is: 0.871943\n",
    "#weight averaging recall is: 0.715856\n",
    "with open(filename,\"r\") as file:\n",
    "    lines = file.readlines()\n",
    "    model.preprocess(lines)\n",
    "    model.train()\n",
    "    result = model.test(lines)\n",
    "    normal_method = model.evaluate(result, classes)\n",
    "\n",
    "thisClassCounter =  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy is: 0.398876\n",
      "error rate is: 0.601124\n",
      "\n",
      "----------------the macro averaging way------------------\n",
      "Marco averaging precision is:  0.13295880149812733\n",
      "Marco averaging recall is: 0.3333333333333333\n",
      "----------------the micro averaging way------------------\n",
      "Mirco averaging precision is: 0.398876404494382\n",
      "Mirco averaging recall is: 0.398876404494382\n",
      "----------------the weight averaging way------------------\n",
      "weight averaging precision is: 0.15910238606236585\n",
      "weight averaging recall is: 0.398876404494382\n"
     ]
    }
   ],
   "source": [
    "with open(filename,\"r\") as file:\n",
    "    lines = file.readlines()\n",
    "    model.preprocess(lines)\n",
    "    model.train_zero_R_baseline()\n",
    "    result = model.test_zero_R_baseline(lines)\n",
    "    baseline_method = model.evaluate(result, classes)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAvoAAAGgCAYAAADb4sPZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nOzdd3xUVf7/8dcJCQSklygaQugQIAQIdaUJohTZdUHXggpiW0FXEZSviCvKIiv8ZHftFVTk61LEgthAEVxgAelVOgmItISEkv75/THJfNNhYgqO7+fjMQ8y93POPedO5obPnDn3XGdmiIiIiIiIfwko6w6IiIiIiEjxU6IvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+KHAsu5ASahQoYLVqVOnrLtxQdLS0oiLiyM9PZ2AgABq1KhBUFBQjjIZGRmcOnWKlJQUAIKDg6latSrOOcyMhIQEkpKSALz7CAz0/GoTExM5e/YszjkAqlevTvny5UlNTSUuLi5HGxkZGVx++eUAmBmnTp0iKSkJ5xxBQUHUrFkzR78SEhJITEwkJCTE2+cLOZ786hUWS0pKIiEhATPDOZfvPs+cOUN8fDw1a9akYsWKAMTFxZGSkuI99qpVqxIcHAxASkoK8fHx3mOtVKkSVapUAfAeN4BzjmrVqlGhQoU8rwtA5cqVqVy5MgCnT5/mzJkz3nqVK1emUqVKeX7nIlnK6vwHOHToEIGBgd5Y9vd51vmRkZFBYGAgNWrUoFy5cqSnp3v7CxAUFET16tUJCMg5ZuTrOX78+HHS09NxznnPufLly2NmnDx5ktTUVAICAggICKB69ere4yvsHM/qa1paGs45qlevnuf4IO/5f+bMGU6fPk1aWhrVqlXznt9Ajr+JQL7neHJyMsePH89TVyS30jr/Ae95nHWuZmRkEB8f7z1/goODqVatGlD434bCYlni4uI4e/YsdevWzfO3IXfs0KFDKcBlwItARyAN+NjMxgE454YCjwIZgAGPm9nnmbEmwDtAbSAeGGZm2zJj+4GkzAfAs2b278zYv4BBQH2gtZltydweDHwARABngSPAfWa2P/sxOOfuAGYC15nZwsxtS4EwICGz2DtmNp2LgZn53eOKK66wX4tevXrZjBkzzMxs7ty51rlz5zxlHn/8cRs2bJhlZGRYSkqK9e3b1+bMmWNmZh999JF17NjRUlJSzMzsmWeesRtuuMHMzDZs2GBhYWGWmJhoZmbvvfeedejQId9+jBw50kaNGuV9/tBDD9kDDzxgGRkZZmZ2+PDhHOV/+OEHu/baay0sLMw2b958wcdTUL2CYidPnrRatWrZtm3bzMxs6dKl1rJlyxz1YmJirEuXLta5c2f79NNPvdvj4uK8P69fv95q1arlPZ6oqCj7+OOPzczsxIkTVqdOHdu6dauZmS1atMjOnj3rfQ2rV69u586dMzOzd955x6666ipLS0uzEydOWP369W379u1mZrZ48WKLj483M7ODBw9a7dq1bf/+/fm+3iJmZXv+A95YdhkZGdaoUSP79ttvzcxs6tSpdtNNN5mZ2ZEjR2z58uXesmPGjLG77747R31fz3GznOfqggULrG3btmZmdu7cOfvss8+85+0LL7xgV199db71cp/jw4cPt7/+9a9mZrZ69WoLCwuz1NRUMyv8/N+wYYNt27bNbrvtNnvhhRdy9D97e4cOHbIqVarYyZMnvdsSEhKsY8eONmDAgDx1RXIryfP/q6++slatWllCQoKZmT311FN2//33e/f7hz/8waZOnep9nv3/+IL+NpwvZmb2ySef2J133plvufxiQCywABhjmTkcUDfz35p4Eues51cCR7OV+wZPcg8wBFiZLbYfaGX55IhAdyA0dxkgGOgPuMzno4CvctUNBVYAK4GB2bYvzf78Ynpo6k4ZOnr0KOvWrWPo0KEADB48mH379rF///4c5TZu3Ei/fv28I+t9+/blvffe88aTk5NJSkryfroPDQ31xlJTU72jzPHx8Tli2evPnj2bESNGAJ4RrRkzZjB58mTvp/a6devmKD9y5Ehefvllb/xCjqegeoXF9uzZQ0hICC1atACgR48eHDhwgHXr1nnL3HPPPUyfPj3PqEL16tW9P8fHx+dpM2tE78yZM5QvX977jUW/fv283wq0bt2a9PR0jh8/DsC///1v7rvvPsqVK0fNmjW58cYb+eCDDwDo3bu3d0SkXr16XHrppcTExOR5vUXg4jn/c1u7di0VKlSgZ8+eANx777189NFHpKamcumll3LllVd6y3bq1Im9e/fm6Iuv5zjkPFdPnTrlHQUMDg6mf//+3vKdO3fO0V5h5/icOXMYOXIkAB06dODSSy/l+++/z1Ee8p7/bdq0oUWLFnlGInO3l5iYiHOOjIwM77bRo0czduxYateunaeuSHYlff5v3LiRbt26eb+pGjhwoLfe7t27WbduHaNHj/buJ/v/8UV14sQJJk6cyPPPP+9LrBzQDvAGzOynzB8DAAdkfTVWHc8HA5xzIZn1ZmXG5gMNnHPh5+unmS0zs9h8tieZ2aLMDyAAq4CGuYq9DjwMJJ+vnYuFEv0yFBMTw+WXX+79ms05R1hYGAcPHsxRrkOHDsyZM4eUlBQSExNZsGCB94/BddddR69evbjsssuoW7cuS5Ys4emnnwY8/2GNHj2aBg0aEBoayvTp03nhhRfy9OPDDz+kQYMGREVFAZ7kulatWkyaNIno6Gi6devGkiVLvOWffPJJhg4dSoMGDXw6noLqFRZr0qQJx44dY9WqVQAsWLCA06dPe4//lVdeoWXLlnTq1Cnf13jcuHE0atSIP/7xj8ydO9ebCMyYMYMJEyYQFhZG06ZNefbZZ7nsssvy1J8xYwaNGjXy/vE8ePAg9evX98bDw8Pz/L4AFi9eTFxcHO3bt8+3XyIXw/nfs2dPb7msDwS53+NVqlShSpUq/PTTTznqpqen89JLL3Hdddd5txXlHM9y++23U69ePZ544gneeeedfMv861//ytEe5H+OnzhxgoyMDLJP4cx+rl7o+V9QH5o1a0a7du14/fXXqVWrFgCff/458fHxDBky5IL2I79tJX3+R0dH8/XXX/Pzzz9jZsyaNYvExEROnjzJtm3bqFevHvfddx/t2rWjb9++rF+/Pke7+f1tOF9s5MiRPPXUU94Br+wKiQUBMcCrzrl1zrmvnHNtAczsOHAfsM45dwB4GxiWWa8ecNjM0jLLGnAQz/SZLO875zY75950zhVlPveDwKdZT5xzfwa2mtl/Cyg/NbO9fzvncn9AKDNK9MtY7pGt//sg+X8ee+wx6tWrR8eOHRk0aBBdu3b1zuNbt24dO3bs4NChQxw+fJjevXszatQoAA4cOMAnn3zCnj17iI2N5eGHH+bWW2/Ns/+3337bO5oPnlHAvXv3EhERwdq1a3nxxRe56aabOHbsGCtXrmTNmjXcf//9Ph1PYfUKi1WrVo358+czbtw42rdvz9KlS4mIiCAoKIh9+/bxxhtveP+w5WfKlCns2bOHOXPmMHbsWO88x6lTpzJ16lQOHjzI1q1bGT9+PDt37sxRd8mSJUycONE7Yp/fMeb3+9q8eTPDhw/n3//+t/ebAZH8lOX5f+DAAdauXcuKFSs4duwYY8eOveB+mRn3338/1atX54EHHgCKfo5neffdd4mJiWHSpEk5+pJl8uTJ7Nq1i7/97W85thd0jhd2DBdy/hfkwQcfZOfOnaxYsYJJkyZx4sQJ4uPjGTduHC+99NIF7UMESvb879mzJ4888ggDBgygS5cu3hH7oKAgUlNTWblyJTfffDPr1q3jkUce4brrriMtLQ0o/G9DQbG5c+dSvnx5Bg4cmOcYCovhGbHvAvyvmbUD/h/wqXMu0DlXFbgfiDaz+sAIYJ5zLuv60twvWPYXtLuZtcEz6n8Cz1z+C+acexxoAozPfN4AuBt4soAqt5lZCyASWA4s9KW9ElXWc4dK4vFrmaP/888/W9WqVb3zRjMyMuzSSy+1ffv2FVrv2WeftaFDh5qZZ2793//+d29sy5YtFhYWZmaeubV//vOfvbHTp0+bc87S0tK82/bv32+VKlXKMff02LFjFhAQkKNchw4d7Ntvv7Vnn33W6tata/Xr17f69etbuXLl7PLLL7dFixYVejyF1SsslltSUpJVr17ddu3aZe+//75deuml3noVKlSwkJAQe/311/N93Zo1a2Zr1661Y8eOWcWKFXPEhgwZYm+//bb3+dKlS61evXq2YcOGHOX69+/vnR9pZjZ27FjvPGAzs61bt1pYWJh99dVX+fZBJMvFcP5nWbFihbVq1crMPPPZIyIivLGEhASrUKGCdx6wmdmoUaOsX79+lpycnKNfxXGOm5kFBwfb8ePHvc+nTp1q7du3z/F3Kj9Z57iZWaVKlezo0aPeWNbfsAs5/83M7rjjjvPOs7/mmmts3rx5tnz5cqtTp473+C655BKrWbOmPfnkk4XWLy3p6emWlpamx0X0+OmnnywkJMSSkpIsLS3NUlNTLSwszPbu3Vtovb///e82bNgwS0tLs7/85S82depUb2zz5s3WpEmTfOutXLnSGjVqZGlpabZmzRpr3LhxjnhoaGi+ba9YscLatWuX7z6zxx544AFr2LChNW3a1Jo2bWrBwcHWpEkT27hxo/35z3+2K664wnt+ABYWFmabNm0y4GfgoOWcB38UCMcz735RrtgxoAEQApwCAjO3OzwXz4ZbrpwQqAsk5rN9P/nM4wfGAGuB6tm23ZK5//2Zj6TMvt+du35m+SSgVn6x0n5kXXDgV0JDQy02Ns/0q4tSz549GTZsGMOGDWPevHlMmzbNO00lS0JCAoGBgVSqVIl9+/bRs2dPPv74Y6Kionj++ef58ssvWbhwIUFBQUyZMoXly5fz2Wef8eGHH/LUU0+xYsUKKleuzAcffMAzzzzD1q1bvft+6qmn2L17N7NmzcrRZt++fXnooYfo378/Bw4cIDo6mk2bNuWZxxceHs7ChQtp1arVBR9PfvUKi/3000/edp944gm2b9/O/Pnz830tx4wZw8CBA0lLS2Pfvn00adIEgNWrV3PttdeyZ88eqlatSp06dViwYAE9evTg+PHjtG3blg8//JAOHTqwbNkyhg4dyscff0zbtm1ztDFz5kxmzZrFl19+yalTp2jbti1ffPEFLVq0YPv27fTr14/XXnuNa665Jv9fuEg2ZXX+x8XFUaFCBSpVqkRGRgajR4/m5MmTvPvuu2RkZNCkSRPeeustevbsybRp01i7dq33m60HH3yQXbt28dFHH+W5Lia7Cz3HExISOH36tHfFrwULFvDAAw8QExODc47nn3+e999/n8WLF1OjRg3vPgo7x2vUqMGwYcMIDw/nqaeeYs2aNQwePJi9e/finCv0/M8ybNgwoqOjvSOkANu3b/deL7Rnzx66du3K0qVLvdsKq1sWUlJSOHjwIKmpqWXaD8nfkSNHvCu3nT17llOnTuX5PzbrGpCAgADS0tI4cuQIISEhlC9fnoSEBM6dO0dISAjOOU6dOkVycjIhISGAZ3pduXLlyMjI4NixY1SsWJGqVasCcPjwYWrXrk358uVJTk7m6NGjhIaGer9VCAgIwMyIi4sjIyOD2rVr5+hL7lhuBw4coF69egQEBBAUFERYWJh3xS/nHImJiVSuXBnn3CE8K+bcYmabnHPRwCLgCqB15s+RZnbUOdcFz0h5XTNLyVzpZqaZzXTODcFzQW9n59wlQJCZxWe2Nxr4g5l1z97HzJV5BlrmqjvZyt4K9DGzOAqQ2fY0M1uY+Q1DLTP7OTM2GHjePN9ClL2y/qRREo9fy4i+mdmOHTusc+fO1qRJE2vfvr1t2bLFzMz69etna9asMTPPahJNmjSxFi1aWOvWrW3+/Pne+klJSXbXXXdZs2bNrHXr1ta3b1/viGBGRoaNGzfOmjVrZpGRkfa73/3O1q1b562bkZFh9evXt2+++SZPv/bs2WM9evSwVq1aWZs2bezDDz/Mt//169fPsXpGQcdzvnqFxUaMGGHNmjWzRo0a2dChQwsc1evRo4d31Z2kpCTr2rWrtWzZ0iIjI61Lly62ZMkSb9mvv/7a2rVrZ5GRkdaiRQv7xz/+4Y01btzYQkJCrE2bNt7Hpk2bzMwsLS3N7r//fmvYsKE1bNgwx4hfnz59rHr16jnqffHFF/n2VcSs7M7/FStWWOvWrS0yMtIiIiJs6NChduLECe9+V6xYYZGRkdakSRPr2bOnxcbGmpnZ999/b4A1b97c+x7/wx/+kO+xXeg5fvDgQevQoYO1atXKIiMjrXfv3rZ+/Xoz86yoBVjDhg297XXs2NF77IWd40eOHLGrr77aGjdubBEREbZ06VJvrLDz/7333rMrrrjCKlWqZNWrV7crrrjC+7oNGjTIIiIirE2bNta+ffsc3+5ldyHfBpSGXbt22ZEjRyw1NbXMR7D1yPs4c+aMbd261TZu3Ghbtmyx06dPW1pamu3cudMSEhIsLS3NEhMTbePGjbZp0ybbvHmzHT9+3Fs/JSXF9u7d643t2LHDzp49m2OEf9OmTbZx40aLiYnJ8T5ISEiwrVu32ubNm23r1q0WHx9vaWlpdurUKdu8ebO37u7du73fOhQWy/1Ys2aNJScnW2pqqh05csR27drlfV+Sd9WdaGA1sCnz3272fyPjfwG2ARvxjLL3zhZrhmf1mx8zYy0ztzcE1mfubzPwMdlG+oGXMttNwzNKvztzeyie6UB7gA2Zj/9aPnkm2VbZAS7JbH9zZj+XAG3yq1cWD43oi4iI+JmMjAx27txJkyZNvBd8ipSFtLQ0du3aRbNmzfKsZuWcO2Rm518OTIpMF+OKiIj4maxBvNwXfIqUtqz3oD8OLP8aKNEXEREREfFDZfJ9XkG3H86n3BPA8Myns81sQil1UURExK+Ej/usxPa9f8qAC+tDeDjBwcFs2bLFO6UoOjqaadOmeW/S5qu1a9cyffp03n//fZ/qDRs2jMWLF3svNK1evTqvv/46zZs3L1I/isvixYuZOHEihw8fpmLFioSEhDBx4kS6detW6EXuRfXqq69y7tw5Hn74YQBGjBjBqlWraNq0KcOHD2f58uVMnTq12NqT0lVWE/fmAc8B3xdUwDnXHbgZz5qkacB/nHPfm9mXpdNFERERKW7Jycm89dZb3HvvvcWyv+joaJ+T/Czjxo3zro703HPPMWHCBObOnVss/SqKxYsXc9tttzF//ny6du0KwI8//simTZtKrM377rvP+/PPP//M3LlziY+P986nHzRokM/7zFrxR8pemUzdsQJuP5zLn/Asm3TGzJLx3BHt5pLvnYiIiJSUiRMn8swzz3D27Nk8sdmzZ9OpUyfatm1LVFQUixYtAmDWrFk57opsZjRo0IBNmzaxdOlSoqOjvbEvv/ySK6+8kvbt29OpUyeWLVt23j6ZGfHx8d4lXNPS0rjmmmuIjo6mZcuW3Hrrrd7+tm7dmpUrV3rrvvbaa/zpT38CPEtm3njjjXTs2JHIyEiefNJzf6WMjAxGjRpF8+bNadOmDe3btycpKSnf12bChAneJB+gadOm+d5x+fnnn6dDhw60bduWjh078t//em7Yeu7cOf70pz8RERFBmzZt6Nu3LwC7du3id7/7HW3atKF169Y88cQTgGeZ7TFjxhAfH0+vXr04e/Ys7dq1Y8qUKcycOTNH2++99x6dOnWiXbt29OjRgy1bPBMyZs6cybXXXsvtt99OdHQ0q1evPu9rLqXjYr4UPwz4Ltvz/XhunpBH5rqno7Oe53f7ZRERESl77dq1o3v37kyfPp3x48fniF1zzTXcfPPNOOfYv38/Xbt25cCBAwwePJiHHnqII0eOcNlll7F06VJq1qxJZGQkS5cu9dbfu3cvEydO5IsvvqBq1ars3r2bHj16sH//fu8dZbObMmUKb775JseOHaNcuXLeDwXlypVj9uzZ1KpVCzPPnaBffvllxowZw4MPPshLL71Ely5dAHjppZe8d0W+4447GD9+PN27dyctLY2BAweyYMECwsPDWbJkCdu2bSMgIIBTp05515XP7ocffuBf//rXBb2Ot912G6NHe1KfVatWMWLECLZs2cIXX3xBXFwc27ZtA+DkyZMAvPjiiwwYMIDHH388x/Ys1atXZ9GiRURHR7NhwwbAk8Bn+c9//sMHH3zAsmXLqFChAsuXL+fWW29l48aNAHz//fesX7/ee28LuThczIk+5Ly9cYFLB5jZ88DzWc9DQ0N1abeIiMhFatKkSXTq1CnHtBGAffv2ceuttxIbG0tgYCDHjx/nwIEDNG7cmMGDBzNr1izGjBnDjBkzGD58eJ79fvHFF+zevZvu3XPcG4mYmBgaNmyYp3z2qTtvv/02Q4YMYe3atZgZ06dP57PPPiMtLY1Tp0559zl06FD++te/cvToUbZv345zjm7dunHmzBm++eYbfv75Z+/+T58+zY4dO7jqqqtITU3lzjvvpFevXgwYMCDPUpO+Wr9+PX/72984ceIEgYGBbNu2jZSUFNq0acOOHTu4//776dGjB/379wege/fujB07ljNnztCjRw/69OnjU3sff/wxGzdupFOnTt5tx44dIyUlBYArr7xSSf5F6GJedecgnlsgZ6mfuU1ERER+xRo2bMjNN9/MpEmTcmy/6aabuO+++9iyZQsbNmygcuXK3ikuw4cPZ+bMmSQkJLBw4UJuueWWPPs1M6699lo2bNjgfRw6dCjfJD+3m266iR9++IFjx44xe/ZsvvvuO5YtW8bmzZsZM2aMtx8VK1bkjjvu4M033+TFF1/0flDIyMjAOceaNWu8be/evZv/+Z//oVq1amzdupVbbrmFHTt2EBkZye7du/P0oX379jmmBRUkJSWFwYMH8/zzz7NlyxaWLVuGmZGSkkLDhg3Ztm0b1157Lf/5z39o1aoVcXFxDB48mP/85z80a9aMF198kYEDB563ndyv7Z133pnjtT18+LD3m4nKlSv7tD8pHRdzoj8XuMM5d4lzrgJwJ/BBGfdJREREisGECROYNWsWhw8f9m6Li4sjPDwc8MzLj4uL88Y6d+5MRkYGjz76KFdffTU1a9bMs8++ffvyxRdfeOeOAxc8X3zJkiXUqlWLWrVqERcXR61atahSpQqJiYk5prAAjBw5kldeeYXvvvuOW2+9FYAqVarQrVs3pkyZ4i13+PBhYmNjOXbsGGfOnKFv375MnjyZ8PBw79Sa3K/JpEmTWLVqlXfb9u3b+d///d8c5ZKSkkhNTaVevXoAvPDCC95YbGwszjkGDRrEtGnTMDNiYmLYtWsXISEh3H777Tz33HM52rgQ1113He+++y4xMTGA54PN2rVrfdqHlL6yWl7zJeD3wGXAYufcaTNr7JxbBDxpZmvNbKlzbg6eWwoDfGBmX5RFf8+nJJcsy+5Cly8TkdKj819+LS6291CdOnV48MEHvResAvzzn//k+uuv54orrqBLly6EhYXlqDN8+HAeffRRPv/883z32aRJE2bNmsVdd93FuXPnSElJoV27dgWuypM1R9/MqFChAvPnzycgIIDbb7+djz/+mIiICK644gq6devGoUOHvPVCQ0OJioqiRt367D6ZAic901fGT32ZaU+Pp0nzCAAqXVKZJ579f2SkZzDxsb+QlppKRkYGUdGduKJ1FzbFxufoz2URHZnw3Av8+cGHOHH0KBWCg6lVJ4Rpz+b85qNq1ao8/fTTdOzYkbCwsBwr42zevJlx48ZhZmRkZHDbbbcRGRnJ5MmTef/99ylfvjxmxquvvnq+X1EO3bt3Z/Lkyfz+978nPT2d1NRUBgwYkONCaLn4OH+8U1loaKjFxp5vUZ/io//oRX67dP7LxSg9PZ0ff/yRpk2bapnDEnD69GmaN2/O63M+IzSsfom3FxlavcTbKCmFvRedc4fMLLSMuvabcDFP3RERERG5qLz66qs0b96c+++/v1SSfJFf4mJfdUdERETkonHfffd5VwvKPfVG5GKjEX0RERERET+kRF9ERERExA8p0RcRERER8UNK9EVERERE/JAuxhUREfkteKpaCe77VMntW0SKTCP6IiIiUirCw8Np3rw5UVFRtGjRgltuuYUzZ84UezvOOU6fPg1A//792bNnT7G3IfJroERfRERESs28efPYsGED27ZtIyEhgZkzZ5Zoe4sWLaJRo0Yl2obIxUqJvoiIiJS65ORkzpw5Q40aNdi8eTPdunWjXbt2RERE8Oyzz3rLvfnmm0RERBAVFUXr1q3573//C8CuXbsYMGAAHTp0oE2bNrz88sv5thMeHs6WLVsA6NmzJ4899hjdunWjUaNG3vXwARITE7n77rvp2LEjkZGR3HfffaSmppbgKyBS8jRHX0RERErNkCFDCA4OZt++fbRv354bb7yRc+fOsXjxYipUqMC5c+fo2rUrV199NdHR0TzyyCNs376dyy+/nNTUVJKTk0lPT+eWW27hvffeo3nz5pw9e5bOnTvTuXNn2rVrV2j7e/bsYenSpaSkpBAREcHKlSvp0qULjzzyCN27d+eNN97AzLj77rt58cUXefjhh0vplREpfhrRFxERkVKTNXXnxIkTNGjQgMcee4xz585x11130bp1azp37syBAwfYsGEDAFdddRW33347//znP9m3bx+VK1dm586dbN26lZtuuomoqCi6du1KYmIi27ZtO2/7N910E+XKlaNixYpERUV55+9/9NFHTJ06laioKNq2bcvy5cvZtWtXib4WIiVNI/oiIiJS6gIDAxk8eDBjx47l1KlTXHrppaxfv57AwED++Mc/kpSUBMCHH37IDz/8wNKlS+nfvz+TJk2idevW1K5d2/thwBfBwcHen8uVK0daWhoAZsZHH31Ew4YNi+cARS4CGtEXERGRMvHNN9/QrFkz4uLiCA0NJTAwkJ07d/L1118DkJaWxp49e4iOjmbMmDEMGTKE1atX06xZMypVqsS7777r3dfu3bs5efJkkfsyaNAgpkyZ4k384+Li2L179y87QJEyphF9ERGR3/Mg4pkAACAASURBVIKLZK37rDn6qamphIeH8+qrr3L8+HFuu+023n//fcLDw7nqqqsASE9PZ/jw4cTFxREYGEidOnWYMWMGgYGBfPrppzz88MNMmzaN9PR06tSpw/vvv1/kfv3jH//gscceIyoqioCAAIKCgvj73/9O48aNi+vQRUqdEn0REREpFfv37893e7169bwr4+S2fPnyfLc3adKEhQsX5hszs3zbXLp0aY5y8+bN8/5cpUqVAlfuEfm10tQdERERERE/pERfRERERMQPKdEXERHxM845IOcUFpGykPUezHpPSunSHH0RERE/k3Ux6YkTJ6hVq5aSrBJillEq7aSnp5dKO8XNzDhx4gRBQUEEBGhsuSwo0RcREfFDYWFhHDx48BctOSmFOxp3rlTa+fF0xVJppyQEBQURFhZW1t34zVKiLyIi4ofKly9P48aNycjI0BSeEvKHJ78slXa2PX1NqbRT3JxzGskvY0r0RURE/JgSrZKTnF46H6DKlStXKu2I/9HZLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih8ok0XfONXHOrXDO/eicW+2ci8inTLBzbqZzbrNzbotz7hPnXO2y6K+IiIiIyK9NWY3ovwa8bmZNgeeAt/Ipcy9QGYg0s1bAz8CjpddFEREREZFfr1JP9J1zIUA7YFbmpvlAA+dceD7FKwFBzrlAPEl/bGn0UURERETk164sRvTrAYfNLA3AzAw4CITlKvcakAAcxTOaXw14sRT7KSIiIiLyq1VWU3cs13OXT5k+meUuA+oC8cCT+e3MOTfaOReb9Th9+nSxdlZERERE5NemLBL9GCA0czoOzjmHZ5T/YK5y9wELzCzJzFKA94Fe+e3QzJ43s9CsR+XKlUuw+yIiIiIiF79ST/TN7CiwHhiauWkwsN/M9ucquhe4xmUCBgJbSq2jIiIiIiK/YmU1dede4F7n3I/AOGAEgHNukXMuOrPMU3jm5W/Fk+DXBiaUfldFRERERH59AsuiUTPbCXTJZ3v/bD+fBIaUZr9ERERERPyF7owrIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB9Soi8iIiIi4oeU6IuIiIiI+CEl+iIiIiIifkiJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiIiPghJfoiIiIiIn5Iib6IiIiIiB8qcqLvnAt1znV1zl1SnB0SEREREZFfzudE3zk3wjkXCxwAlgPNMrfPd86NKub+iYiIiIhIEfiU6Gcm8q8Cc4HfAy5beDnwp+LrmoiIiIiIFFWgj+X/Akw2s78658rliu0kc3RfRERERETKlq9Td8KA7wqIJQNVfll3RERERESkOPia6McCrQuItQX2/rLuiIiIiIhIcfA10X8f+Ktzrj//Nz/fnHNtgbHAO8XZORERERERKRpf5+g/A7QCFgKnMrd9DdQAPgSmFV/XRERERESkqHxK9M0sFfijc64ncDVQBzgBfGVm3xZ/90REREREpCh8HdEHwMyWAkuL2qhzrgmeaT61gXhgmJlty6dcDzzfElQCygHDzWxlUdsVEREREfmt8CnRd86FFRLOAE6ZWeIF7Oo14HUzm+mcGwK8BXTJ1dbleD4M9DOz7c65YCDYl/6KiIiIiPxW+Tqivx+wwgo45/YB083spQLiIUA7oG/mpvnAi865cDPbn63o/cAsM9sOYGZJQJKP/RURERER+U3yddWdEXiW2Pwv8BgwHBgHrAYOAY8A64F/Oef+XMA+6gGHzSwNwMwMOIhnjf7sIoCKzrnFzrkNzrkXnHOVfOyviIiIiMhvkq+JfjSw1My6mtk0M3vXzKaaWRc8c/abmtkNwNvAqEL2k/tbAZdPmSCgJ3BDZrvVgKfy25lzbrRzLjbrcfr0aV+OSURERETE7/ia6P8Jz1r6+ZmVGQf4CGhUQLkYINQ5FwjgnHN4RvkP5ip3APjMzOIyR/8/ADrmt0Mze97MQrMelStXvuADEhERERHxR74m+kFAkwJiTfGsjAOQTAHz6c3sKJ7pPUMzNw0G9ueanw8wG+jlnKuQ+fxaYKOP/RURERER+U3yNdH/EJjsnBvunKsJ4Jyr6Zy7E5icGQdoA+wqZD/3Avc6537EM8d/ROa+FjnnogHMbAXwKbDBObcZz5r9T/rYXxERERGR3yRfV915ELgEeBN40zmXwf99WJiXGQfYA/xPQTsxs53kWk4zc3v/XM+fA57zsY8iIiIiIr95vt4ZNxG40TnXHM98+brAT8BqM9uRrdxHxdpLERERERHxSVHvjLsD2HHegiIiIiIiUiaKlOg75y7Bs6pOnjvVmtnqX9opERERERH5ZXxK9DNXwHkNuIX/W2Ent4K2i4iIiIhIKfF11Z0ngT7AMDw3uRoF3Al8C+wDrivOzomIiIiISNH4mugPAZ4G/p35fLWZvWNmfYD/AgOKs3MiIiIiIlI0vib69YDtZpYOpADVs8XeBW4oro6JiIiIiEjR+ZroHwWqZf58EOicLXYFnjvnioiIiIhIGfN11Z3v8ayfvxCYDTzhnLsUSAbuAr4q3u6JiIiIiEhR+JroT8Qzcg/wLFADGAxUAr4AHii+romIiIiISFH5emfcXcCuzJ+Tgb9kPkRERERE5CLi0xx959w3zrnmBcSaOue+KZ5uiYiIiIjIL+Hrxbg9gaoFxKoC3X9Rb0REREREpFj4mugDWAHbe+BZlUdERERERMrYeefoO+fGA+MznxqwzDmXO9kPwvOh4eXi7Z6IiIiIiBTFhVyM+z3wd8ABTwKzgNhcZVKA7cCnxdo7EREREREpkvMm+mb2HfAdQOZI/htmdrikOyYiIiIiIkXn6/KaE0uqIyIiIiIiUnx8vWEWzrnewI1APSA4V9jMrHdxdExERERERIrO13X0HwK+BgYAFYDkXI+U4u6giIiIiIj4ztcR/QeB14H7zSyjBPojIiIiIiLFwNd19EOAOUryRUREREQubr4m+ouBdiXRERERERERKT6+Tt2ZCMxyziUDXwFxuQuYme6OKyIiIiJSxnxN9H/I/PefeO6Sm59yRe+OiIiIiIgUB18T/TspOMEXEREREZGLhK83zJpZQv0QEREREZFi5PMNswCcc9WBVkAY8LmZxTnngswstVh7JyIiIiIiReLrDbMCnHPPALHAMuA9oEFm+FPn3IRi7p+IiIiIiBSBr8trPgk8DIwHIgGXLfYJcF0x9UtERERERH6BolyMO8HM/umcy726zm6gcfF0S0REREREfomi3Bl3YwExA4J/WXdERERERKQ4+Jro7wV+V0CsK7D9l3VHRERERESKg69Td14HJjvnTgBzM7dVcM5dD4wGHinOzomIiIiISNH4uo7+P5xzYcC/gBcyN38PZAD/MLM3i7l/IiIiIiJSBD6vo29mo51zLwB9gDrACWCxme0p7s6JiIiIiEjRFOmGWWa2D3ijmPsiIiIiIiLFxNcbZv3JOfdoAbGxzrkbiqdbIiIiIiLyS/i66s44IKWAWFJmXEREREREypiviX4TYFMBsS2ZcRERERERKWO+JvqpQK0CYiF4bpolIiIiIiJlzNdEfwXwkHMuRz3nXDngQWBlcXVMRERERESKztdVd54ClgFbnHPvAoeBK4DbgXCge3F2TkREREREisbXG2atcc5dBUwFJuH5RiADz0j/nWa2pvi7KCIiIiIivirKDbNWAlc65yoCNYA4MztX7D0TEREREZEi83WOfnZJmf+mFkdHRERERESk+Pic6Dvn+jjnVgDngINAZOb2V51zNxdz/0REREREpAh8vTPuDcCXwM/AI7nqHwCGF1/XRERERESkqHwd0X8SeNnMrgdezRXbArQull6JiIiIiMgvUpQ7435UQCwRz8W5IiIiIiJSxnxN9I8BDQqItQBif1l3RERERESkOPia6C8A/uqcyz5Fx5xzV+CZsz+n2HomIiIiIiJF5muiPx44BKzLfAC8A+wAjgDPFF/XRERERESkqHxK9M0sEbgSuBvPxbeLga3Ag0Av3Tjr4rRr1y66du1K06ZN6dixI9u2bctTZsGCBURGRhIVFUXLli0ZP348ZgbAmTNnGD58OK1bt6ZZs2aMGzfOGzMzxo4dS8uWLYmMjKRXr17s3r3bu99p06bRqlUroqKi6Ny5M2vWeG6evG3bNqKioryP8PBwatas6a2XnJzMqFGjaNKkCS1btmTo0KF5+jxx4kScc2zZssWnY82vXmGxL774gujoaCIjI+ncuTMbN27MU++dd97BOcfChQu924YPH+59TTt06MCSJUu8sbVr19KlSxfatm1LixYteO6557yxxx9/nBYtWtCmTRs6duzIN998k6c9ERERkfO54DvjOueCgC7AbjObCcwsoT5JMbv33nu55557GDZsGPPmzWPEiBGsXLkyR5k+ffrw+9//noCAAFJSUrjyyivp1KkTgwYNYvLkyQBs2rSJtLQ0Bg4cyLx587jhhhv45JNPWLZsGRs2bCAoKIhJkybx+OOPM2fOHDZu3MgLL7zA1q1bqVy5MrNmzWLkyJGsXr2aiIgINmzY4G1/1KhROOe8z8eNG0dAQAA//vgjzjl++umnHP1dt24dq1atIiwszKdjLaheQbG4uDiGDh3K8uXLadGiBd999x233nprjg8CsbGxvPbaa3Tu3DnH/qZPn0716tUB2LBhA3369OHYsWM457j77ruZOHEigwYN4uTJkzRv3pyBAwcSERFBt27dmDBhAhUrVmTjxo307NmTn376ieDg4MJ/0SIiIiLZ+DKinwZ8jeeiW/mVOHr0KOvWrfOOiA8ePJh9+/axf//+HOWqVKlCQIDn7ZCUlERycrL3+caNG+nXrx/OOYKCgujbty/vvfeet25ycjJJSUmYGQkJCYSGhnpjqampnDlzBoD4+Pgcsez1Z8+ezYgRIwDPNwgzZsxg8uTJ3uS/bt26OcqPHDmSl19+OceHg/Mda0H1Covt2bOHkJAQWrTwvO179OjBgQMHWLdunbfMPffcw/Tp06lQoUKOfWYl+VnHnrvN+Ph47/GWL1/e+41Gv379qFixIgCtW7cmPT2d48eP53ndRERERApzwYm+eeZq7AZqlVx3pLjFxMRw+eWXExjo+fLGOUdYWBgHDx7MU3bFihVERkYSEhJC7969GTBgAAAdOnRgzpw5pKSkkJiYyIIFC7zJ83XXXUevXr247LLLqFu3LkuWLOHpp58GoE2bNowePZoGDRoQGhrK9OnTeeGFF/K0++GHH9KgQQOioqIAT3Jdq1YtJk2aRHR0NN26dcsx7eXJJ59k6NChNGiQcwGo8x1rQfUKizVp0oRjx46xatUqwDPF6fTp097jf+WVV2jZsiWdOnXK9/UfN24cjRo14o9//CNz5871JvszZsxgwoQJhIWF0bRpU5599lkuu+yyPPVnzJhBo0aN8v2AJCIiIlIYXy/GfQaY4JzLO+9BLlq5R5Kz5tfn1rVrVzZt2kRMTAxr1qxh+fLlADz22GPUq1ePjh07MmjQILp27UpQUBDgme6yY8cODh06xOHDh+nduzejRo0C4MCBA3zyySfs2bOH2NhYHn74YW699dY87b799tve0XzwfAuwd+9eIiIiWLt2LS+++CI33XQTx44dY+XKlaxZs4b777/fp2MtrF5hsWrVqjF//nzGjRtH+/btWbp0KREREQQFBbFv3z7eeOMN7web/EyZMoU9e/YwZ84cxo4dS0pKCgBTp05l6tSpHDx4kK1btzJ+/Hh27tyZo+6SJUuYOHEiH3zwQYH7FxERESmIr4n+HUBtYJdzbp1z7nPn3KJsj89KoI/yC9SrV4/Y2FjS0tIAT+IbExOT7xz1LHXq1GHAgAHMnTsXgODgYKZPn86GDRv49ttvqVmzJhEREQDMnDmTXr16Ub16dQICArjjjjv49ttvAZg7dy6tWrXyTrsZPnw4y5YtIz093dvWgQMHWLFiBbfccot3W/369QkICPB+KGjTpg0NGjRg69atfPfdd+zYsYMGDRoQHh5ObGws11xzDZ9//nmhx1pYvcJiAN27d2fp0qX88MMPPPfccxw+fJgWLVqwcuVK78/h4eGsWrWKESNG8MYbb+R5Tfv06UNiYiKbN2/m+PHjLFiwgBtvvBGAhg0b0qlTJ1asWOEt/9133zF8+HA+/fRTmjVr5suvXERERATwPdEPxrOU5grgVObzitkelYq1d/KLhYSE0LZtW2bNmgXA/PnzCQ8PJzw8PEe5nTt3kpGRAUBiYiILFy4kMjISgISEBM6ePQvAvn37eOWVV3jkkUcAT5K6ZMkSUlNTAfj0009p1aqVN/b9999z+vRpb6xFixaUK1fO2+6MGTO4/vrrc8xnr127Nr179+bLL78EPB8G9u3b513x5/Dhw+zfv5/9+/cTGhrKl19+Sb9+/Qo91sLqFRYDclwI/Mwzz3DVVVfRuHFjbrnlFo4cOeKt17lzZ9566y3uvvtu0tLS2LVrl7fe6tWrOXr0KA0bNqRGjRoEBwfz3XffAXD8+HFWrVrlfd2WLVvGbbfdxscff0ybNm2K8FsXERER8WHVHQAz61VSHZGS89prrzFs2DAmT55M1apVeeeddwDo378/Tz/9NNHR0cydO5fZs2cTFBREeno6Q4YM4a677gJg79693HjjjQQGBhIYGMj06dO98+lHjhzJ9u3bad26NeXLl6du3bq89tprAFx//fWsWbOG6OhoKlSoQJUqVbxJOHhG3GfOnMmMGTPy9PnVV1/lzjvv5LHHHqNcuXK8/vrrOS7I9fVYf4kJEybw/fffk5aWRpcuXXjrrbfOWyc9PZ1hw4Zx6tQpypUrxyWXXMK8efOoUaMGAHPmzGH06NGkpaWRmprKmDFj6NChAwAjRowgOTmZ4cOHe/f33nvv0bp163zbEhEREcmPK2i+doEVnGuO58ZZnYHLgZ+AlcDfzGzHBe6jCZ4bbdUG4oFhZpZ3wXNP2Tp41uxfbmZDLmT/oaGhFhsbeyFFi0X4uNKZsbR/yoBSaUdELpzOf5HfLp3/v4xz7pCZabWJEuTT1B3n3DXARqAn8BXwd+DLzOcbM+MX4jXgdTNrCjwHFDZE+jKwyJd+ioiIiIj81vk6R38a8C3Q0MxGmtnTZjYSaAQszYwXyjkXArQDsuZwzAcaOOfC8yl7K/Az8J2P/RQRERER+U3zNdFvDPzLzFKzbzSzFOCfmfHzqQccNrO0zLoGHARyLAPjnLscGA2MO98OnXOjnXOxWY+siz9FRERERH6rfLoYF8+KOyEFxC4FfrzA/eS+MMDlU+YN4FEzO517bfQ8OzN7Hng+63loaKhvFx78WjxVrRTaOFXybYiIiIhIifM10X8EeMs5d9jMvsramDk3/0ngrgvYRwwQ6pwLNLM058ni6+EZ1c+uS2ZbAJWBis65L83sQq8DEBERERH5zfI10X8JqAp87pw7AxwD6gCX4Fk954Vso+9mZi1z78DMjjrn1gNDgZnAYGC/me3PVa5m1s/OuWHAwAtddUdERERE5LfO10T/v+SddlMU9wIznXOPAwl47riLc24R8KSZrS2GNkREREREfrN8vWHWsOJo1Mx24pmak3t7/wLKz8Qz+i8iIiIiIhfA11V3RERERETkV0CJvoiIiIiIH1KiLyIiIiLih5Toi4iIiIj4ISX6IiIiIiJ+SIm+iIiIiIgfUqIvIiIiIuKHlOiLiIiI+Kldu3bRtWtXmjZtSseOHdm2bVueMgsWLCAyMpKoqChatmzJ+PHjMbMSjwEhzrmtzrm/OeccgHPueufcJufcBsUuLFYYJfoiIiIifuree+/lnnvu4ccff+TRRx9lxIgRecr06dOHDRs2sGHDBtavX8/XX3/Np59+WuIx4CjQFrgauC6zO4uBKDOLUuyCYwVSoi8iIiLih44ePcq6desYOnQoAIMHD2bfvn3s378/R7kqVaoQEOBJCZOSkkhOTvY+L+kYEAxUADIAzCzRzDIUu/BYYZToi4iIiPihmJgYLr/8cgIDAwFwzhEWFsbBgwfzlF2xYgWRkZGEhITQu3dvBgwYUOIxIATPqP4S4LOsmHOuq3Nuk2IXHiuIEn0RERERP5V7GnfWPPncunbtyqZNm4iJiWHNmjUsX768xGN4EtZ6QAegW7Y+rjCzSMUuPFYQJfoiIiIifqhevXrExsaSlpYGeJL8mJgYwsLCCqxTp04dBgwYwNy5c0slZmbH8IxM36DYL4/lpkRfRERExA+FhITQtm1bZs2aBcD8+fMJDw8nPDw8R7mdO3eSkeGZ7p2YmMjChQuzVsUp8ZhzrgowENiU+byZcy5AsQuPFSbwfAVERERE5NfptddeY9iwYUyePJmqVavyzjvvANC/f3+efvppoqOjmTt3LrNnzyYoKIj09HSGDBnCXXfdBVCiMTxz9FcC84A3M7t8A3CLcy4VKKfYBcUK5Aqaq/VrFhoaarGxsaXWXvi4814LUSz2B99S8o08dark2xDxI6V2/k8ZcP5CIlKqdP7/Ms65Q2YWWtb98GeauiMiIiIi4oeU6IuIiIiI+CHN0RcRERG5mD1VrZTa0fRdf6MRfRERERERP6REX0RERETEDynRFxERERHxQ0r0RURERET8kBJ9ERERERE/pERfRERERMQPKdEXEREREfFDSvRFRERERPyQEn0RERERET+kRF9ERERExA8p0RcRERER8UNK9EVERERE/JASfRERERERP6REX0RERETEDynRFxERERHxQ0r0RURERET8kBJ9ERERERE/pERfRERE5P+3d+dRUlZ3/sffX9nUYIgmEjEtARORRkAaARUNOsgMI25jxMwvEaOAWyQTFY1BiRwxi0ZMIppxNBNciaPg8kOcuBuQFjy0ghh1lMjYQA8SZMQFDVtz5496uu1uGuiwdMvT79c5dbrq3lvPvU/Veao+dfvWU1IOGfQlSZKkHDLoS5IkSTlk0JckSZJyyKAvSZIk5ZBBX5IkScohg74kSZKUQwZ9SZIkKYcM+pIkSVIOGfQlSZKkHDLoS5IkSTlk0JckSZJyyKAvSZIk5ZBBX5IkScohg74kSZKUQwZ9SZIkKYcM+pIkSVIOGfQlSZKkHDLoS5IkSTlk0JckSZJyyKAvSZIk5ZBBX5IkScohg74kSZKUQwZ9SZIkKYcM+pIkSVIONUnQj4iDImJ2RCyMiLkR0a2eNv8cEfMj4tWI+FNE/EtTjFWSJEnaFTXVjP5twG9TSl2A64FJ9bSpAI5PKXUHjgYuioijGnGMkiRJ0i6r0YN+RLQHegOTs6IHgc4R0almu5TS8yml5dn1D4A3gM6NN1JJkiRp19UUM/oHAMtSShsAUkoJWAJ03NwdsqU9RwLPbqZ+dERUVF1Wr169E4YtSZIk7TqaaulOqnM7NtcwIoqAacAFKaVl9W4spV+llIqqLm3btt2BQ5UkSZJ2PU0R9JcCRRHREiAigsIs/5K6DSNif+Bp4KcppamNOkpJkiRpF9boQT+ltAKYDwzLik4DylNK5TXbRUQH4BngFymluxp1kJIkSdIurqmW7pwPnB8RC4ExwEiAiPhDRPTJ2lxDYd3+RRHxcnYZ3jTDlSRJknYtLZui05TSmxS+XFu3fEiN6+cC5zbmuCRJkqS88JdxJUmSpBwy6EuSJEk5ZNCXJEmScsigL0mSJOWQQV+SJEnKIYO+JEmSlEMGfUmSJCmHDPqSJElSDhn0JUmSpBwy6EtSjv35z3+mf//+dOnShX79+vH6669v0qasrIz+/fuz5557MnTo0Fp1Dz/8MD179qRXr14ccsghjB07lpQSAPfffz8lJSV0796dHj16cPPNN1ffb+PGjVx22WV0796drl27MnLkSNatWwdAeXk5LVu2pFevXtWXRYsWVd/3hhtuoHv37vTq1YsjjjiCsrKy6rrJkydXj6ekpITHHnusQfv6gx/8gE6dOhERvPrqq/U+VuPHj9+k/vHHH6dPnz707NmTI444ggULFlTXvfjiixx55JGUlJRQXFzM9ddfX113++2306NHD1q2bMlvfvObWv1ceeWVFBcXc+ihh9KvXz+effbZWuOs+bjsvvvusZ9CfwAAE9VJREFU3HTTTVt9LrZUJ6n5atnUA5Ak7Tznn38+5513HmeffTYPPPAAI0eOZM6cObXadOjQgRtvvJH58+fz1FNP1aobNGgQp5xyCrvtthvr1q3j6KOP5vDDD+fkk0+mqKiIxx57jP32248PPviAww47jN69e3PUUUcxadIkXnnlFebNm0erVq0455xzmDhxIj/84Q8B+MIXvsDLL7+8yXgXLFjAzTffzGuvvUbbtm2ZPHkyo0aNYu7cubz33ntceOGFvPnmm3To0IHS0lK++c1vsmLFiq3u69ChQ7n88ss5+uij632c5s2bxwsvvEDHjh2ry1atWsWwYcOYNWsWxcXFzJw5kzPOOKP6g8C5557L+PHjOfnkk3nvvffo2rUrJ554It26deOwww5jypQpXHvttZv09Y1vfIOrrrqKPfbYgwULFnDsscfyzjvv1Ar1AMuXL6dz585861vf2upzsaU6Sc2XM/qSlFMrVqxg3rx5DBs2DIDTTjuNt99+m/Ly8lrtioqK6NevH23atNlkG3vttRe77VZ4q1izZg1r166tvn3UUUex3377AdCuXTu6du3K22+/DRQC+6BBg2jdujURwZAhQ7jnnnsaNO7169fz8ccfA/D+++9TVFQEFP5LkFJi9erVm9RtbV8HDBhQ3bautWvXMmrUKG655RYiorp80aJFtG/fnuLiYgCOOeYYFi9ezLx586rbvP/++wB8/PHHtG7dmn322QeAQw89lOLi4urHqqbjjz+ePfbYA4AePXpQWVnJypUrN2l39913M3jw4OrHeEvPxZbqJDVfvgpIUk4tXbqU/fffn5YtC/+8jQg6duzIkiVL/qbtzJ49m549e9K+fXuOO+44TjjhhE3avP7668yZM4eBAwcC0LdvX6ZNm8ZHH33EunXruO+++2p9wPjwww/p27cvvXv35pprrqGyshIoBOTRo0fTuXNnioqK+PWvf129JOhLX/oSt956K7179+arX/0qI0aM4M4779zufR03bhzDhg2jc+fOtcoPOugg3n33XV544QWgsDxm9erV1ftxxx13cNVVV9GxY0e6dOnCtddeWx3KG+qOO+7ga1/7Wr0fQm6//XZGjhxZq2xLz0VDnidJzYtBX5JyrOYMNbBN67b79+/PK6+8wtKlSykrK2PWrFm16isqKjjllFO49dZb2X///QH47ne/y+DBgxkwYAADBw7kkEMOoVWrVkBhqVBFRQVlZWU8/fTTzJo1i1/+8pcALF68mEceeYRFixZRUVHBJZdcwhlnnAEUPhzccsstvPjiiyxevJhJkyYxdOhQNmzYsM37OmfOHMrKyrjwwgs3qWvXrh0PPvggY8aM4bDDDmPGjBl069atej8mTJjAhAkTWLJkCa+99hpjx47lzTffbPDj+swzzzB+/Hjuu+++Teqef/55PvzwQ4YMGVKrfEvPxdaeJ0nNj0FfknLqgAMOoKKiojoIp5RYunRprXXof4t9992XE044galTp1aXLVu2jEGDBvHjH/+Y008/vbo8Ihg3bhzz58+ntLSUrl270q1bNwDatGlD+/btAdhnn30YMWJEdSidOnUq3bt3p0OHDgAMHz6c5557jsrKSp588knatWvHwQcfDMBJJ53EqlWrWLp06Tbv68yZM3njjTfo3LkznTp1oqKigsGDB1d/yXfAgAHMmDGDl156ieuvv55ly5ZRXFzMypUrefjhh6vXzx944IEcfvjhzJ49u0GP5cyZMxk+fDjTp0+v3p+aJk2axFlnnUWLFi3qvX99z0VD6iQ1LwZ9Scqp9u3bU1JSwuTJkwF48MEH6dSpE506dWrwNt588002btwIwEcffcSjjz5Kz549AXjnnXc47rjj+NGPfsRZZ51V635r1qypXr++cuVKrrvuOi6//HKgsJ5+/fr1QGF9/EMPPURJSQlQCMylpaXV6/CnT59OcXExLVq04MADD2TevHnVX76dM2cOGzdu5Ctf+co27+uYMWNYtmwZ5eXllJeXU1RUxBNPPMHxxx9fvY9VfvKTnzBw4EC+/vWvs/fee7P77rszc+bM6n184YUX6N69+1Yf0+eee44zzzyTadOmceihh25Sv3r1ah544AFGjBhRq3xLz8WW6iQ1X551R5Jy7LbbbuPss8/m5z//OZ///Oe56667ABgyZAjXXHMNffr0YdGiRRxzzDF88sknrFmzhqKiIq688kouvPBCpk6dyr333kurVq2orKxk6NChnHPOOUBhbfuSJUuYOHEiEydOBOCiiy5i+PDhfPDBBxxzzDG0aNGCyspKLr74Yk466SQASktLGTduHC1atGDDhg0MHDiQsWPHAnDqqadSVlZGnz59aNOmDXvttVd1eO/duzdXXHEFxx57LK1ataJVq1ZMmTKF1q1bb3FfAUaNGsW0adNYvnw5gwYNom3btrz11ltbffyuuuoqSktL2bBhA0ceeSSTJk0CoEWLFkyZMoXRo0ezYcMG1q9fz2WXXUbfvn2BwmlAx4wZw6pVq5g2bRrXXXcd06dPp6SkhJEjR7J27VqGDx9e3c8999xDjx49gE9PW3rQQQfVGsuWnost1UlqviKP59ktKipKFRUVjdZfpzH/2Sj9lO/+nZ3fydUf7Pw+pBzx+Jear1wd/9DorwER8T8ppfpPh6UdwqU7kiRJUg4Z9CVJkqQcMuhLkiRJOWTQlyRJknLIoC9JkiTlkEFfkiRJyiGDviRJkpRDBn1JkiQphwz6kiRJUg4Z9CVJkqQcMuhLkiRJOWTQlyRJknLIoC9JkiTlkEFfkiRJyiGDviRJkpRDBn1JkiQphwz6kiRJUg4Z9CVJkqQcMuhLkiRJOWTQlyRJknLIoC9JkiTlkEFfkiRJyiGDviRJkpRDBn1JkiQphwz6kiRJUg4Z9CVJkqQcMuhLkiRJOWTQlyRJknLIoC9JkiTlkEFfkiRJyiGDviRJkpRDBn1JkiQphwz6kiRJUg4Z9CVJkqQcMuhLkiRJOWTQlyRJknLIoC9JkiTlkEFfkiRJyiGDviRJkpRDBn1JkiQphwz6kiRJUg4Z9CVJkqQcapKgHxEHRcTsiFgYEXMjottm2v04IhZll5809jglSZKkXVVTzejfBvw2pdQFuB6YVLdBRAwAvg30BLoBx0fE4EYdpSRJkrSLavSgHxHtgd7A5KzoQaBzRHSq0/SfgTtTSh+nlNYCt1MI/pIkSZK2oilm9A8AlqWUNgCklBKwBOhYp11HYHGN2+X1tJEkSZJUj5ZN1G+qczsa0G5zbYiI0cDoGkWVEbF8G8f2mRXQFli9UzsZv9mHWVIT8viXmq9GOf6hKV4D9m3sDpubpgj6S4GiiGiZUtoQEUFhln9JnXZLgE41bn+1njYApJR+BfxqJ4z1MyUiKlJKRU09DkmNz+Nfar48/rWtGn3pTkppBTAfGJYVnQaUp5TK6zSdCpwVEZ+LiDbACOC+RhuoJEmStAtrqrPunA+cHxELgTHASICI+ENE9AFIKc0ApgB/Av4LeDKl9HjTDFeSJEnatUThu7DaFUTE6GyZkqRmxuNfar48/rWtDPqSJElSDjXV0h1JkiRJO5FBX5IkScohg74k7QIi4uSImLCVNhdExCWNNSZJny0RcWdEfD+7fnVE3NDUY1LTaqofzNIOUPVbBE09DkkNt63HbUrpEeCRrbS5dZsHJqlR+R6uxuCM/g4UEZMj4sWIeCUiHo2I9ln58Ih4OSIWZPWdsvITIqIsK385Ig7PylNEtK2x3ZU17lMeEWMj4o/AXRGxX0T8MSJeiojXIuKm7EfIiIjWETEhIv6U9fF4Vv6niDiyxvbPj4j7G+VBknIqO26vjojnI2JhRHy7Tt2lETEDuDYruywi5kbEvOzUwgdk5Zs7bs+OiAey6wdl/SzI2v00K6+ewYuIFhFxQ0S8ml1ujojWWd2dEXFLRDydjfWhqjpJO88Oei3oERGzsvavR8QVTbdH+qxzRn/HujiltBIgIsYA47I35rHAN1JK70TEnll9F2ASMCCltDAiWgF7NrCfjsDAlFKKiN2Bk1JKqyOiBTCNwo+QPQBcAXwN6JNSWhsRVT81fRMwCpiT3R6VXSRtn5RSOioiDgTmRkRpSmlpVtcmpXQsQER8B+gCHJlSqoyIM4HfAKew+eO2pu8D/5lS+nm2vX3qaXMecFh2qaTw34CLgKrlP72A44B1wHMUXjf+Y7v2XlJDbO9rQTkwKCvbA5gdEU+llF5s7B3RZ59Bf8c6IztI2wB7AMuBvwJ3p5TeAUgpfQIQEX8P/CGltDArXw980MB+7kifnhd1N+AXEXE0EEB74GUKQf9E4NKU0tqsj3ez+0wGxmf/cSguVKVZ277bkjK/A0gp/XdElALfAO7N6m6v0e6fgD7AS9k/4FpQCOOw+eO2pueACRHxOWAm8HQ9bQYBk6q2ExH/DlzAp0H/oZTSX7O6uRQChaSdb3tfC/YAbomIXsBG4AAKH9wN+tqEQX8HyYL294H+KaV3I+JkYNw2bq6SwsFeZfc69atrXB8NfBE4PKW0JiJ+VU/7WlJKf42Iu4BzgBIKsweSdryaP1RS87gN4KcppdvZBimlByNiNvD3FF53LgaG1GkWdfqvO541Na5X4vuB1Fi297Xg58BfgJKU0oaIeIitvO+r+XKN/o6zN/Ah8F621vX8rHw68N2I2A8gIvbMlu88ARyfLeEhIlpFRLvsPouAqvX63wQ+t5V+l2ch/8vA6TXqHgEujog22bZqLgH4V+B7wDHA77dxnyXVNgIg+07N0UDpZto9AlxYteQmO/5LatRt7rglKzsIWJFSuhu4HDiinj6eAs7O1vm2BEZS/8y/pKazLa8FewMVWcg/mMIHfqlezuDsOI8Bw4A3gApgNjA4pfRc9kW5JyMiUVgPOzSl9FZEjAT+I1ufX0nhw8FcCrNz/xoRK4A/Av+7hX5vAqZGxMvA/1D7jfwXwM+A+RGxDlhGNuuXUqrI7rOwajmRpO22NiKeB/YF/qXG+vxaUkr3RMQXgRnZ60JLCt/Zmc8WjtsaTqewVHAdhRnBC+rp5rcUluPMy27PoPB6IekzYhtfC34K3BMRZ1BYr/9sU4xdu4b4dKm3mpMonNXnDQpfEn67qccj7eqyN+m9Ukqrt9pYkqRG4NKdZigiLqAQ8m8x5EuSJOWTM/qSJElSDjmjL0mSJOWQQV+SJEnKIYO+JEmSlEMGfUmSJCmHDPqStB0iYkZEPN7U45AkqS6DviRJkpRDBn1JkiQphwz6kpqViDgjIlJEdK5TvltELIuIidntL0fE7yLirYj4a0S8HRH/FhHttrL9qyNiTT3lb0TEnXXK+kXEkxHxUUSsjohpdce1mT5SRFwREVdmY14VEQ9ExBfrtPtZRLycbX95RDwaEQfXaXNnNra/i4j52b7Oi4g+EdE6Im6MiHcj4i8RcW1E7Fbn/gdHxIPZGP4aEX+MiF5b2wdJ0s5n0JfU3Px/4GPgO3XKBwIdgN9nt78IfARcDvwjcHXWZvqOGERE9AOeAzYAw4Azga8Cz0REmwZs4gKgBBgJ/BD4B+CmOm32AyYAJ2XtWwKzI+JLddq1ByYCNwBDgdbAQ8C/Z9fPAP4NGJONtWofOgGzga8A52X33QjMiIh9G7APkqSdqGVTD0CSGlNK6eOIeIRC0P9ZjarvAG+llOZm7V4HLqmqjIjngbeA0ojoltVvj+uBPwEnppQ2Zn28APw3MBy4dSv3fw/4Vsp+3jwivg5cEhHDqspSSiNrjL8F8ASwAvgWcEuNbe0N/F1KaUHWNih8oGmXUjora/NkRJwInA7cnZVdTeFD03EppY+z+z4LLAIupfDBQJLURJzRl9Qc/R7oFhGHAmQz6N/k09l8ouDiiHgtIj4B1gOlWXWX7ek8IvYAjgamALtFRMuIaAm8SyH892vAZp6sCvSZ1ynMvn+5Rj/HR0RpRKyi8J+DT4C29Yz/L1UhP7Mw+/tEnXYLgQNq3P4H4BFgbY19qHqcGrIPkqSdyKAvqTl6AljJp8t3TgDaAffWaHMx8CvgcQofAg7P/gLsvp397wO0oDCrv77OpS/QsQHbWFXn9rqaY4uIvhRC+CpgBNA/2/a7bDr+9zezrfr6qHnffYFR9ezD6Q3cB0nSTuTSHUnNTkppQ0RMBf5fRIyhEPjLUkoLazQbCjyWUrq0qiAijm3A5tcALSNit6olOZl9alx/n8Ja9gnAA/Vs46OG7ckW/ROFGfxTU0obALIZ9713wLar/C+FD00311O3dgf2I0naBgZ9Sc3V74HvAUMozOjXXU++J7C8TtmZDdjuUgqz9V2ANwAiooTC7DdQ/T2B54FDUko7ax37nhRm4Gt+2Pg2O/Z1/0mgJ/By1YcJSdJnh0FfUnM1GygHbgNaAffXqX8CuDQiLgFepXDmmmMbsN3HgNXA7yLipxQC/uVsugzmUmBmRDwMTKawlKhD1sezKaUpf/MebTr+i4FbI+J+CoF8NJsu09ke44AyCmcKuhVYRuEMPkcAi1NKdc8CJElqRK7Rl9QsZV9kvZfCqSGfSSnVnb2/BrgTGEthec2XKcyIb22771FYNrMX8DCFM/d8j8LZbmq2KwOOBILCaSyfoHAWoDYUvpC7XVJKj1MI9v8IPAqcBpwKfLC9267RRzmFL90uBW6kMMN/A1AEzN1R/UiStk3UPmmDJEmSpDxwRl+SJEnKIYO+JEmSlEMGfUmSJCmHDPqSJElSDhn0JUmSpBwy6EuSJEk5ZNCXJEmScsigL0mSJOWQQV+SJEnKof8D8RjaV71rVpwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 800x480 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "compare_two_evaluation_class(normal_method, baseline_method)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}