RF_Sheng.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Factory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#libraries for modeling\n",
    "from multiprocessing.pool import ThreadPool\n",
    "from pyspark import SparkContext\n",
    "from pyspark.sql import SQLContext, SparkSession, Window, Row\n",
    "from pyspark.ml.linalg import Vectors\n",
    "from pyspark.ml.feature import VectorAssembler\n",
    "from pyspark.ml.evaluation import RegressionEvaluator\n",
    "import pyspark.sql.functions as F\n",
    "import itertools\n",
    "from itertools import repeat\n",
    "import pickle\n",
    "import pyspark\n",
    "import copy\n",
    "\n",
    "from pyspark.ml.classification import MultilayerPerceptronClassifier\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.evaluation import MulticlassClassificationEvaluator\n",
    "from pyspark.ml.classification import RandomForestClassifier\n",
    "from pyspark.ml.classification import LogisticRegression\n",
    "from pyspark.ml.classification import GBTClassifier\n",
    "from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "\n",
    "#libraries for plotting\n",
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "\n",
    "class CreateBestModel:\n",
    "    def __init__(self, algo, avgprecision, avgrecall, avgfscore, hyperparams, ootmodel, ootprecision, ootrecall, ootfscore):\n",
    "        self.algo = algo\n",
    "        self.gsPrecision = avgprecision\n",
    "        self.gsFScore = avgfscore\n",
    "        self.gsRecall = avgrecall\n",
    "        self.hyperParams = hyperparams\n",
    "        self.model = ootmodel\n",
    "        self.ootPrecision = ootprecision\n",
    "        self.ootFScore = ootfscore\n",
    "        self.ootRecall = ootrecall\n",
    "\n",
    "#function-based\n",
    "def sample(df, sampling_method, ratio):\n",
    "\n",
    "    notfraud = df.select('*').where(df.Class == 0.0)\n",
    "    fraud = df.select('*').where(df.Class == 1.0)\n",
    "\n",
    "    if sampling_method == \"over\":\n",
    "        nrows = notfraud.select(\"Class\").count()\n",
    "        sample_size = int(nrows*ratio/(1-ratio))\n",
    "        sampled = fraud.rdd.takeSample(True, sample_size, 46)\n",
    "        fraud = sqlContext.createDataFrame(sampled)\n",
    "\n",
    "    elif sampling_method == \"under\":\n",
    "        nrows = fraud.select(\"Class\").count()\n",
    "        sample_size = int(nrows*(1-ratio)/ratio)\n",
    "        sampled = notfraud.rdd.takeSample(False, sample_size, 46)\n",
    "        notfraud = sqlContext.createDataFrame(sampled)\n",
    "    else:\n",
    "        return df\n",
    "    \n",
    "    sampled = fraud.union(notfraud)\n",
    "\n",
    "    #shuffle undersampled dataframe\n",
    "    nrows = sampled.select(\"Class\").count()\n",
    "    shuffled = sampled.rdd.takeSample(False, nrows, 46)\n",
    "    shuffled_df = sqlContext.createDataFrame(shuffled)\n",
    "\n",
    "    return shuffled_df\n",
    "\n",
    "def generateParamGrid(*args):\n",
    "    \n",
    "    grid = list(itertools.product(*args))\n",
    "    return grid\n",
    "\n",
    "def generateClassifier(algo, params, features):\n",
    "\n",
    "    ############################################################################\n",
    "    #TODO: complete this section\n",
    "\n",
    "    def lr(params,features):\n",
    "        print(params)\n",
    "        if len(params) > 2:\n",
    "            lrClassifier = LogisticRegression(featuresCol = 'features',\n",
    "                                          labelCol = 'Class',\n",
    "                                          threshold=params[0],\n",
    "                                           maxIter=params[1],\n",
    "                                           weightCol=params[2])\n",
    "                                          #regParam=params[2])\n",
    "                                          #elasticNetParam=params[2])\n",
    "        else:\n",
    "            lrClassifier = LogisticRegression(featuresCol = 'features',\n",
    "                                          labelCol = 'Class',\n",
    "                                          threshold=params[0],\n",
    "                                           maxIter=params[1])\n",
    "        return lrClassifier\n",
    "\n",
    "\n",
    "    def gbm(params,features):\n",
    "        gbmClassifier = GBTClassifier(featuresCol = 'features',\n",
    "                                      labelCol = 'Class',\n",
    "                                      maxDepth = params[0],\n",
    "                                      minInfoGain = params[1])\n",
    "        return gbmClassifier\n",
    "\n",
    "    def rf(params,features):\n",
    "        rfClassifier = RandomForestClassifier(featuresCol='features',\n",
    "                                              labelCol='Class',\n",
    "                                              maxDepth=params[0],\n",
    "                                              minInfoGain=params[1],\n",
    "                                              numTrees=params[2])\n",
    "\n",
    "        return rfClassifier\n",
    "\n",
    "    def mlp(params,features):\n",
    "        input_layers = len(features)\n",
    "        layers = [input_layers, params[1], 2]\n",
    "        print(layers)\n",
    "        mlpClassifier = MultilayerPerceptronClassifier(featuresCol = 'features',\n",
    "                                                       labelCol = 'Class',\n",
    "                                                       maxIter = params[0],\n",
    "                                                       layers = layers,\n",
    "                                                       stepSize = params[2])\n",
    "        return mlpClassifier\n",
    "\n",
    "    def svm(params, features):\n",
    "        svmClassifier = LinearSVC(featuresCol = 'features',\n",
    "                         labelCol='Class', \n",
    "                         maxIter=params[0],\n",
    "                         regParam=params[1],\n",
    "                         tol =params[2]\n",
    "                         )\n",
    "        \n",
    "        return svmClassifier\n",
    "\n",
    "    def xg(params,features):\n",
    "        return\n",
    "    ############################################################################\n",
    "\n",
    "    getClassifier = {\n",
    "        'lr':lr,\n",
    "        'gbm':gbm,\n",
    "        'rf':rf,\n",
    "        'mlp':mlp,\n",
    "        'svm':svm,\n",
    "        'xg':xg}\n",
    "\n",
    "    return getClassifier[algo](params,features)\n",
    "\n",
    "def crossValidate(df, folds, k, classifier, features, sampling_method, ratio, pool):\n",
    "\n",
    "    def build(fold, df, classifier, features, sampling_method, ratio):\n",
    "\n",
    "        validation = fold\n",
    "        train = df.subtract(fold)\n",
    "\n",
    "#         #add class weight\n",
    "#         notfraud_count = train.select(\"Class\").where(train.Class == 0.0).count()\n",
    "#         total_count = train.select(\"Class\").count()\n",
    "#         balance_ratio = notfraud_count / total_count\n",
    "#         train=train.withColumn(\"classWeights\", F.when(train.Class == 1.0,balance_ratio).otherwise(1-balance_ratio))\n",
    "        \n",
    "        train = sample(train, sampling_method, ratio)\n",
    "        fraud_count = train.select(\"Class\").where(train.Class == 1.0).count()\n",
    "        tot_count = train.select(\"Class\").count()\n",
    "        fraud_ratio = fraud_count / tot_count\n",
    "        print(\"train: \" + str(tot_count))\n",
    "        print(\"fraud ratio: \" + str(fraud_ratio))\n",
    "        \n",
    "        vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "        vector_train = vectorAssembler.transform(train)\n",
    "        vector_validate = vectorAssembler.transform(validation)\n",
    "        model = classifier.fit(vector_train)\n",
    "        pred = model.transform(vector_validate)\n",
    "        pos = pred.filter(pred.prediction == 1.0).count()\n",
    "        if pos != 0:\n",
    "            precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pos\n",
    "        else:\n",
    "            precision = 0\n",
    "        fraud = pred.filter(pred.Class == 1.0).count()\n",
    "        if fraud != 0:\n",
    "            recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / fraud\n",
    "        else:\n",
    "            recall = 0\n",
    "        precision_recall = precision + recall\n",
    "        if precision_recall != 0:\n",
    "            f_score = 2 * precision * recall /(precision_recall)\n",
    "        else:\n",
    "            f_score = 0\n",
    "        #print(\"\\n precision, recall, f_score: \" + str(precision) + \", \" + str(recall) + \", \" + str(f_score))\n",
    "        return [precision, recall, f_score]\n",
    "\n",
    "    #call multiprocessing here\n",
    "    cvperformance = pool.map(lambda fold: build(fold, df, classifier, features, sampling_method, ratio), folds)\n",
    "\n",
    "    #calculate metrics\n",
    "    precision_sum = sum([x[0] for x in cvperformance])\n",
    "    recall_sum = sum([x[1] for x in cvperformance])\n",
    "\n",
    "    avg_precision = precision_sum/k\n",
    "    avg_recall = recall_sum/k\n",
    "    if avg_precision+avg_recall == 0:\n",
    "        avg_fscore = 0\n",
    "    else:\n",
    "        avg_fscore = 2 * avg_precision * avg_recall /(avg_precision+avg_recall)\n",
    "    return [avg_precision,avg_recall,avg_fscore]\n",
    "\n",
    "def gridSearch(df, folds, k, algo, grid, features, sampling_method, ratio, pool):\n",
    "\n",
    "    best_hyper = None\n",
    "    best_precision = 0\n",
    "    best_recall = 0\n",
    "    best_fscore = 0\n",
    "\n",
    "    for i in range(len(grid)):\n",
    "        params = list(grid[i])\n",
    "        print(params)\n",
    "        classifier = generateClassifier(algo, params, features)\n",
    "        modelPerformance = crossValidate(df, folds, k, classifier, features, sampling_method, ratio, pool)\n",
    "        print(modelPerformance)\n",
    "        if modelPerformance[2] > best_fscore:\n",
    "            best_hyper = params\n",
    "            best_precision = modelPerformance[0]\n",
    "            best_recall = modelPerformance[1]\n",
    "            best_fscore = modelPerformance[2]\n",
    "\n",
    "    return best_hyper, best_precision, best_recall, best_fscore\n",
    "\n",
    "def TrainTest(traindf,testdf,algo,features,params):\n",
    "    vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "    classifier = generateClassifier(algo, params, features)\n",
    "    vector_train = vectorAssembler.transform(traindf)\n",
    "    vector_test = vectorAssembler.transform(testdf)\n",
    "    m = classifier.fit(vector_train)\n",
    "    pred = m.transform(vector_test)\n",
    "    pos = pred.filter(pred.prediction == 1.0).count()\n",
    "    if pos != 0:\n",
    "        precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pos\n",
    "    else:\n",
    "        precision = 0\n",
    "    fraud = pred.filter(pred.Class == 1.0).count()\n",
    "    if fraud != 0:\n",
    "        recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / fraud\n",
    "    else:\n",
    "        recall = 0\n",
    "    precision_recall = precision + recall\n",
    "    if precision_recall != 0:\n",
    "        f_score = 2 * precision * recall /(precision_recall)\n",
    "    else:\n",
    "        f_score = 0\n",
    "    print(\"\\n precision, recall, f_score: \" + str(precision) + \", \" + str(recall) + \", \" + str(f_score))\n",
    "    predictionAndLabels = pred.select('Class','prediction').rdd.map(lambda lp: (float(lp.prediction), lp.Class))\n",
    "    metrics = MulticlassMetrics(predictionAndLabels)\n",
    "    print(metrics.confusionMatrix().toArray())\n",
    "    #evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"rawPrediction\", labelCol=\"Class\",metricName=\"areaUnderPR\")\n",
    "    #print(\"Area under Precision Recall Curve = %s\" % evaluator.evaluate(pred))\n",
    "    \n",
    "    return m, precision, recall, f_score\n",
    "\n",
    "def tune(df, k, stratification_flag, sampling_method, ratio, modelobj_flag, features, algo, *args, **kwargs):\n",
    "\n",
    "    \"\"\"\n",
    "    Entry point of this suite of functions. returns cv metrics or a model object\n",
    "    Example:\n",
    "        >>> cv_hyper, cv_precision, cv_recall, cv_fscore = tune(df, 5, True,\n",
    "        'None', 0, False, features, 'mlp', [100], [15], [0.03])\n",
    "    :param df: data for modeling purpose\n",
    "    :type df: : pyspark dataframe\n",
    "    :param k: number of folds for cross validation\n",
    "    :type k: int\n",
    "    :param stratification_flag: specifies whether fraud ratio is fixed for each fold. True for stratification\n",
    "    :type stratification_flag: boolean\n",
    "    :param sampling_method: \"over\" for oversampling minority class, \"under\" for undersampling majority class, \"None\"\n",
    "    :type sampling_method: str\n",
    "    :param ratio: targeted fraud ratio after sampling.\n",
    "    :type ratio: float\n",
    "    :param modelobj_flag: specifies whether to return a model object for out of time test. if False, returns cv performancce\n",
    "    :type modelobj_flag: float\n",
    "    :param features: features for training\n",
    "    :type features: list\n",
    "    :param *args: a sequence of params for hyperparams tuning. ex. [values for params1], [values for params2],...\n",
    "    :type *args: list\n",
    "    :returns: model object or cross validation metrics depending on modelobj_flag\n",
    "    \"\"\"\n",
    "\n",
    "\n",
    "    pool = ThreadPool(3)\n",
    "\n",
    "    #reduce df dimenions to include features and class\n",
    "    cols = features+['Class', 'index']\n",
    "    df = df.select(cols)\n",
    "    df = df.select(*(F.col(c).cast(\"double\").alias(c) for c in df.columns))\n",
    "    df.cache()\n",
    "    #df.drop(\"index\")\n",
    "    \n",
    "    ########################ClassWeights#################################\n",
    "    if algo in [\"lr\", \"svm\"] and [\"ClassWeigts\"] in args:\n",
    "        #add class weight\n",
    "        balance_ratio = args[-1][0]\n",
    "        df=df.withColumn(\"classWeights\", when(df.Class == 1.0,balance_ratio).otherwise(1-balance_ratio))\n",
    "    ########################ClassWeights#################################\n",
    "    \n",
    "    folds = []\n",
    "    \n",
    "    if stratification_flag == False:\n",
    "        tot_count = df.select(\"Class\").count()\n",
    "        n = int(tot_count / k)\n",
    "\n",
    "        #create sub-dataframe iteratively\n",
    "        fold_start = 1\n",
    "        fold_end = n\n",
    "        for i in range(k):\n",
    "            fold = df.select('*').where(df.index.between(fold_start, fold_end))\n",
    "            folds.append(fold)\n",
    "            fold_start = fold_end + 1\n",
    "            fold_end = fold_start + n\n",
    "            if i == k-2:\n",
    "                end = tot_count\n",
    "                \n",
    "    if stratification_flag == True:\n",
    "        fraud = df.select(\"*\").where(df.Class == 1.0)\n",
    "        #shuffle undersampled dataframe\n",
    "        nrows = fraud.select(\"Class\").count()\n",
    "        shuffled = fraud.rdd.takeSample(False, nrows, 46)\n",
    "        fraud = sqlContext.createDataFrame(shuffled)\n",
    "        #add row index to dataframe\n",
    "        fraud = fraud.withColumn('dummy', F.lit('7'))\n",
    "        fraud = fraud.withColumn(\"temp_index\", F.row_number().over(Window.partitionBy(\"dummy\").orderBy(\"dummy\")))\n",
    "        fraud = fraud.drop('dummy')\n",
    "        fraud_count = fraud.select(\"Class\").count()\n",
    "        each_fraud = int(fraud_count/k)\n",
    "\n",
    "        notfraud = df.select(\"*\").where(df.Class == 0.0)\n",
    "        nrows = notfraud.select(\"Class\").count()\n",
    "        shuffled = notfraud.rdd.takeSample(False, nrows, 46)\n",
    "        notfraud = sqlContext.createDataFrame(shuffled)\n",
    "        #add row index to dataframe\n",
    "        notfraud = notfraud.withColumn('dummy', F.lit('7'))\n",
    "        notfraud = notfraud.withColumn(\"temp_index\", F.row_number().over(Window.partitionBy(\"dummy\").orderBy(\"dummy\")))\n",
    "        notfraud = notfraud.drop('dummy')\n",
    "        notfraud_count = notfraud.select(\"Class\").count()\n",
    "        each_notfraud = int(notfraud_count/k)\n",
    "\n",
    "        fraud_start = 1\n",
    "        fraud_end = each_fraud\n",
    "        notfraud_start = 1\n",
    "        notfraud_end = each_notfraud\n",
    "\n",
    "        for i in range(k):\n",
    "            fraud_fold  = fraud.select('*').where(fraud.temp_index.between(fraud_start, fraud_end))\n",
    "            notfraud_fold = notfraud.select('*').where(notfraud.temp_index.between(notfraud_start, notfraud_end))\n",
    "            fold = fraud_fold.union(notfraud_fold).drop(\"temp_index\")\n",
    "            folds.append(fold)\n",
    "            fraud_start = fraud_end + 1\n",
    "            fraud_end = fraud_start + each_fraud\n",
    "            notfraud_start = notfraud_end + 1\n",
    "            notfraud_end = notfraud_start + each_notfraud\n",
    "            if i == k-2:\n",
    "                fraud_end = fraud_count\n",
    "                notfraud_end = notfraud_count\n",
    "\n",
    "\n",
    "    #generate hyperparam combo\n",
    "    grid = generateParamGrid(*args)\n",
    "\n",
    "    #conduct grid search:\n",
    "    best_hyper, best_precision, best_recall, best_fscore = gridSearch(df, folds, k, algo, grid, features, sampling_method, ratio, pool)\n",
    "\n",
    "    if modelobj_flag == True:\n",
    "        #generate a model obj\n",
    "        traindf = sample(df, sampling_method, ratio)\n",
    "        testdf = sqlContext.read.csv(\"oot.csv\", header = True)\n",
    "        cols = features+['Class', 'index']\n",
    "        testdf = testdf.select(cols)\n",
    "        testdf = testdf.select(*(F.col(c).cast(\"double\").alias(c) for c in testdf.columns))\n",
    "        model, precision, recall, fscore = ootTest(traindf, testdf, algo,features,best_hyper)\n",
    "        \n",
    "        modelobj = CreateBestModel(algo, best_precision, best_recall, best_fscore, best_hyper, \n",
    "                                   model, precision, recall, fscore)\n",
    "        return modelobj\n",
    "\n",
    "    return best_hyper, best_precision, best_recall, best_fscore\n",
    "\n",
    "def save(content, filename):\n",
    "\n",
    "    pickle.dump(content, open(filename, \"wb\"))\n",
    "\n",
    "def load(filename):\n",
    "\n",
    "    content = pickle.load(open(filename, \"rb\"))\n",
    "    return content\n",
    "\n",
    "def generateStratifiedFolds(df,k):\n",
    "\n",
    "    return folds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc=pyspark.SparkContext.getOrCreate()\n",
    "sqlContext = SQLContext(sc)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STEP 1: Train base_train and test base_test with default params."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+--------------------+----------+\n",
      "|            features|Class|       rawPrediction|prediction|\n",
      "+--------------------+-----+--------------------+----------+\n",
      "|[0.004722604,0.97...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.002797544,0.94...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.002574717,0.97...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.007631052,0.93...|  0.0|[19.9929002437546...|       0.0|\n",
      "|[0.010660071,0.98...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.018060156,0.93...|  0.0|[19.9856889064570...|       0.0|\n",
      "|[9.11E-5,0.993164...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[5.09E-5,0.956463...|  0.0|[19.9911261080119...|       0.0|\n",
      "|[0.001525702,0.94...|  0.0|[19.9944845217981...|       0.0|\n",
      "|[0.002950673,0.99...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.012572412,0.98...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.001882326,0.94...|  0.0|[19.9947020343690...|       0.0|\n",
      "|[5.09E-5,0.943140...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[0.004441781,0.97...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[1.31E-4,0.958263...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[5.08E-4,0.939638...|  0.0|[19.9844698863501...|       0.0|\n",
      "|[0.0,0.978097354,...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[7.15E-4,0.978279...|  0.0|[19.9919914815814...|       0.0|\n",
      "|[0.003838928,0.93...|  0.0|[19.9946040611091...|       0.0|\n",
      "|[6.56E-5,0.979268...|  0.0|[19.9946040611091...|       0.0|\n",
      "+--------------------+-----+--------------------+----------+\n",
      "only showing top 20 rows\n",
      "\n",
      "+-----+------------+\n",
      "|Class|count(Class)|\n",
      "+-----+------------+\n",
      "|  0.0|       45305|\n",
      "|  1.0|          91|\n",
      "+-----+------------+\n",
      "\n",
      "+----------+-----------------+\n",
      "|prediction|count(prediction)|\n",
      "+----------+-----------------+\n",
      "|       0.0|            45311|\n",
      "|       1.0|               85|\n",
      "+----------+-----------------+\n",
      "\n",
      "Precision is  0.8470588235294118\n",
      "Recall is  0.7912087912087912\n",
      "F1 score is  0.8181818181818181\n",
      "Test Area Under ROC:  0.973422141262335\n"
     ]
    }
   ],
   "source": [
    "# import pyspark\n",
    "# from pyspark.sql.types import DateType\n",
    "# from pyspark.sql import SQLContext, SparkSession, Window\n",
    "# import pyspark.sql.functions as F\n",
    "# from pyspark.ml.regression import LinearRegression\n",
    "# from pyspark.ml.linalg import Vectors\n",
    "# from pyspark.ml.feature import VectorAssembler\n",
    "# from pyspark.ml.evaluation import RegressionEvaluator\n",
    "# from pyspark.sql.functions import abs, sqrt\n",
    "# import pyspark.sql.functions as func\n",
    "# #from pyspark.mllib.tree import RandomForest, RandomForestModel\n",
    "# #from pyspark.mllib.util import MLUtils\n",
    "# from pyspark.ml.classification import RandomForestClassifier\n",
    "# from pyspark.ml.evaluation import BinaryClassificationEvaluator\n",
    "# sc=pyspark.SparkContext.getOrCreate()\n",
    "# sqlContext = SQLContext(sc)\n",
    "trainingData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "sc1=pyspark.SparkContext.getOrCreate()\n",
    "sqlContext1 = SQLContext(sc1)\n",
    "testData = sqlContext1.read.csv(\"base_test.csv\", header = True)\n",
    "trainingData = trainingData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainingData.columns))\n",
    "testData = testData.select(*(F.col(c).cast(\"double\").alias(c) for c in testData.columns))\n",
    "vectorAssembler = VectorAssembler(inputCols = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28'], outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainingData)\n",
    "df_test = vectorAssembler.transform(testData)\n",
    "rf = RandomForestClassifier(featuresCol = 'features', labelCol = 'Class')\n",
    "rfModel = rf.fit(df_train)\n",
    "pred = rfModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "pred.show()\n",
    "pred.groupby('Class').agg({'Class': 'count'}).show()\n",
    "pred.groupby('prediction').agg({'prediction': 'count'}).show()\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)\n",
    "evaluator = BinaryClassificationEvaluator(labelCol=\"Class\", metricName=\"areaUnderROC\")\n",
    "print(\"Test Area Under ROC: \", evaluator.evaluate(pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STEP 2   compare random cv vs stratified cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 0, 20]\n",
      "train: 152482\n",
      "fraud ratio: 0.0016592122348867407\n",
      "train: 152538\n",
      "fraud ratio: 0.0017372720240202441\n",
      "train: 152562\n",
      "fraud ratio: 0.0014420366801693738\n",
      "train: 152553\n",
      "fraud ratio: 0.0018223174896593315\n",
      "train: 152499\n",
      "fraud ratio: 0.0016655846923586384\n",
      "[0.8711749716005036, 0.7126736355806124, 0.7839934087417116]\n",
      "avg precision: 0.8711749716005036\n",
      "avg recall: 0.7126736355806124\n",
      "avg f-score: 0.7839934087417116\n"
     ]
    }
   ],
   "source": [
    "sqlContext.clearCache()\n",
    "\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "hiddenlayer = int((len(features) + 2) / 2)\n",
    "cv_hyper, cv_precision, cv_recall, cv_fscore = tune(df, 5, False, 'None', 0, False, features, 'rf', [5], [0], [20])\n",
    "print(\"avg precision:\", cv_precision)\n",
    "print(\"avg recall:\", cv_recall)\n",
    "print(\"avg f-score:\", cv_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 0, 20]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8537173576431558, 0.7436805922792173, 0.7949090334089431]\n",
      "grid search precision: 0.8537173576431558\n",
      "grid search recall: 0.7436805922792173\n",
      "grid search f-score: 0.7949090334089431\n",
      "grid search hyper: [5, 0, 20]\n"
     ]
    }
   ],
   "source": [
    "# stratified\n",
    "df = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "cv_hyper, cv_precision, cv_recall, cv_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [5], [0], [20])\n",
    "print(\"grid search precision:\", cv_precision)\n",
    "print(\"grid search recall:\", cv_recall)\n",
    "print(\"grid search f-score:\", cv_fscore)\n",
    "print(\"grid search hyper:\", cv_hyper)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## conclusion: stratified is better and variance in stratified is small"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STEP 3 GRID SEARCH"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8638682082564904, 0.6982548915917504, 0.7722825456768365]\n",
      "[4, 0, 150]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8653719676549866, 0.711158117398202, 0.7807225567102772]\n",
      "[4, 0, 200]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8636782492387376, 0.6983077736647277, 0.7722389657078597]\n",
      "[5, 0, 100]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8656776094276093, 0.759862506610259, 0.8093260224415355]\n",
      "[5, 0, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8619312169312169, 0.7566367001586463, 0.8058590373089098]\n",
      "[5, 0, 200]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8649831649831649, 0.7566367001586463, 0.807190417081391]\n",
      "[6, 0, 100]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.9202604513503514, 0.759862506610259, 0.8324050450999914]\n",
      "[6, 0, 150]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.9116605140589961, 0.763088313061872, 0.8307842614875773]\n",
      "[6, 0, 200]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.9066130349713699, 0.759862506610259, 0.8267763145507108]\n",
      "grid search precision: 0.9202604513503514\n",
      "grid search recall: 0.759862506610259\n",
      "grid search f-score: 0.8324050450999914\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "#Use ClassWeight for logistric regression\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [4, 5, 6], [0], [100, 150, 200])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## conclusion: best hyperparams are [6, 0, 100]."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  STEP4 FEATURE SELECTION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train before sampling: 181584\n",
      "fraud ratio: 0.0016961846858754076\n",
      "test: 45396\n",
      "Precision is  0.8902439024390244\n",
      "Recall is  0.8021978021978022\n",
      "F1 score is  0.8439306358381503\n",
      "Area under Precision Recall Curve = 0.8729666338346731\n",
      "[[4.5296e+04 9.0000e+00]\n",
      " [1.8000e+01 7.3000e+01]]\n",
      "(29,[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28],[0.009867401545201923,0.005338619207018819,0.014067584427549233,0.04424261666710388,0.05864724799553147,0.018563845851873158,0.00667604673991056,0.0950267908123432,0.007330665944102226,0.0659203609111006,0.08533000648155876,0.03907037063181175,0.12262089784475727,0.0048027525628128835,0.06683721020248032,0.005071227380361747,0.1134369740398188,0.10147668544431061,0.04140360372540403,0.005456033286675302,0.007851156524588008,0.021302672247741187,0.008887676319430158,0.0023555094186932814,0.007501056704429167,0.004381669952297106,0.02147775902509658,0.008959415366051028,0.006096142739947034])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA4IAAAIOCAYAAADtDLN2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzs3Xu4bXVZL/DvCyje8oao5Q0Uq4NpWoid7FhJKuQFL2BQmrdCTe1iVtoxM7OLZtlRqbyHmoKKGipqpoaXvCEiiooikm6vGCgoKSLv+WOOLbPV2rCAPdbaa/8+n+eZz55jzDF/7zvGmmvO9d1jzDGquwMAAMA4dtnoBgAAAFhfgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEgMuoqv6xqt6w0X0AwOUlCAKwZlMA6lVut92ONZ5cVR/bXuPN5LeSPGCjm7gk08/lkI3uA4Ad024b3QAAm86/Jnnginlf24hGLk1VXbm7L9je43b3N7b3mNvLXOsMwM7FHkEALqvvdPeXV9wuTJJa+P2q+kxV/VdVfbSq/tues6r6y6o6bXr8zKp6elVdZXrswUn+OMmtlvY2Pnh67H/s4Zqe/7il6a6qR1XVa6rqW0n+fJq/b1W9sarOq6qvVtUrquqGS8+7dVW9rarOnZb5SFX9/LY2wMpDQ6vq36rq76vqr6vq7Ko6q6p+q6p2r6ojq+rrVfW5qnrg0nP2mvr95ap6d1V9u6o+WVV3XVHrTlX1/unxr1TVM6vqyqvUfkZVnZXkPVV15vTwq6YaZ07L3qKq/rmqvlxV36qqk6rqHqts0ydW1XOn7bGlqn5vxTLXnGp+aerrE1X1S0uP/3RVnVBV51fVF6Zlr7mt7QnA+hMEAdienprkYUkelWTfJH+R5LlVdfelZb6V5KFJ/leS30hyWJL/Oz12TJK/TnJakh+cbsdcxh7+OMnxSW6d5Miq+sEk70zysST7J/mFJNdIclxVbf0cfHmSL02P3y7Jk5N8+zLW/ZUk5yW5Q5K/TPK3SV6X5FNJ9ktyVJIXVNUPrXje05M8K8ltk7w1yT9X1Y2SZPr3TUk+PPX1sCSHZ7Fdlz0gSSX5P0l+Ncntp/m/nsU23Dp9jWm8uyT58STHJnlNVf3oivF+J8lHk/xEkqcleXpV/e+pp5rG+NkkD8ni5/zYJBdMj986yb8kOW6qcd9p3V50iVsPgHVV3b3RPQCwSVTVP2YROpZD0ru6+6CqunoWh4jetbvftfScv03yw939i9sY8xFJHtfd+0zTT05ySHf/2IrlOsmh3f3qpXlnJnlOdz9jaZnndPdjlpZ5SpI7dvcBS/Ouk+TsJHfo7g9U1blJHtPdR12G7XC97r7HNP1vSXbv7uWw9NUk7+3ue03zrpRFCP7l7n51Ve2V5LNJntjdfzYts0uSTyZ5ZXc/sar+LMkvTdvvommZByd5bpLrdPf5U+3rdvdtLm17bWNd3pfkDd391KVt+t7uPnxpmU8nOaq7n1pVd0nyliS36u5PrDLeS5J8t7sftjTvtlmE2Rt091cvqR8A1ofvCAJwWb0zyRFL0/81/btvkqskefMUQra6UpIzt05Mh3f+dpJ9sthDtet0215OXDH9k0nuVFXfXGXZWyT5QJK/yWJv3YOSvC3Jsd39yctY95Std7q7q+qrWexV2zrvu1V1TpLrr3jee5eWuaiq3p/FtkwWe03fuzUETt6d5MpZbL+tNT+0lgansP7HSe6RxZ7CK2XxMztlxaIrp7+41PftknxptRA4+ckk+ywfKprF3spksb0FQYAdgCAIwGV1fnefvsr8rYdZ3jPJ51Y89t0kqaqfSnJ0kj/J4vDDrye5V5JnrKFu5+JAsdWVVlnuW6v09cYkj1tl2a8kSXc/uar+KclBSe6W5I+r6hHdfVkOZ/zuKv2uNu+yfC2jpuesZnn+ynXelmckOTCLbfHpJOcneUkWwXLZJfW98mew0i5JXpDkmas89oU19gnAzARBALaXjyf5TpKbdffbt7HMHZN8obv/dOuMqrrZimUuyOp7CM/KYi/W1ufdYHn6EpyU5P5J/qO7Vwac7+vuT2cRjp5VVX+f5NeyPt9r+6kkb0++f0jp/km2Hs758ST3r6pdlvYK/kwW2+gzlzLud/M/t+PPJHlJdx871btKFnvpPnUZ+j0pyQ9W1f/axl7Bk7I4bHS1/ywAYAfhZDEAbBfdfV4We5yeUVUPrap9quq2VfWIqtp6KOmnktyoqn6lqm5eVY/M4uQny85McrOq+omqul5V7T7Nf3uSR1XVflV1uyT/mLWd0OXIJNdKckxV3WGq+wtV9byq+oGquup0Zs+fm87keYcsAtPHr8j2uAweWVWHVNWPZHGCmZsl+fvpsb9L8kNJ/q6q/td00p2/zOJ7kOdfyrhnJjmgqm44fScyWWz/+0zb9tZJXpbFoaGXxduSvD/JsVV1t6rau6ruUlX3nh5/WpL9q+ofqup20+vgHlX13MtYB4AZCYIAbE9/lMUZNx+X5NQszoJ5vyxOipLufn2Sv8oi8JySxdkrn7RijGOzOOvn27LYC7g1KP5ukjOS/FsWe8xekDV836y7v5jFnsiLkrx56uvILPZefifJ95JcJ4uzep6W5LVZfG/vsZdlxa+Ax0+1PpLFYZv36e4tU+9fyOJw1dslOTmLPZSvSPKHaxj3d5P8fJLPZ3Gilkx1vprkXVmc+fN90/01m/ZMHpTkPVkEyU8k+X+ZDi/t7lOS3CnJXklOmNbrLzIdhgvAjsFZQwFgAyydNfT23b3yBDcAMCt7BAEAAAYjCAIAAAzGoaEAAACDsUcQAABgMIIgAADAYHaaC8pf73rX67322muj2wAAANgQH/rQh77W3XuuZdmdJgjutddeOfFEZ98GAADGVFX/sdZlHRoKAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgdtvoBuZw1t+/bLax93zkA2YbGwAAYD3YIwgAADAYQRAAAGAwgiAAAMBgBEEAAIDBzBoEq+rAqjqtqk6vqsev8vidquqkqrqwqg5Zmn/bqnpvVZ1aVadU1S/N2ScAAMBIZguCVbVrkiOTHJRk3ySHV9W+Kxb7XJIHJ3n5ivnnJ/nV7r5VkgOT/G1VXXuuXgEAAEYy5+Uj9k9yenefkSRVdXSSg5N8fOsC3X3m9NhFy0/s7k8t3f9iVX01yZ5Jvj5jvwAAAEOY89DQGyX5/NL0lmneZVJV+ye5cpLPrPLYEVV1YlWdeNZZZ13uRgEAAEYyZxCsVeb1ZRqg6geTvDTJQ7r7opWPd/fzunu/7t5vzz33vJxtAgAAjGXOILglyU2Wpm+c5ItrfXJVXTPJG5M8sbvft517AwAAGNacQfCDSW5ZVXtX1ZWTHJbkuLU8cVr+tUle0t2vmrFHAACA4cwWBLv7wiSPTvKWJJ9I8sruPrWqnlJV90qSqrp9VW1JcmiS51bVqdPT75/kTkkeXFUnT7fbztUrAADASOY8a2i6+/gkx6+Y96Sl+x/M4pDRlc97WZKXzdkbAADAqGa9oDwAAAA7HkEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADCY3Ta6gZ3FV//h/8029vUf8VuzjQ0AAIzHHkEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDC7bXQDXH5fOPI3Zxv7Ro961mxjAwAAG2vWPYJVdWBVnVZVp1fV41d5/E5VdVJVXVhVh6x47EFV9enp9qA5+wQAABjJbEGwqnZNcmSSg5Lsm+Twqtp3xWKfS/LgJC9f8dzrJvnjJHdIsn+SP66q68zVKwAAwEjm3CO4f5LTu/uM7r4gydFJDl5eoLvP7O5Tkly04rl3S/LW7j67u89J8tYkB87YKwAAwDDmDII3SvL5pekt07y5nwsAAMAlmDMI1irzens+t6qOqKoTq+rEs8466zI1BwAAMKo5g+CWJDdZmr5xki9uz+d29/O6e7/u3m/PPfe83I0CAACMZM4g+MEkt6yqvavqykkOS3LcGp/7liR3rarrTCeJues0DwAAgCtotiDY3RcmeXQWAe4TSV7Z3adW1VOq6l5JUlW3r6otSQ5N8tyqOnV67tlJ/jSLMPnBJE+Z5gEAAHAFzXpB+e4+PsnxK+Y9aen+B7M47HO1574oyYvm7A8AAGBEs15QHgAAgB2PIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYHbb6AbYXD7+d/eabex9f+O42cYGAAAuZo8gAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMJjdNroBuDTved49Zhv7jke8YbaxAQBgR2WPIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYzKxBsKoOrKrTqur0qnr8Ko/vXlXHTI+/v6r2muZfqaqOqqqPVtUnquoJc/YJAAAwktmCYFXtmuTIJAcl2TfJ4VW174rFHpbknO7eJ8kzkzxtmn9okt27+9ZJfjLJw7eGRAAAAK6YOfcI7p/k9O4+o7svSHJ0koNXLHNwkqOm+69OckBVVZJOcvWq2i3JVZNckOTcGXsFAAAYxpxB8EZJPr80vWWat+oy3X1hkm8k2SOLUPitJF9K8rkkz+jus1cWqKojqurEqjrxrLPO2v5rAAAAsBOaMwjWKvN6jcvsn+R7SX4oyd5Jfreqbv4/Fux+Xnfv19377bnnnle0XwAAgCHMGQS3JLnJ0vSNk3xxW8tMh4FeK8nZSX45yZu7+7vd/dUk70my34y9AgAADGPOIPjBJLesqr2r6spJDkty3IpljkvyoOn+IUne3t2dxeGgd66Fqyf5qSSfnLFXAACAYcwWBKfv/D06yVuSfCLJK7v71Kp6SlXda1rshUn2qKrTkzw2ydZLTByZ5BpJPpZFoHxxd58yV68AAAAj2W3Owbv7+CTHr5j3pKX7387iUhErn/fN1eYDAABwxV3qHsGq+uGqeltVfWyavk1VPXH+1gAAAJjDWg4NfX6SJyT5bpJMh2geNmdTAAAAzGctQfBq3f2BFfMunKMZAAAA5reWIPi1qrpFpmsAVtUhWVzoHQAAgE1oLSeLeVSS5yX50ar6QpLPJnnArF0BAAAwm0sNgt19RpJfmK7nt0t3nzd/WwAAAMxlLWcN/fOqunZ3f6u7z6uq61TVU9ejOQAAALa/tXxH8KDu/vrWie4+J8kvztcSAAAAc1pLENy1qnbfOlFVV02y+yUsDwAAwA5sLSeLeVmSt1XVi7M4c+hDkxw1a1cAAADMZi0ni3l6VX00yQFJKsmfdvdbZu8MAACAWaxlj2C6+01J3jRzLwAAAKyDtZw19L5V9emq+kZVnVtV51XVuevRHAAAANvfWvYIPj3JPbv7E3M3AwAAwPzWctbQrwiBAAAAO4+17BE8saqOSfK6JN/ZOrO7XzNbVwAAAMxmLUHwmknOT3LXpXmdRBAEAADYhNZy+YiHrEcjAAAArI9LDYJVdZUkD0tyqyRX2Tq/ux86Y18AAADMZC0ni3lpkhsmuVuSE5LcOMl5czYFAADAfNYSBPfp7j9K8q3uPirJ3ZPcet62AAAAmMtaguB3p3+/XlU/luRaSfaarSMAAABmtZazhj6vqq6T5IlJjktyjSR/NGtXAAAAzGYtQfBt3X1OkncmuXmSVNXes3YFAADAbNZyaOixq8x79fZuBAAAgPWxzT2CVfWjWVwy4lpVdd+lh66ZpctIAAAAsLlc0qGhP5LkHkmuneSeS/PPS/LrczYFAADAfLYZBLv7n6vqDUn+oLv/fB17AgAAYEaX+B3B7v5ekrusUy8AAACsg7WcNfTfq+o5SY5J8q2tM7v7pNm6AgAAYDZrCYI/Pf37lKV5neTO278dAAAA5napQbC7f349GgEAAGB9XOp1BKvqWlX1N1V14nT766q61no0BwAAwPa3lgvKvyiLS0bcf7qdm+TFczYFAADAfNbyHcFbdPf9lqb/pKpOnqshAAAA5rWWPYL/VVU/s3Wiqu6Y5L/mawkAAIA5rWWP4COTHDV9L7CSnJ3kQbN2BQAAwGzWctbQk5P8eFVdc5o+d/auAAAAmM1azhq6R1U9K8m/JXlHVf2/qtpj9s4AAACYxVoODT06yTuTbD1hzK8kOSbJL8zVFGy0N73wF2cb+6CHHT/b2AAAsBZrCYLX7e4/XZp+alXde66GAAAAmNdazhr6jqo6rKp2mW73T/LGuRsDAABgHmvZI/jwJI9N8rJpepck36qqxybp7r7mXM3BSF754gNnG/v+D3nzbGMDALD5rOWsoT+wHo0AAACwPtayRzBVdZskey0v392vmaknAAAAZnSpQbCqXpTkNklOTXLRNLuTCIIAAACb0Fr2CP5Ud+87eycAAACsi7WcNfS9VSUIAgAA7CTWskfwqCzC4JeTfCdJZXG20NvM2hkAAACzWEsQfFGSByb5aC7+jiAAAACb1FqC4Oe6+7jZOwEAAGBdrCUIfrKqXp7k9VkcGprE5SMAAAA2q7UEwatmEQDvujTP5SMAAAA2qUsNgt39kPVoBAAAgPWxzSBYVb/f3U+vqmdnsQfwv+nu35y1MwAAAGZxSXsEPzH9e+J6NAIAAMD62GYQ7O7XT/8etX7tAAAAMLddNroBAAAA1tesQbCqDqyq06rq9Kp6/CqP715Vx0yPv7+q9lp67DZV9d6qOrWqPlpVV5mzVwAAgFHMFgSratckRyY5KMm+SQ6vqn1XLPawJOd09z5JnpnkadNzd0vysiSP6O5bJfm5JN+dq1cAAICRXGoQrKofrqq3VdXHpunbVNUT1zD2/klO7+4zuvuCJEcnOXjFMgcn2fodxFcnOaCqKotrFp7S3R9Jku7+z+7+3tpWCQAAgEuylj2Cz0/yhEx75Lr7lCSHreF5N0ry+aXpLdO8VZfp7guTfCPJHkl+OElX1Vuq6qSq+v011AMAAGANLvWC8kmu1t0fWOyo+74L1/C8WmXeyusRbmuZ3ZL8TJLbJzk/yduq6kPd/bb/9uSqI5IckSQ3velN19ASAAAAa9kj+LWqukWmEFdVhyT50hqetyXJTZamb5zki9taZvpe4LWSnD3NP6G7v9bd5yc5PslPrCzQ3c/r7v26e78999xzDS0BAACwliD4qCTPTfKjVfWFJL+d5BFreN4Hk9yyqvauqitncTjpcSuWOS7Jg6b7hyR5e3d3krckuU1VXW0KiD+b5ONrqAkAAMCluMRDQ6tqlyT7dfcvVNXVk+zS3eetZeDuvrCqHp1FqNs1yYu6+9SqekqSE7v7uCQvTPLSqjo9iz2Bh03PPaeq/iaLMNlJju/uN17OdQQAAGDJJQbB7r5oCnOv7O5vXdbBu/v4LA7rXJ73pKX7305y6Dae+7IsLiEBAADAdrSWk8W8taoel+SYJN8Pg9199mxdAbN7wUvuNtvYv/arb5ltbAAArri1BMGHTv8+amleJ7n59m8HAACAuV1qEOzuvdejEQAAANbHpQbBqvrV1eZ390u2fzsAAADMbS2Hht5+6f5VkhyQ5KQkgiAAAMAmtJZDQx+zPF1V10ry0tk6AgAAYFZruaD8SucnueX2bgQAAID1sZbvCL4+i7OEJovguG+SV83ZFAAAAPNZy3cEn7F0/8Ik/9HdW2bqBwAAgJmt5dDQX+zuE6bbe7p7S1U9bfbOAAAAmMVaguBdVpl30PZuBAAAgPWxzUNDq+qRSX4jyc2r6pSlh34gyXvmbgwAAIB5XNJ3BF+e5E1J/iLJ45fmn9fdZ8/aFQAAALPZZhDs7m8k+UaSw5Okqq6fxQXlr1FV1+juz61PiwAAAGxPl/odwaq6Z1V9Oslnk5yQ5Mws9hQCAACwCa3lZDFPTfJTST7V3XsnOSC+IwgAALBprSUIfre7/zPJLlW1S3e/I8ltZ+4LAACAmazlgvJfr6prJHlXkn+qqq9mcWF5AAAANqG17BE8OMn5SX47yZuTfCbJPedsCgAAgPlc6h7B7v5WVd0syS27+6iqulqSXedvDQAAgDms5ayhv57k1UmeO826UZLXzdkUAAAA81nLoaGPSnLHJOcmSXd/Osn152wKAACA+awlCH6nuy/YOlFVuyXp+VoCAABgTmsJgidU1R8muWpV3SXJq5K8ft62AAAAmMtaguDjk5yV5KNJHp7k+CRPnLMpAAAA5rPNs4ZW1U27+3PdfVGS5083AAAANrlL2iP4/TODVtWx69ALAAAA6+CSgmAt3b/53I0AAACwPi4pCPY27gMAALCJbfM7gkl+vKrOzWLP4FWn+5mmu7uvOXt3AAAAbHfbDILdvet6NgIAAMD6WMvlIwAAANiJCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMHsttENAOP4m5ffbbaxH/vLb5ltbACAnY09ggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACD2W2jGwCY0xNedeBsY//FoW+ebWwAgDnNukewqg6sqtOq6vSqevwqj+9eVcdMj7+/qvZa8fhNq+qbVfW4OfsEAAAYyWxBsKp2TXJkkoOS7Jvk8Krad8ViD0tyTnfvk+SZSZ624vFnJnnTXD0CAACMaM49gvsnOb27z+juC5IcneTgFcscnOSo6f6rkxxQVZUkVXXvJGckOXXGHgEAAIYzZxC8UZLPL01vmeatukx3X5jkG0n2qKqrJ/mDJH8yY38AAABDmjMI1irzeo3L/EmSZ3b3Ny+xQNURVXViVZ141llnXc42AQAAxjLnWUO3JLnJ0vSNk3xxG8tsqardklwrydlJ7pDkkKp6epJrJ7moqr7d3c9ZfnJ3Py/J85Jkv/32WxkyAQAAWMWcQfCDSW5ZVXsn+UKSw5L88opljkvyoCTvTXJIkrd3dyf5P1sXqKonJ/nmyhAIAADA5TNbEOzuC6vq0UnekmTXJC/q7lOr6ilJTuzu45K8MMlLq+r0LPYEHjZXPwDr5e7HzXftwjfey7ULAYArbtYLynf38UmOXzHvSUv3v53k0EsZ48mzNAcAADCoWS8oDwAAwI5HEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABiMIAgAADEYQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADAYQRAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMZreNbgCAK+4XX/cHs419/L2fNtvYAMDGEAQBuFzu/tqnzzb2G+/z+7ONDQDMfGhoVR1YVadV1elV9fhVHt+9qo6ZHn9/Ve01zb9LVX2oqj46/XvnOfsEAAAYyWxBsKp2TXJkkoOS7Jvk8Krad8ViD0tyTnfvk+SZSbYef/S1JPfs7lsneVCSl87VJwAAwGjm3CO4f5LTu/uM7r4gydFJDl6xzMFJjpruvzrJAVVV3f3h7v7iNP/UJFepqt1n7BUAAGAYcwbBGyX5/NL0lmneqst094VJvpFkjxXL3C/Jh7v7OysLVNURVXViVZ141llnbbfGAQAAdmZzBsFaZV5flmWq6lZZHC768NUKdPfzunu/7t5vzz33vNyNAgAAjGTOILglyU2Wpm+c5IvbWqaqdktyrSRnT9M3TvLaJL/a3Z+ZsU8AAIChzBkEP5jkllW1d1VdOclhSY5bscxxWZwMJkkOSfL27u6qunaSNyZ5Qne/Z8YeAQAAhjNbEJy+8/foJG9J8okkr+zuU6vqKVV1r2mxFybZo6pOT/LYJFsvMfHoJPsk+aOqOnm6XX+uXgEAAEYy6wXlu/v4JMevmPekpfvfTnLoKs97apKnztkbAJvP3V/z97ON/cb7PnK2sQFgRzPrBeUBAADY8QiCAAAAgxEEAQAABiMIAgAADEY2XrY9AAAU4ElEQVQQBAAAGIwgCAAAMBhBEAAAYDCCIAAAwGAEQQAAgMEIggAAAIMRBAEAAAYjCAIAAAxGEAQAABiMIAgAADCY3Ta6AQDYUd3j2KNmG/sN93vQbGMDwKWxRxAAAGAwgiAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjCAIAAAwGEEQAABgMIIgAADAYARBAACAwQiCAAAAgxEEAQAABrPbRjcAAFzsHq8+Zrax33DIL802NgCbiz2CAAAAgxEEAQAABuPQUAAY3L1e/frZxj7ukHvONjYAl58gCACsu3sf+/bZxn7d/e4829gAOwuHhgIAAAxGEAQAABiMQ0MBgCHc79gTZxv72PvtN9vYAHMQBAEAZnLYa86cbeyj77vXbGMDOz9BEABgJ/JXr/3ybGP/3n1uONvYwPryHUEAAIDB2CMIAMDldvSxX5t1/MPud71Zx4dR2SMIAAAwGHsEAQDYVN76irNmG/suh+8529iwI7FHEAAAYDD2CAIAwKU48UVfnW3s/R56/dnGhm0RBAEAYAf0mWd/Zbaxb/GYG8w2NpuDIAgAACRJvvyMz8429g0ft/eq87/yzA/PVvMGv3O72cbe7HxHEAAAYDD2CAIAAEP5yrNOmG3sG/zmz8429vZkjyAAAMBgBEEAAIDBCIIAAACDEQQBAAAGIwgCAAAMRhAEAAAYjMtHAAAAzOirR75h1vGv/6h7XObn2CMIAAAwGEEQAABgMIIgAADAYGYNglV1YFWdVlWnV9XjV3l896o6Znr8/VW119JjT5jmn1ZVd5uzTwAAgJHMFgSratckRyY5KMm+SQ6vqn1XLPawJOd09z5JnpnkadNz901yWJJbJTkwyd9N4wEAAHAFzblHcP8kp3f3Gd19QZKjkxy8YpmDkxw13X91kgOqqqb5R3f3d7r7s0lOn8YDAADgCpozCN4oyeeXprdM81ZdprsvTPKNJHus8bkAAABcDtXd8wxcdWiSu3X3r03TD0yyf3c/ZmmZU6dltkzTn8liz99Tkry3u182zX9hkuO7+9gVNY5IcsQ0+SNJTruc7V4vydcu53MvrxFqjrCOaqq5GWuOsI5qqrnZ6qmp5masOcI6braaN+vuPdey4JwXlN+S5CZL0zdO8sVtLLOlqnZLcq0kZ6/xuenu5yV53hVttKpO7O79rug4am5sPTXVVHPHrKemmpux5gjrqKaam62emtvXnIeGfjDJLatq76q6chYnfzluxTLHJXnQdP+QJG/vxS7K45IcNp1VdO8kt0zygRl7BQAAGMZsewS7+8KqenSStyTZNcmLuvvUqnpKkhO7+7gkL0zy0qo6PYs9gYdNzz21ql6Z5ONJLkzyqO7+3ly9AgAAjGTOQ0PT3ccnOX7FvCct3f92kkO38dw/S/Jnc/a35AofXqrmDlFPTTXV3DHrqanmZqw5wjqqqeZmq6fmdjTbyWIAAADYMc35HUEAAAB2QIIgAADAYARBAACAwQiCbBdVda+quspG97EzqqprVNUhVfU7VfWYqjqwqtb1d7eq/nwdatywqm443d+zqu5bVbeauy47n+myRfetqh+duc41q+oWq8y/zcx137aWebBSVd2pqn5kuv8zVfW4qrr7jPVuuvVvg1p4SFU9u6oeOV0/mhlU1V02ugc2h6GCYFXtWlUPr6o/rao7rnjsiTPVfHRVXW+6v09VvbOqvl5V76+qW89U8zVV9YCqusYc42/DMUm2VNVLq+oXq2rXOYtV1W7Tz/LNVXVKVX2kqt5UVY+oqivNWXtFH5+aefz7J3lHkgOTPDrJ/kkemOTkGV8/z1pxe3aS39g6PVPNhyd5b5L3VdUjk7whyT2SvKaqHjZTzZtX1Yuq6qlT2H5+VX2sql5VVXvNUfNS+tnuZwfbiPefS+lnljOgVdXrlu4fnOTtSe6Z5J+r6sEz1bx/kk8mObaqTq2q2y89/I8z1bxKVV03yfWq6jpVdd3ptleSH5qp5rp/nqz3Z/VAnyd/m+Qvs7hs158meXqSqyb5nar6q5nKHp+L/9b8yyR3T/L+JLfPTGdE3Ij39qrapaoeWlVvnF4/H6qqo6vq5+aotwYvnGPQ5f/kqqorVdUTq+q4qvrzqrraDPWuVlW/X1W/N73/PXiq9/S53pPWex2nOuu+nt+vPdJZQ6vqBUmulsXF6R+Y5ITufuz02End/RMz1Dy1u2813X9jkhd092unN4c/6+47XuIAl6/mF7L4o/rOSf41ySuSvLG7L9jetZZqfniqd0gW14P8sSSvTfKK7j5hhnqvSPL1JEcl2TLNvnGSByW5bnf/0gw1z0uy9Rempn+vluT8JN3d15yh5ilJfqq7z5/+oP+n7r7b9Eb1D9390zPU3JLk35L8Sy5ez2ckeVySdPdRM9T8aJI7ZPFHyX8k2ae7v1xV10nyju6+7Qw135nF78a1kjwgyYuTvDLJXZP8SnffeYaa193WQ0k+0t033s71NuL9Z13Xcar54e6+3XT/37P4+X12+p15W3f/+Aw1T05yUHd/qar2T/KSJH/Y3a9Z7mc71/ytJL+dRej7Qi7+/Tw3yfO7+zkz1NyIz5N1/awe6PPk1Cw+m6+axevnRtNny5WSfLi7f2yGmh/v7n2n+x9Kcvvuvmia/shMv5sb8d7+4iw+u/41i7+Dzk3yriR/kOSfu/vZM9Q8blsPJblzd199hprf//2rqr9OskcW2/feSfbo7l/dzvVemeTzWbxmfyTJJ7L4Wd4zyQ27+4Hbs95Uc13Xcaqz7uv5fd09zC3JKUv3d8vif6Nek2T3LN4E56h52tL9D26rn+1c88PTvz+QxYfo8UnOyuKFfNeZap60YvqGSX4ziz8gPj/ndl3lsU/NtI7PzuKPvRsszfvsHLWWxv9oLv4Pm6suv06TfGymmtdM8rdJXp7FHwpJcsbM67m8Xh/Z1mMz1vzcOtX8XpIzknx26bZ1+oIZ6m3E+8+6ruNU86Sl+x9Yp5/lx1ZM/2CSD03veyfNUXOp1mPmHH+17bfOnyfr+lk90OfJx6Z/r5LknCRXnaZ3TfLxmWq+JYtQkiTHJrnZdH+Ple/127HmRry3n7Ji+n3Tv7sn+cRMNc/JYg/rz664/VySr6zDtj05yZWm+zXHZ0qSk5fG/3Iu/ntolnobsY4btZ5bb6Mdn33lrXe6+8IkR1TVk7I4jGiuXa+vrqp/TPKUJK+tqt/O4gPtgCSfm6lmJ0l3n5fkpVkcBnLdJPdP8vgs9vTMqru/nORZSZ5VVTebocQ5VXVokmP74v9d3CXJoVm8OW533f2YqvrJJK+oxaFoz8nF/6M7lzcmeXNVnZDkoCSvSr6/16Uu6YmXV3efm+S3p3V92bQnae7DyL9XVVfq7u9m8cGWZHEo3Iy1L6qqH87if42vVlX7dfeJVbVPFn8YzeGMJAd09//43a+qz89QbyPef9Z7HZPkNlV1bha/E1epqhv2Yo/ylTPfz/LcqrpFd38mSXqxZ/Dnkrwuyazfbe3uZ1fVTyfZK7n4c7y7XzJHuWns9fw8We/P6mE+T6rq3VmEkxckeWVVvS+L8PDOmWr+WpKXVNWTk3wji681fDjJdZI8dqaaG/He/t2t7wdV9RNJLkiS7v5OVc31c31fkvN7laOuquq0mWpeq6ruk8Xn8u7TZ3a6u2dcz63jH99TOpq53oas41KN9VrPJBkuCJ5YVQd295u3zujup1TVF5P8/RwFu/v/1uI7Kq9Icoss3oCPyOKPhV+Zo2aSb67Sx9lJ/mG6zeEqVfXT3f3vq9T+jxnqHZbkaUn+rqq2flBfO4vv0x02Q70kSXd/qKp+IYvv652Qxf+szukHkrw7yXeS/El3/+s0/+tJtvuhzElSVc9J8vLu/vequnOS35h6mNMpWRwa+u7u3rI0f48kvztTzd9P8vokF2VxyMcTqurHs9gj+usz1fzbLP4AWi2EPX17F9ug9591XcfJc7N4zb5nxfyrJXn4TDXPyeIQzc9sndHd51XVgVmEpNlU1Uuz+HmenMUe2GQRIuYIghvxebLen9WjfJ5cI8kTstgz//5anOjoPlmEwlfPVPMPkjwxi9+XW2bx/dktWRyhcNFMNTfivf33kryjqr6d5EqZXjdVtWcW33mfwxmZAudK3X2nmWqekORe0/33VdUNuvsrtTjR29dmqHdiVV2ju7/Z3Q/dOnN67Z43Q71k/dcx2Zj1XNSYQidcIdN3Vw7L4vCoY7L4buDJ61R7jyxey3P9gm6r7g8muV13Hz9jja3b9YeSHJ112K4b8bPcyNfPij6ul+Sc7v7epS7MDmO012xVfSLJvu0DfLsb5PNkiN+TFX3M/t5eVZXFd8jW5bWzo2zbjVJVNcJ74NzrKQhOquou3f1WNa/w+DfL4o3psCz+d/MVSY7u7lnPhraiB9tVzSvax07zGqqqaybZc+shjEvzb9Pdp2zvehtVcxp/R3nNvqK7Pz1jzVcl+c3u/tJcNVbU2+lfQyOs49L4O8rvyWw1N2rbbqOXne7vg230sa6fm3PWG+n9YLYvH262W1Z8oVjN7VLrdkk+nOR7O+s6jrJdR6m5ET/POWtmcYjiF7M4hPDULM7Yt/WxWU5oshE1d5TXz3rVzOKQxXOyOBHHcVtvO8vPc71rjrCOl9DHTvd7sqNs26WaO/XfBxuxnnPWG+39YKjvCNYln2p3DzW3S+0rZXHNu8OyOCHFCUn+ZIY6tusM23WkmoO8hv4wyU/2xZc4eGlV/WF3vyYznWxog2om2flfs0uePPP4y0Z4DY2wjt83wO/Jum/bUf4+WO/13KDtOtT7wVBBMMn/yeKaMiu//F5ZXKhbzcupqu6S5PAszvj4gSy+z3ZEd39rjnqxXefarsPUzBivod16Onywuz9QVT+f5A1VdePMd4bCda850Gs2SdIzXJv1EozwGhphHUf6PdmIn+cQfx9k/ddzIz6nh3g/+H7hOQffAW3EqXZHqfmHWVx37nG9OKPc3GzX+YxSc4TX0EZc4mAjao7ymk2SlRcjv3IWZyn8Vs9wEfKM8RoaYR2TcX5PNmLbjvL3wXqv50Zs11HeD5KMFwQ34lS7Q9Ts7p+fY9xLYLvOZJSaGeM1tBGXOFj3mgO9ZrfW/oHl6aq6d+b73/ERXkMjrONIvycb8fMc4u+DrP96bsTn9BDvB1vNfZHoHc2nkjyjqs6sqqdV1W3V3LRsV66oEV5D/5Lk6Svrdfd3u/ufdqKaQ+vu1yW580zDj/AaGmEdR7IR23aUvw/Wez03YrsO9X4w5OUjaoBTJ29UzfVmu3JFjfAa2ka9uS9xsO41R1FV912a3CXJfkl+trv/94w1d/rX0AjrOJId6Oe50/19sIN8hu2MlwZa/5ojBsFlVXW7JC9Kcpvu3lXNzct25Yoa4TU0wjru7KrqxUuTFyY5M8nzu/ur61R/p38NjbCOI/HznI/fzc1dc7RDQ5MsTrVbVfesqn9K8qYsdj3fT83Nx3blihrhNTTCOo6kux+ydPv17v6zuUPgCK+hEdZxJH6e8/G7uRPVHGmPYK1+qt3X9fqfOnmnq7nebFeuqBFeQyOs44hqcUrxZye5YxZnD313kt/q7i0z1NrpX0MjrONI/Dzn43dz56n5/dqDBcF3ZHGq3WPX61S7o9Rcb7YrV9QIr6ER1nFEVfXWLLbxS6dZD0jyK919lxlq7fSvoRHWcSR+nvPxu7nz1Px+7ZGCIABsdlV1cnff9tLmAcAlGfI7ggCwiX2tqh5QVbtOtwck+c+NbgqAzcUeQQDYRKrqpkmek+R/Z/EdwX/P4juC/7GhjQGwqQiCAAAAg9ltoxsAANauqvZO8pgke2Xpc7y777VRPQGw+QiCALC5vC7JC5O8PslFG9wLAJuUQ0MBYBOpqvd39x02ug8ANjdBEAA2kar65SS3TPIvSb6zdX53n7RhTQGw6Tg0FAA2l1sneWCSO+fiQ0N7mgaANbFHEAA2kar6ZJLbdPcFG90LAJuXC8oDwObykSTX3ugmANjcHBoKAJvLDZJ8sqo+mIu/I9jdffAG9gTAJuPQUADYRKrqZ5cnk/xMksO7+1Yb1BIAm5BDQwFgE+nuE5J8I8ndk/xjkgOS/MNG9gTA5uPQUADYBKrqh5McluTwJP+Z5Jgsjuz5+Q1tDIBNyaGhALAJVNVFSd6V5GHdffo074zuvvnGdgbAZuTQUADYHO6X5MtJ3lFVz6+qA7L4jiAAXGb2CALAJlJVV09y7ywOEb1zkqOSvLa7/2VDGwNgUxEEAWCTqqrrJjk0yS919503uh8ANg9BEAAAYDC+IwgAADAYQRAAAGAwgiAAw6uq71XVyUu3vS7HGNeuqt/Y/t0BwPbnO4IADK+qvtnd17iCY+yV5A3d/WOX8Xm7dvf3rkhtALis7BEEgFVU1a5V9VdV9cGqOqWqHj7Nv0ZVva2qTqqqj1bVwdNT/jLJLaY9in9VVT9XVW9YGu85VfXg6f6ZVfWkqnp3kkOr6hZV9eaq+lBVvauqfnS91xeAsey20Q0AwA7gqlV18nT/s919nyQPS/KN7r59Ve2e5D1V9S9JPp/kPt19bv3/9u7YpasoDOP49wlCIcSlcGhoykGigoJAJIyg1cUmx5Ya2vsLWtqDoLWh3Noc2hskUSyChoqagiCKKDB6He4VNC4aym86389yufe+577c8eGcw0lOAi+TPAfuAeeq6iJAkvkDev6uqrm+9gVwu6reJbkCPKQ7I1CSpJEwCEqSBL92AtwuN4DzSRb7+0ngLPAZuJ/kKvAXOA1MHaLnU+hmGIFZYDnJzruxQ3xPkqT/ZhCUJGlYgLtVtbLnYbe88xRwqaq2knwAxgfG/2HvFox/a37212PAt4EgKknSyLhHUJKkYSvAnSTHAZJMJzlBNzP4pQ+B14Azff0PYGLX+I/ATJKxJJPA9aEmVfUdeJ/kZt8nSS6M5pckSeoYBCVJGvYYeAO8SrIJPKJbSfMEuJxkFVgC3gJU1Ve6fYSbSR5U1SfgGbDRj1nbp9cScCvJOvAaWNinVpKkI/P4CEmSJElqjDOCkiRJktQYg6AkSZIkNcYgKEmSJEmNMQhKkiRJUmMMgpIkSZLUGIOgJEmSJDXGIChJkiRJjTEISpIkSVJjtgFASxC6AuGFPAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1080x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.gridspec as gridspec\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "from pyspark.mllib.evaluation import MulticlassMetrics\n",
    "trainData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "testData = sqlContext.read.csv(\"base_test.csv\", header = True)\n",
    "trainData = trainData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainData.columns))\n",
    "testData = testData.select(*(F.col(c).cast(\"double\").alias(c) for c in testData.columns))\n",
    "train_count = trainData.select(\"Class\").count()\n",
    "train_fraud_count = trainData.select(\"Class\").where(trainData.Class == 1).count()\n",
    "test_count = testData.select(\"Class\").count()\n",
    "print(\"train before sampling: \" + str(train_count))\n",
    "print(\"fraud ratio: \" + str(train_fraud_count/train_count))\n",
    "print(\"test: \" + str(test_count))\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainData)\n",
    "df_test = vectorAssembler.transform(testData)\n",
    "\n",
    "rfClassifier = RandomForestClassifier(featuresCol = 'features', labelCol = 'Class', maxDepth=6, minInfoGain=0.0, numTrees= 200)\n",
    "rfModel = rfClassifier.fit(df_train)\n",
    "pred = rfModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class == 1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)\n",
    "evaluator = BinaryClassificationEvaluator(rawPredictionCol=\"rawPrediction\", labelCol=\"Class\",metricName=\"areaUnderPR\")\n",
    "print(\"Area under Precision Recall Curve = %s\" % evaluator.evaluate(pred))\n",
    "predictionAndLabels = pred.select('Class','prediction').rdd.map(lambda lp: (float(lp.prediction), lp.Class))\n",
    "metrics = MulticlassMetrics(predictionAndLabels)\n",
    "print(metrics.confusionMatrix().toArray())\n",
    "\n",
    "featureImportance = rfModel.featureImportances\n",
    "print(featureImportance)\n",
    "tmp = pd.DataFrame({'Feature': features, 'Feature importance': featureImportance})\n",
    "tmp = tmp.sort_values(by='Feature importance',ascending=False)\n",
    "plt.figure(figsize = (15,8))\n",
    "plt.title('Features importance',fontsize=14)\n",
    "s = sns.barplot(x='Feature',y='Feature importance',data=tmp)\n",
    "s.set_xticklabels(s.get_xticklabels(),rotation=90)\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STEP 5   Based on features important, stratified cv with top features and best hyperparams & features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8538245088245089, 0.759862506610259, 0.804107891152442]\n",
      "grid search precision: 0.8538245088245089\n",
      "grid search recall: 0.759862506610259\n",
      "grid search f-score: 0.804107891152442\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8572899030793767, 0.763088313061872, 0.807450877121384]\n",
      "grid search precision: 0.8572899030793767\n",
      "grid search recall: 0.763088313061872\n",
      "grid search f-score: 0.807450877121384\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2', 'Amount', 'V27', 'V22', 'V20']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.9064276484655485, 0.759862506610259, 0.8266992191315437]\n",
      "grid search precision: 0.9064276484655485\n",
      "grid search recall: 0.759862506610259\n",
      "grid search f-score: 0.8266992191315437\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2', 'Amount', 'V27', 'V22', 'V20']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2', 'Amount', 'V27', 'V22', 'V20', 'V24', 'V8', 'V6', 'V28']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.9234993447568183, 0.7598096245372818, 0.8336956592043311]\n",
      "grid search precision: 0.9234993447568183\n",
      "grid search recall: 0.7598096245372818\n",
      "grid search f-score: 0.8336956592043311\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4', 'V3', 'V18', 'V11', 'V26', 'V21', 'V5', 'V2', 'Amount', 'V27', 'V22', 'V20', 'V24', 'V8', 'V6', 'V28']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8614779372674111, 0.7695399259650978, 0.8129177282601269]\n",
      "grid search precision: 0.8614779372674111\n",
      "grid search recall: 0.7695399259650978\n",
      "grid search f-score: 0.8129177282601269\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10', 'V14', 'V9', 'V4']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## features = ['V12', 'V16', 'V17', 'V7', 'V10']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8281137602033125, 0.733897408778424, 0.7781641448609568]\n",
      "grid search precision: 0.8281137602033125\n",
      "grid search recall: 0.733897408778424\n",
      "grid search f-score: 0.7781641448609568\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V12', 'V16', 'V17', 'V7', 'V10']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Based on T test, remove V13, V22, V23, V25, V26"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145268\n",
      "fraud ratio: 0.001700305641985847\n",
      "train: 145266\n",
      "fraud ratio: 0.0016934451282474908\n",
      "train: 145270\n",
      "fraud ratio: 0.0017002822330832244\n",
      "[0.8556228282057802, 0.759862506610259, 0.8049044989039639]\n",
      "grid search precision: 0.8556228282057802\n",
      "grid search recall: 0.759862506610259\n",
      "grid search f-score: 0.8049044989039639\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "features = ['V1','V2','V3','V4','V5','V6','V7','V8','V9','V10','V11','V12','V14','V15','V16','V17','V18','V19','V20','V21','V24','V27','V28','Amount']\n",
    "gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'None', 0, False, features, 'rf', [6], [0], [100])\n",
    "print(\"grid search precision:\", gs_precision)\n",
    "print(\"grid search recall:\", gs_recall)\n",
    "print(\"grid search f-score:\", gs_fscore)\n",
    "print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conclusion: All features are the best"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## STEP6 undersampling with all features."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 82333\n",
      "fraud ratio: 0.003000012145798161\n",
      "train: 82000\n",
      "fraud ratio: 0.003\n",
      "train: 82000\n",
      "fraud ratio: 0.003\n",
      "train: 82000\n",
      "fraud ratio: 0.003\n",
      "train: 82333\n",
      "fraud ratio: 0.003000012145798161\n",
      "[0.8566827697262479, 0.763088313061872, 0.8071814795635384]\n",
      "grid search precision: 0.8566827697262479\n",
      "grid search recall: 0.763088313061872\n",
      "grid search f-score: 0.8071814795635384\n",
      "grid search hyper: [6, 0, 100]\n",
      "[6, 0, 100]\n",
      "train: 49200\n",
      "fraud ratio: 0.005\n",
      "train: 49399\n",
      "fraud ratio: 0.0050001012166238186\n",
      "train: 49200\n",
      "fraud ratio: 0.005\n",
      "train: 49200\n",
      "fraud ratio: 0.005\n",
      "train: 49399\n",
      "fraud ratio: 0.0050001012166238186\n",
      "[0.8407556899182979, 0.7695399259650978, 0.8035730396241875]\n",
      "grid search precision: 0.8407556899182979\n",
      "grid search recall: 0.7695399259650978\n",
      "grid search f-score: 0.8035730396241875\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "#undersampling\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "for i in [0.003, 0.005]:\n",
    "        gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'under', i, False, features, 'rf', [6], [0], [100])\n",
    "        print(\"grid search precision:\", gs_precision)\n",
    "        print(\"grid search recall:\", gs_recall)\n",
    "        print(\"grid search f-score:\", gs_fscore)\n",
    "        print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "        sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Oversampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[6, 0, 100]\n",
      "train: 145456\n",
      "fraud ratio: 0.002997470025299747\n",
      "train: 145456\n",
      "fraud ratio: 0.002997470025299747\n",
      "train: 145457\n",
      "fraud ratio: 0.0029974494180410706\n",
      "train: 145456\n",
      "fraud ratio: 0.002997470025299747\n",
      "train: 145459\n",
      "fraud ratio: 0.002997408204373741\n",
      "[0.8559770955165693, 0.7565838180856689, 0.8032173093828963]\n",
      "grid search precision: 0.8559770955165693\n",
      "grid search recall: 0.7565838180856689\n",
      "grid search f-score: 0.8032173093828963\n",
      "grid search hyper: [6, 0, 100]\n",
      "[6, 0, 100]\n",
      "train: 145748\n",
      "fraud ratio: 0.004994922743365261\n",
      "train: 145748\n",
      "fraud ratio: 0.004994922743365261\n",
      "train: 145749\n",
      "fraud ratio: 0.004994888472648183\n",
      "train: 145748\n",
      "fraud ratio: 0.004994922743365261\n",
      "train: 145751\n",
      "fraud ratio: 0.004994819932624819\n",
      "[0.8502824262317056, 0.7629825489159175, 0.8042704240886565]\n",
      "grid search precision: 0.8502824262317056\n",
      "grid search recall: 0.7629825489159175\n",
      "grid search f-score: 0.8042704240886565\n",
      "grid search hyper: [6, 0, 100]\n"
     ]
    }
   ],
   "source": [
    "#oversampling\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V13', 'V14', 'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V23', 'V24', 'V25', 'V26', 'V27', 'V28']\n",
    "for i in [0.003, 0.005]:\n",
    "        gs_hyper, gs_precision, gs_recall, gs_fscore = tune(df, 5, True, 'over', i, False, features, 'rf', [6], [0], [100])\n",
    "        print(\"grid search precision:\", gs_precision)\n",
    "        print(\"grid search recall:\", gs_recall)\n",
    "        print(\"grid search f-score:\", gs_fscore)\n",
    "        print(\"grid search hyper:\", gs_hyper)\n",
    "\n",
    "\n",
    "        sqlContext.clearCache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## No sampling is better."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## step7 Extrapolation result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train before sampling: 181584\n",
      "fraud ratio: 0.0016961846858754076\n",
      "test: 56746\n",
      "Precision is  0.9615384615384616\n",
      "Recall is  0.6756756756756757\n",
      "F1 score is  0.7936507936507937\n"
     ]
    }
   ],
   "source": [
    "trainData = sqlContext.read.csv(\"base_train.csv\", header = True)\n",
    "testData = sqlContext.read.csv(\"oot.csv\", header = True)\n",
    "trainData = trainData.select(*(F.col(c).cast(\"double\").alias(c) for c in trainData.columns))\n",
    "testData = testData.select(*(F.col(c).cast(\"double\").alias(c) for c in testData.columns))\n",
    "train_count = trainData.select(\"Class\").count()\n",
    "train_fraud_count = trainData.select(\"Class\").where(trainData.Class == 1).count()\n",
    "test_count = testData.select(\"Class\").count()\n",
    "print(\"train before sampling: \" + str(train_count))\n",
    "print(\"fraud ratio: \" + str(train_fraud_count/train_count))\n",
    "print(\"test: \" + str(test_count))\n",
    "features = ['Amount', 'V1', 'V2', 'V3', 'V4', 'V5', 'V6', 'V7', 'V8', 'V9', 'V10', 'V11', 'V12', 'V14', \n",
    "            'V15', 'V16', 'V17', 'V18', 'V19', 'V20', 'V21', 'V22', 'V24', 'V26', 'V27', 'V28']\n",
    "vectorAssembler = VectorAssembler(inputCols = features, outputCol = 'features')\n",
    "df_train = vectorAssembler.transform(trainData)\n",
    "df_test = vectorAssembler.transform(testData)\n",
    "rfClassifier = RandomForestClassifier(featuresCol = 'features', labelCol= 'Class', maxDepth=6, minInfoGain=0.0, numTrees= 100)\n",
    "rfModel = rfClassifier.fit(df_train)\n",
    "pred = rfModel.transform(df_test)\n",
    "pred = pred.select(\"features\", \"Class\", \"rawPrediction\", \"prediction\")\n",
    "precision = pred.filter(pred.Class == pred.prediction).filter(pred.Class== 1.0).count() / pred.filter(pred.prediction == 1.0).count()\n",
    "recall = pred.filter(pred.Class == pred.prediction).filter(pred.Class ==1.0).count() / pred.filter(pred.Class == 1.0).count()\n",
    "f1 = 2 * precision * recall /(precision + recall)\n",
    "print(\"Precision is \", precision)\n",
    "print(\"Recall is \", recall)\n",
    "print(\"F1 score is \", f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}