run.py

import statistics

import deap.gp
import numpy
import numpy as np
from time import process_time
import sklearn.metrics

from src.utils import fetch_dataset
from src.models.classifiers import mlp

import operator

from deap import algorithms
from deap import gp
from deap import creator, base, tools

X_train = []
X_test = []
y_train = []
y_test = []
opaque_model_prediction_test = []
opaque_model_prediction_train = []
toolbox = None
mlp_time = 0
time_start = 0
time_end = 0
mlp_classifier = None
dataset_name = ''
logbook = tools.Logbook()


def protectedDiv(left, right):
    try:
        if right == 0:
            right = 1
        return left / right
    except ZeroDivisionError:
        return 1


def setUpGP(n_parameters: int, evaluate_function):
    global toolbox

    pset = gp.PrimitiveSet("MAIN", n_parameters)
    pset.addPrimitive(operator.add, 2)
    pset.addPrimitive(operator.sub, 2)
    pset.addPrimitive(operator.mul, 2)
    pset.addPrimitive(protectedDiv, 2)
    pset.renameArguments(ARG0='x', ARG1='y', ARG2='z', ARG3='t')

    creator.create("FitnessMulti", base.Fitness, weights=(1.0, -1.0, -1.0))
    creator.create("Individual", gp.PrimitiveTree, fitness=creator.FitnessMulti)

    toolbox = base.Toolbox()
    toolbox.register("expr", gp.genHalfAndHalf, pset=pset, min_=1, max_=2)
    toolbox.register("individual", tools.initIterate, creator.Individual, toolbox.expr)
    toolbox.register("population", tools.initRepeat, list, toolbox.individual)
    toolbox.register("compile", gp.compile, pset=pset)

    toolbox.register("evaluate", evaluate_function)
    toolbox.register("select", tools.selNSGA2)
    toolbox.register("mate", gp.cxOnePoint)
    toolbox.register("expr_mut", gp.genFull, min_=0, max_=2)
    toolbox.register("mutate", gp.mutUniform, expr=toolbox.expr_mut, pset=pset)

    toolbox.decorate("mate", gp.staticLimit(key=operator.attrgetter("height"), max_value=50))
    toolbox.decorate("mutate", gp.staticLimit(key=operator.attrgetter("height"), max_value=50))

    return toolbox


def fitness_function(individual):
    # Evaluate fitness of an individual within a generation.
    import math
    func = toolbox.compile(expr=individual)
    split_points = 0
    countDivision = 0
    countMult = 0
    countPrimitive = 0
    countTerminals = 0
    complexity = 0
    for i in individual.copy():
        split_points += i.arity if type(i) == deap.gp.Primitive else 0
        countPrimitive += 1 if type(i) == deap.gp.Primitive else 0
        complexity += getComplexityFactor(i.name) if type(i) == deap.gp.Primitive else 0
        countTerminals += 1 if type(i) == deap.gp.Terminal else 0
        countDivision += 1 if type(i) == deap.gp.Primitive and i.name == 'protectedDiv' else 0
        countMult += 1 if type(i) == deap.gp.Primitive and i.name == 'mul' else 0
    y_pred = []

    for x in enumerate(X_train):
        function_result = int(func(*x[1]) > 0.5)
        y_pred.append(function_result)

    # avgTreeLength = individual.__len__() / split_points if split_points != 0 else 0
    ari = 1 / (1 + math.exp(-(countTerminals * countPrimitive)))
    complextTerminals = 1 / (1 + math.exp(-complexity))

    return sklearn.metrics.f1_score(opaque_model_prediction_train, y_pred), ari, complextTerminals


def getComplexityFactor(primitive):
    if primitive == 'add':
        return 1
    elif primitive == 'sub':
        return 1
    elif primitive == 'mul':
        return 5
    elif primitive == 'protectedDiv':
        return 10


# Calculates score for test dataset in comparison with black-box
def calculateScore(individuals, pareto):
    global opaque_model_prediction_test
    hallOfFame = []
    f1_score_list = []
    f1_score_sum = 0
    hof_height_sum = []
    hof_node_sum = []

    for i in individuals:
        func = toolbox.compile(expr=i)
        y_gp = []
        for x in enumerate(X_test):
            function_result = int(func(*list(x[1])) > 0.5)
            y_gp.append(function_result)

        gp_f1score = sklearn.metrics.f1_score(opaque_model_prediction_test, np.array(y_gp))
        gp_accuracy_score = sklearn.metrics.accuracy_score(opaque_model_prediction_test, np.array(y_gp))
        hallOfFame.append((gp_f1score, i, gp_accuracy_score))
        f1_score_sum += gp_f1score

    for individual in hallOfFame:
        hof_height_sum.append(individual[1].height)
        hof_node_sum.append(individual[1].__len__())

    hallOfFame.sort(key=lambda x: x[0], reverse=True)

    mlp_fscore = sklearn.metrics.f1_score(y_test, opaque_model_prediction_test)
    mlp_accuracy = sklearn.metrics.accuracy_score(y_test, opaque_model_prediction_test)

    return hallOfFame[0][0], hallOfFame[0][1], mlp_fscore, hallOfFame[0][2], \
           mlp_accuracy, \
           sum(hof_height_sum) / len(hallOfFame), sum(hof_node_sum) / len(hallOfFame), \
           pareto,


def executeGeneticProgramming():
    global X_train
    global logbook
    pareto = tools.ParetoFront()
    generation = 40

    toolbox = setUpGP(len(X_train[0]), fitness_function)

    pop = toolbox.population(n=300)
    hof = tools.HallOfFame(10)
    fscore_stats = tools.Statistics(lambda ind: ind.fitness.values[0])
    # avgTree_stats = tools.Statistics(lambda ind: ind.fitness.values[1])
    ari_stats = tools.Statistics(lambda ind: ind.fitness.values[1])
    complexTerminals_stats = tools.Statistics(lambda ind: ind.fitness.values[2])
    mstats = tools.MultiStatistics(fscore_stats=fscore_stats, ari_stats=ari_stats,
                                   complexTerminals_stats=complexTerminals_stats)
    mstats.register("avg", numpy.mean)
    mstats.register("std", numpy.std)
    mstats.register("min", numpy.min)
    mstats.register("max", numpy.max)

    logbook.header = ["gen", "evals"] + mstats.fields

    algorithms.eaSimple(pop, toolbox, 0.5, 0.1, generation, mstats, halloffame=hof, verbose=True)
    pareto.update(pop)

    logbook.record(gen=generation, evals=len(pop), **mstats.compile(pop))

    return calculateScore(hof, pareto),


def generateReport(n_experiments, best_pareto=None):
    import pandas as pd
    global logbook
    global dataset_name

    gp_fscore_sum = []
    gp_accuracy_sum = []
    gp_height_sum = []
    gp_node_sum = []

    mlp_fscore_sum = []
    mlp_accuracy_sum = []

    best_gp_fscore = 0
    best_gp_function = None
    gp_sum_time = 0
    dt_sum_time = 0
    best_pareto
    accumulatedPareto = []
    for i in range(0, n_experiments):
        time_start = process_time()
        gp_fscore, gp_function, mlp_fscore, accuracy_score, mlp_accuracy, gp_height, gp_node, pareto_ = \
            executeGeneticProgramming()[0]
        time_end = process_time()
        gp_sum_time += time_end - time_start

        accumulatedPareto.append(pareto_)

        mlp_fscore_sum.append(mlp_fscore)
        mlp_accuracy_sum.append(mlp_accuracy)

        gp_fscore_sum.append(gp_fscore)
        gp_accuracy_sum.append(accuracy_score)
        gp_height_sum.append(gp_height)
        gp_node_sum.append(gp_node)

        if gp_fscore > best_gp_fscore:
            best_gp_fscore = gp_fscore
            best_pareto = pareto_

    fit_max = logbook.chapters["fscore_stats"].select("max")

    import matplotlib.pyplot as plt
    import seaborn as sns
    sns.set()
    fit_max_whats_inside = [0.8635875402792695, 0.9001029866117405, 0.8635875402792695, 0.8013300083125521,
                            0.8320855614973262, 0.8400392541707556, 0.8626198083067093, 0.8320855614973262,
                            0.9134808853118711, 0.8320855614973262, 0.8320855614973262, 0.8253119429590018,
                            0.8583690987124464, 0.8181011535048802, 0.8120603015075376, 0.7885714285714287,
                            0.7596717467760844, 0.7989179440937783, 0.8839662447257384, 0.8030467899891185,
                            0.7735644637053087, 0.8013300083125521, 0.760655737704918, 0.8013300083125521,
                            0.8367129135538954, 0.7860508953817154, 0.8635875402792695, 0.8413793103448276,
                            0.7804444444444445, 0.8635875402792695]
    fit_max_const = [0.856868395773295, 0.8580576307363929, 0.7688734030197446, 0.7688734030197446, 0.7608695652173915,
                     0.8094262295081968, 0.8590604026845639, 0.779107725788901, 0.8053024026512013, 0.9213483146067416,
                     0.8405017921146953, 0.7688734030197446, 0.8601921024546425, 0.7688734030197446, 0.7688734030197446,
                     0.9475890985324948, 0.8664495114006515, 0.8153681963713981, 0.7688734030197446, 0.8137472283813747,
                     0.8053024026512013, 0.8370370370370371, 0.784, 0.8342361863488624, 0.8545454545454545,
                     0.8129370629370629, 0.775609756097561, 0.9243856332703213, 0.8053024026512013, 0.7874493927125505]
    fit_max_gpx = [0.7706855791962175, 0.7241379310344829, 0.8466483011937557, 0.7706855791962175, 0.8436363636363636,
                   0.7706855791962175, 0.8072805139186296, 0.7936170212765957, 0.783573806881243, 0.8645720476706391,
                   0.8077858880778589, 0.9863013698630138, 0.6672597864768682, 0.8645720476706391, 0.7770419426048566,
                   0.7706855791962175, 0.7706855791962175, 0.8244111349036403, 0.8194130925507901, 0.7706855791962175,
                   0.8307045215562566, 0.655072463768116, 0.8008342022940564, 0.8152059134107709, 0.8645720476706391,
                   0.831389183457052, 0.8645720476706391, 0.7706855791962175, 0.7882882882882882, 0.7713950762016412]
    fig, ax1 = plt.subplots()
    ax1.set_ylabel("F1-Score")
    df = pd.DataFrame.from_dict(data=dict(remo_gp=fit_max, ferreira_et_al=fit_max_gpx, evans_et_al=fit_max_whats_inside,
                                          bojarczuk_et_al=fit_max_const), orient='index').T
    print(df)
    ax = sns.stripplot(data=df)
    ax.set_ylim(0.6, 1)
    plt.savefig("pareto_results/" + dataset_name + "/fscore.png")

    results_log = [{
        'mlp_fscore': sum(mlp_fscore_sum) / n_experiments,
        'mlp_fscore_std': statistics.pstdev(mlp_fscore_sum),
        'gp_fscore_avg': sum(gp_fscore_sum) / n_experiments,
        'gp_fscore_std': statistics.pstdev(gp_fscore_sum),
        'total_pareto': best_pareto.items.__len__(),
        'fit_max': fit_max
    }]
    df_logbook = pd.DataFrame(results_log)
    df_logbook.to_csv("pareto_results/" + dataset_name + "/results.csv")
    # generateParetoCharts(accumulatedPareto)
    # generateTree(best_pareto)


def generateParetoCharts(paretos):
    import matplotlib.pyplot as plt
    global dataset_name

    fscoreData = []
    ariData = []
    complexTermData = []
    for p in paretos:
        for i in p.items:
            fscore, ari, complexTerminals = fitness_function(i)
            fscoreData.append(fscore)
            ariData.append(ari)
            complexTermData.append(complexTerminals)

    # fig = plt.figure()
    # ax = fig.add_subplot(111, projection='3d')

    # xline = complexTermData
    # zline = ariData
    # yline = fscoreData
    # ax.scatter(xline, yline, zline, c='red', label=['complex terminals', 'fscore', 'ari'], lineWidth=0.5)
    # ax.set_xlabel('complex terminals', fontweight='bold')
    # ax.set_ylabel('fscore', fontweight='bold')
    # ax.set_zlabel('ari', fontweight='bold')
    #
    # plt.savefig("pareto_results/" + dataset_name + "/breast_pareto.png")

    # plt.show()

    fig1, ax1 = plt.subplots()
    # breast_gpx_fscore = [0.761904761904762, 0.37340153452685426, 0.822463768115942, 0.7686116700201208,
    #                      0.7482219061166429, 0.35543766578249336, 0.8293650793650793, 0.7514677103718199,
    #                      0.7329376854599406, 0.3517060367454068, 0.8556521739130434, 0.7237762237762237,
    #                      0.37340153452685426, 0.8615384615384616, 0.8541300527240774, 0.7410714285714286,
    #                      0.2872340425531915, 0.8871224165341811, 0.8699029126213592, 0.8028933092224231,
    #                      0.761904761904762, 0.37340153452685426, 0.8650646950092421, 0.761904761904762,
    #                      0.37340153452685426, 0.8463073852295409, 0.8066528066528067, 0.761904761904762,
    #                      0.761904761904762, 0.37340153452685426]
    # breast_gpx_split = [0.8807970779778823, 0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085, 0.8807970779778823, 0.5,
    #                     0.9820137900379085, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9996646498695336,
    #                     0.9820137900379085, 0.8807970779778823, 0.5]
    # breast_const_fscore = [0.8731884057971014, 0.8163265306122449, 0.7702702702702703, 0.33591731266149866,
    #                        0.8490230905861458, 0.7407407407407406, 0.33591731266149866, 0.8432432432432433,
    #                        0.8220064724919094, 0.807843137254902, 0.7992633517495397, 0.7702702702702703,
    #                        0.35989717223650386, 0.8553259141494435, 0.7432835820895523, 0.33591731266149866,
    #                        0.8597785977859779, 0.8447937131630648, 0.7846153846153846, 0.7586206896551724,
    #                        0.35989717223650386, 0.8492201039861351, 0.7687776141384389, 0.33591731266149866,
    #                        0.8697247706422019, 0.8496503496503497, 0.7702702702702703, 0.7702702702702703,
    #                        0.7702702702702703, 0.35989717223650386]
    # breast_const_simplicity = [0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999752619396515,
    #                            0.9998134705161004, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272,
    #                            0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.999999992351647, 0.9999752619396515, 0.9985950204031521, 0.9259109423490272,
    #                            0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272,
    #                            0.9259109423490272, 0.6236575784782976]
    # bank_gpx_fscore = [0.8635875402792695, 0.6140240050536956, 0.6140240050536956, 0.6140240050536956, 0.6140240050536956, 0.39337016574585637, 0.9001029866117405, 0.7596717467760844, 0.6674418604651163, 0.39337016574585637, 0.8635875402792695, 0.7596717467760844, 0.39337016574585637, 0.8013300083125521, 0.6140240050536956, 0.6140240050536956, 0.6140240050536956, 0.6140240050536956, 0.39337016574585637, 0.8320855614973262, 0.7596717467760844, 0.39337016574585637, 0.8400392541707556, 0.8393782383419689, 0.7596717467760844, 0.39337016574585637, 0.8626198083067093, 0.8013300083125521, 0.8013300083125521, 0.7596717467760844, 0.39337016574585637, 0.8320855614973262, 0.7684319833852544, 0.6140240050536956, 0.6140240050536956, 0.39337016574585637, 0.9134808853118711, 0.789598108747045, 0.7596717467760844, 0.39337016574585637, 0.8320855614973262, 0.7596717467760844, 0.39337016574585637, 0.8320855614973262, 0.7596717467760844, 0.39337016574585637, 0.8253119429590018, 0.7596717467760844, 0.39337016574585637, 0.8583690987124464, 0.7596717467760844, 0.39337016574585637, 0.8181011535048802, 0.8013300083125521, 0.7596717467760844, 0.39337016574585637, 0.8120603015075376, 0.7596717467760844, 0.39337016574585637, 0.7885714285714287, 0.7596717467760844, 0.39337016574585637, 0.7596717467760844, 0.39337016574585637, 0.7989179440937783, 0.7678916827852997, 0.7596717467760844, 0.39337016574585637, 0.8839662447257384, 0.8413793103448276, 0.7596717467760844, 0.39337016574585637, 0.8030467899891185, 0.8013300083125521, 0.6691176470588235, 0.6140240050536956, 0.6140240050536956, 0.39337016574585637, 0.7735644637053087, 0.7658058771148709, 0.7596717467760844, 0.39337016574585637, 0.8013300083125521, 0.7596717467760844, 0.39337016574585637, 0.760655737704918, 0.7596717467760844, 0.39337016574585637, 0.8013300083125521, 0.6674418604651163, 0.39337016574585637, 0.8367129135538954, 0.7763864042933811, 0.7596717467760844, 0.39337016574585637, 0.7860508953817154, 0.7596717467760844, 0.39337016574585637, 0.8635875402792695, 0.7596717467760844, 0.39337016574585637, 0.8413793103448276, 0.7596717467760844, 0.39337016574585637, 0.7804444444444445, 0.7633587786259544, 0.7596717467760844, 0.39337016574585637, 0.8635875402792695, 0.7596717467760844, 0.39337016574585637]
    # bank_gpx_split = [0.9820137900379085, 0.8807970779778823, 0.8807970779778823, 0.8807970779778823, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.8807970779778823, 0.8807970779778823, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9999546021312976, 0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9999546021312976, 0.9820137900379085, 0.8807970779778823, 0.8807970779778823, 0.5, 0.9999999979388463, 0.9999546021312976, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9999998874648379, 0.9996646498695336, 0.9820137900379085, 0.8807970779778823, 0.8807970779778823, 0.5, 0.9999999979388463, 0.9999998874648379, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9999938558253978, 0.8807970779778823, 0.5, 0.9999938558253978, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9999999997210531, 0.9999998874648379, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5]
    # bank_const_fscore = [0.856868395773295, 0.7664670658682635, 0.657439446366782, 0.40484048404840484, 0.8580576307363929, 0.7688734030197446, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.7608695652173915, 0.7389277389277389, 0.657439446366782, 0.40484048404840484, 0.8094262295081968, 0.7300215982721382, 0.6569343065693429, 0.6166456494325347, 0.6166456494325347, 0.6166456494325347, 0.6166456494325347, 0.40484048404840484, 0.8590604026845639, 0.7688734030197446, 0.40484048404840484, 0.779107725788901, 0.7389277389277389, 0.7389277389277389, 0.7389277389277389, 0.657439446366782, 0.40484048404840484, 0.8053024026512013, 0.7725072604065827, 0.7688734030197446, 0.40484048404840484, 0.9213483146067416, 0.7688734030197446, 0.40484048404840484, 0.8405017921146953, 0.7688734030197446, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.8601921024546425, 0.6166456494325347, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.9475890985324948, 0.7688734030197446, 0.40484048404840484, 0.8664495114006515, 0.8545454545454545, 0.7688734030197446, 0.40484048404840484, 0.8153681963713981, 0.7688734030197446, 0.40484048404840484, 0.7688734030197446, 0.40484048404840484, 0.8137472283813747, 0.7556512378902045, 0.7443519619500594, 0.6166456494325347, 0.6166456494325347, 0.6166456494325347, 0.40484048404840484, 0.8053024026512013, 0.7688734030197446, 0.40484048404840484, 0.8370370370370371, 0.7755991285403051, 0.7389277389277389, 0.657439446366782, 0.40484048404840484, 0.784, 0.7389277389277389, 0.657439446366782, 0.40484048404840484, 0.8342361863488624, 0.657439446366782, 0.40484048404840484, 0.8545454545454545, 0.7688734030197446, 0.40484048404840484, 0.8129370629370629, 0.7688734030197446, 0.40484048404840484, 0.775609756097561, 0.7688734030197446, 0.657439446366782, 0.40484048404840484, 0.9243856332703213, 0.8601921024546425, 0.7688734030197446, 0.40484048404840484, 0.8053024026512013, 0.7688734030197446, 0.40484048404840484, 0.7874493927125505, 0.7598566308243727, 0.6870056497175142, 0.657439446366782, 0.40484048404840484]
    # bank_const_simplicity = [0.9999752619396515, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999999423206956, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9895009751316033, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999999999996865, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.9999995650173181, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9999999423206956, 0.9998134705161004, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999995650173181, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.999996719629881, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.9999995650173181, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.999996719629881, 0.9259109423490272, 0.6236575784782976, 0.999996719629881, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.999996719629881, 0.9999752619396515, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976]
    # data_gpx_fscore = [0.9357142857142857, 0.8592057761732852, 0.672645739910314, 0.9090909090909091, 0.9015151515151515, 0.7185185185185186, 0.9280575539568345, 0.9015151515151515, 0.6641791044776119, 0.8627450980392156, 0.8549019607843137, 0.6640926640926641, 0.8996282527881041, 0.8833922261484098, 0.7185185185185186, 0.9056603773584906, 0.8638132295719845, 0.672645739910314, 0.8950617283950617, 0.7185185185185186, 0.9337979094076655, 0.784, 0.672645739910314, 0.8993710691823898, 0.6641791044776119, 0.9347826086956521, 0.7185185185185186, 0.8136882129277566, 0.8065843621399177, 0.672645739910314, 0.8812260536398467, 0.7185185185185186, 0.9157509157509157, 0.9015151515151515, 0.8920863309352517, 0.6009389671361502, 0.813953488372093, 0.6640926640926641, 0.7664233576642335, 0.7345454545454545, 0.7272727272727273, 0.7185185185185186, 0.9219858156028369, 0.672645739910314, 0.9428571428571428, 0.8615384615384616, 0.7607843137254902, 0.6641791044776119, 0.8851351351351352, 0.6816479400749064, 0.9574468085106382, 0.6816479400749064, 0.7626459143968872, 0.7563025210084034, 0.7185185185185186, 0.9187279151943464, 0.9022082018927445, 0.6816479400749064, 0.8671875, 0.7185185185185186, 0.9003690036900369, 0.6641791044776119, 0.9300699300699301, 0.8463949843260188, 0.6641791044776119, 0.9342560553633218, 0.8950617283950617, 0.7185185185185186, 0.8482490272373542, 0.8203389830508474, 0.7185185185185186, 0.9246575342465754, 0.6640926640926641, 0.9574468085106382, 0.6641791044776119, 0.940766550522648, 0.7185185185185186, 0.8592057761732852, 0.8561643835616437, 0.6816479400749064]
    # data_gpx_split = [0.9820137900379085, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9999938558253978, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9999938558253978, 0.9996646498695336, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9996646498695336, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9999546021312976, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.9999999999999982, 0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5]
    # data_const_fscore = [0.9465648854961832, 0.8992248062015504, 0.7085201793721972, 0.917910447761194, 0.8954248366013072, 0.8954248366013072, 0.7224334600760456, 0.9343629343629344, 0.6987951807228916, 0.8954248366013072, 0.7085201793721972, 0.9537366548042704, 0.8353413654618475, 0.7224334600760456, 0.9022556390977443, 0.6987951807228916, 0.8688524590163934, 0.7085201793721972, 0.8391608391608392, 0.8244274809160306, 0.6538461538461539, 0.9358490566037736, 0.728, 0.7085201793721972, 0.8954248366013072, 0.7224334600760456, 0.8976377952755905, 0.6987951807228916, 0.8995983935742973, 0.7224334600760456, 0.8995983935742973, 0.7224334600760456, 0.8995983935742973, 0.7983193277310924, 0.5576923076923077, 0.8995983935742973, 0.8856088560885609, 0.7224334600760456, 0.8995983935742973, 0.6987951807228916, 0.9358490566037736, 0.7224334600760456, 0.9420849420849421, 0.8954248366013072, 0.831275720164609, 0.7085201793721972, 0.8142292490118578, 0.7722772277227723, 0.7085201793721972, 0.8995983935742973, 0.6795366795366795, 0.8976377952755905, 0.873469387755102, 0.6987951807228916, 0.9160839160839161, 0.8790322580645161, 0.6451612903225806, 0.9520295202952029, 0.8954248366013072, 0.6987951807228916, 0.948529411764706, 0.9381818181818182, 0.7509578544061303, 0.7085201793721972, 0.8612099644128114, 0.8582995951417005, 0.7833333333333333, 0.7224334600760456, 0.904, 0.7085201793721972, 0.8656716417910448, 0.7874015748031495, 0.7226277372262774, 0.7224334600760456, 0.9098039215686275, 0.8654545454545454, 0.6538461538461539, 0.8790322580645161, 0.7085201793721972, 0.933852140077821, 0.9323308270676692, 0.7085201793721972]
    # data_const_simplicity = [0.9999752619396515, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9999752619396515, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999752619396515, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.999996719629881, 0.9259109423490272, 0.6236575784782976]
    # data_gpx_fscore = [0.9014084507042253, 0.7392996108949418, 0.5145631067961166, 0.8708487084870847, 0.7905138339920948, 0.6940298507462687, 0.5479452054794519, 0.8656716417910448, 0.8549618320610687, 0.8487084870848708, 0.5479452054794519, 0.8561872909698995, 0.7905138339920948, 0.8389513108614232, 0.7905138339920948, 0.8708487084870847, 0.8285714285714286, 0.487603305785124, 0.9025270758122744, 0.8088235294117647, 0.9003690036900369, 0.8905109489051095, 0.5479452054794519, 0.8203389830508475, 0.7905138339920948, 0.5145631067961166, 0.8708487084870847, 0.8088235294117647, 0.8602150537634409, 0.7905138339920948, 0.8913857677902624, 0.5479452054794519, 0.9110320284697508, 0.8905109489051095, 0.8088235294117647, 0.896797153024911, 0.8075471698113209, 0.7950819672131149, 0.5882352941176471, 0.8708487084870847, 0.8561872909698995, 0.8088235294117647, 0.8708487084870847, 0.8487084870848708, 0.5479452054794519, 0.8359375000000001, 0.5882352941176471, 0.9003690036900369, 0.7905138339920948, 0.5427509293680297, 0.8593155893536122, 0.8088235294117647, 0.4913793103448276, 0.8604651162790699, 0.8088235294117647, 0.8708487084870847, 0.6846846846846847, 0.8676470588235294, 0.7905138339920948, 0.8708487084870847, 0.8913857677902624, 0.8088235294117647, 0.8741258741258742, 0.8659003831417625, 0.8088235294117647, 0.8771929824561403, 0.8708487084870847, 0.8636363636363635, 0.5427509293680297, 0.8708487084870847, 0.6846846846846847, 0.8708487084870847]
    # data_gpx_split = [0.9820137900379085, 0.8807970779778823, 0.5, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9999938558253978, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.5, 0.9996646498695336, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9999546021312976, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.8807970779778823, 0.5, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.5, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9820137900379085, 0.5, 0.9975273768433653, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.5]
    # data_const_fscore = [0.8625954198473282, 0.8409090909090909, 0.7871485943775101, 0.8808664259927798, 0.8582089552238805, 0.8472222222222222, 0.8156028368794326, 0.7871485943775101, 0.8943661971830986, 0.8652482269503546, 0.8582089552238805, 0.8644688644688645, 0.7871485943775101, 0.890566037735849, 0.5399239543726235, 0.8808664259927798, 0.8582089552238805, 0.6877828054298643, 0.8404669260700389, 0.7578125, 0.8847583643122677, 0.7871485943775101, 0.7578125, 0.890566037735849, 0.7871485943775101, 0.8808664259927798, 0.5399239543726235, 0.890566037735849, 0.8582089552238805, 0.8731343283582088, 0.7885304659498207, 0.7578125, 0.8644688644688645, 0.7871485943775101, 0.8682170542635659, 0.8582089552238805, 0.9044117647058824, 0.8808664259927798, 0.697508896797153, 0.7773584905660378, 0.697508896797153, 0.8652482269503546, 0.697508896797153, 0.8582089552238805, 0.7871485943775101, 0.8652482269503546, 0.6877828054298643, 0.8652482269503546, 0.697508896797153, 0.8582089552238805, 0.7871485943775101, 0.888888888888889, 0.8582089552238805, 0.6877828054298643, 0.9144981412639406, 0.7578125, 0.8644688644688645, 0.8582089552238805, 0.8582089552238805, 0.8764044943820224, 0.8582089552238805, 0.46715328467153283, 0.8582089552238805, 0.8682170542635659, 0.7578125, 0.8582089552238805, 0.7578125]
    # data_const_simplicity = [0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976]data_gpx_fscore = [0.9014084507042253, 0.7392996108949418, 0.5145631067961166, 0.8708487084870847, 0.7905138339920948, 0.6940298507462687, 0.5479452054794519, 0.8656716417910448, 0.8549618320610687, 0.8487084870848708, 0.5479452054794519, 0.8561872909698995, 0.7905138339920948, 0.8389513108614232, 0.7905138339920948, 0.8708487084870847, 0.8285714285714286, 0.487603305785124, 0.9025270758122744, 0.8088235294117647, 0.9003690036900369, 0.8905109489051095, 0.5479452054794519, 0.8203389830508475, 0.7905138339920948, 0.5145631067961166, 0.8708487084870847, 0.8088235294117647, 0.8602150537634409, 0.7905138339920948, 0.8913857677902624, 0.5479452054794519, 0.9110320284697508, 0.8905109489051095, 0.8088235294117647, 0.896797153024911, 0.8075471698113209, 0.7950819672131149, 0.5882352941176471, 0.8708487084870847, 0.8561872909698995, 0.8088235294117647, 0.8708487084870847, 0.8487084870848708, 0.5479452054794519, 0.8359375000000001, 0.5882352941176471, 0.9003690036900369, 0.7905138339920948, 0.5427509293680297, 0.8593155893536122, 0.8088235294117647, 0.4913793103448276, 0.8604651162790699, 0.8088235294117647, 0.8708487084870847, 0.6846846846846847, 0.8676470588235294, 0.7905138339920948, 0.8708487084870847, 0.8913857677902624, 0.8088235294117647, 0.8741258741258742, 0.8659003831417625, 0.8088235294117647, 0.8771929824561403, 0.8708487084870847, 0.8636363636363635, 0.5427509293680297, 0.8708487084870847, 0.6846846846846847, 0.8708487084870847]
    data_gpx_fscore = [0.8156424581005587, 0.8072916666666666, 0.67524115755627, 0.8105263157894738, 0.8033240997229917, 0.7278481012658228, 0.67524115755627, 0.8019801980198019, 0.6123778501628664, 0.7848837209302326, 0.7484662576687118, 0.67524115755627, 0.8106796116504853, 0.8019801980198019, 0.7957559681697612, 0.6123778501628664, 0.7933884297520661, 0.7849999999999999, 0.6204620462046204, 0.8167539267015707, 0.783068783068783, 0.67524115755627, 0.825986078886311, 0.6204620462046204, 0.8200589970501475, 0.67524115755627, 0.825986078886311, 0.67524115755627, 0.7616580310880828, 0.725552050473186, 0.67524115755627, 0.8200589970501475, 0.67524115755627, 0.825986078886311, 0.6470588235294118, 0.7861271676300579, 0.7772925764192139, 0.6204620462046204, 0.7878787878787878, 0.67524115755627, 0.825986078886311, 0.67524115755627, 0.8242074927953891, 0.8163265306122448, 0.8081395348837209, 0.67524115755627, 0.8157099697885196, 0.7957559681697612, 0.6138613861386139, 0.825986078886311, 0.6204620462046204, 0.8423772609819121, 0.7559055118110236, 0.67524115755627, 0.7772925764192139, 0.6470588235294118, 0.8048048048048047, 0.7772925764192139, 0.67524115755627, 0.8150289017341041, 0.8072916666666666, 0.67524115755627, 0.8091603053435115, 0.7926509186351706, 0.783068783068783, 0.67524115755627, 0.814404432132964, 0.7681159420289855, 0.6138613861386139, 0.7760416666666666, 0.6470588235294118, 0.8159203980099502, 0.7538940809968847, 0.6470588235294118, 0.8579088471849866, 0.782608695652174, 0.59672131147541, 0.7789473684210527, 0.7560321715817695, 0.755784061696658, 0.67524115755627, 0.8010075566750628, 0.67524115755627]
    data_gpx_split = [0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5, 0.9820137900379085, 0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9996646498695336, 0.9820137900379085, 0.8807970779778823, 0.5, 0.8807970779778823, 0.5]
    data_const_fscore = [0.8118811881188118, 0.66875, 0.8721461187214611, 0.8146341463414634, 0.6325878594249201, 0.8781725888324873, 0.7952380952380953, 0.6439628482972136, 0.882494004796163, 0.8110403397027601, 0.66875, 0.8192219679633868, 0.8126649076517151, 0.8110403397027601, 0.66875, 0.8357487922705314, 0.66875, 0.8164251207729468, 0.8110403397027601, 0.6439628482972136, 0.8154269972451792, 0.8123249299719889, 0.8110403397027601, 0.66875, 0.8721461187214611, 0.66875, 0.8192219679633868, 0.66875, 0.8173913043478261, 0.6325878594249201, 0.8110403397027601, 0.66875, 0.8284313725490196, 0.6439628482972136, 0.8444444444444444, 0.8110403397027601, 0.8110403397027601, 0.6439628482972136, 0.8466819221967963, 0.7415730337078652, 0.66875, 0.8586666666666666, 0.8110403397027601, 0.8110403397027601, 0.6292834890965732, 0.8431876606683805, 0.8110403397027601, 0.66875, 0.8329297820823244, 0.66875, 0.8213457076566126, 0.8110403397027601, 0.8110403397027601, 0.8110403397027601, 0.6439628482972136, 0.8110403397027601, 0.66875, 0.8201058201058201, 0.8110403397027601, 0.66875, 0.8390243902439023, 0.8110403397027601, 0.66875, 0.8146341463414634, 0.66875, 0.8200000000000001, 0.8110403397027601, 0.66875, 0.8058968058968059, 0.6439628482972136, 0.8329297820823244, 0.6292834890965732, 0.8277404921700224, 0.8207792207792208, 0.7918781725888325, 0.6439628482972136, 0.8721461187214611, 0.6439628482972136, 0.8110403397027601, 0.66875, 0.8021978021978023, 0.7572254335260117, 0.66875]
    data_const_simplicity = [0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999999423206956, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9985950204031521, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9998134705161004, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9259109423490272, 0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976]
    line1, = ax1.plot(data_gpx_split, data_gpx_fscore, 'ro', label='gpx')
    line2, = ax1.plot(data_const_simplicity, data_const_fscore, 'gs', label='constrained')
    line3, = ax1.plot(ariData, fscoreData, 'k+', label='ari-remo-gp')
    ax1.legend(handles=[line1, line2, line3])
    ax1.set(xlabel='objectives', ylabel='fscore',
            title='')
    ax1.grid()

    fig1.savefig("pareto_results/" + dataset_name + "/compare_ari.png")

    fig2, ax2 = plt.subplots()
    line1, = ax2.plot(data_gpx_split, data_gpx_fscore, 'ro', label='gpx')
    line2, = ax2.plot(data_const_simplicity, data_const_fscore, 'gs', label='constrained')
    line3, = ax2.plot(complexTermData, fscoreData, 'k+', label='complex-remo-gp')
    ax2.legend(handles=[line1, line2, line3])
    ax2.set(xlabel='objectives', ylabel='fscore',
            title='')
    ax2.grid()

    fig2.savefig("pareto_results/" + dataset_name + "/compare_complex.png")

    fig3, ax3 = plt.subplots()
    line1, = ax3.plot(data_gpx_split, data_gpx_fscore, 'ro', label='gpx')
    line2, = ax3.plot(data_const_simplicity, data_const_fscore, 'gs', label='constrained')
    line3, = ax3.plot(ariData, fscoreData, 'k+', label='ari-remo-gp')
    line4, = ax3.plot(complexTermData, fscoreData, 'y+', label='complex-remo-gp')
    ax3.legend(handles=[line1, line2, line3, line4])
    ax3.set(xlabel='objectives', ylabel='fscore',
            title='')
    ax3.grid()

    fig3.savefig("pareto_results/" + dataset_name + "/compare_ari_complex.png")

    # ax1.set(xlabel='automated readability index', ylabel='fscore',
    #         title='')
    # ax1.grid()
    #
    # fig1.savefig("pareto_results/" + dataset_name + "/breast_ari.png")
    # # plt.show()
    #
    # fig2, ax2 = plt.subplots()
    # ax2.plot(complexTermData, fscoreData, 'ro')
    #
    # ax2.set(xlabel='complex terminals', ylabel='fscore',
    #         title='')
    # ax2.grid()
    #
    # fig2.savefig("pareto_results/" + dataset_name + "/complexTerminals.png")
    # # plt.show()


def generateTree(best_pareto):
    import pygraphviz as pgv
    global dataset_name
    # nodes, edges, labels = gp.graph(best_gp_function)
    #
    # g = pgv.AGraph()
    # g.add_nodes_from(nodes)
    # g.add_edges_from(edges)
    # g.layout(prog="dot")
    #
    # for i in nodes:
    #     n = g.get_node(i)
    #     n.attr["label"] = labels[i]
    #
    # g.draw("hof_results/tree_hall_of_fame.pdf")

    pareto_draw = sorted(best_pareto.items, key=lambda i: i.__len__())

    for i_pareto in pareto_draw:
        nodes, edges, labels = gp.graph(i_pareto)

        g = pgv.AGraph()
        g.add_nodes_from(nodes)
        g.add_edges_from(edges)
        g.layout(prog="dot")

        for i in nodes:
            n = g.get_node(i)
            n.attr["label"] = labels[i]

        g.draw("pareto_results/" + dataset_name + "/tree_pareto_" + str(pareto_draw.index(i_pareto)) + ".pdf")


def main(dataset):
    # Get global scope variables
    global X_train, X_test, y_train, y_test, toolbox, opaque_model_prediction_test, opaque_model_prediction_train, mlp_time

    if dataset == 'ionosphere':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_ionosphere()
    elif dataset == 'breast_cancer':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_breast_cancer()
    elif dataset == 'digits1_7':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_digits(1, 7)
    elif dataset == 'digits3_9':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_digits(3, 9)
    elif dataset == 'wine':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_wine()
    elif dataset == 'banknotes':
        # Fetch dataset and set train/test variables
        X_train, X_test, y_train, y_test = fetch_dataset.fetch_banknotes()

    # Execute blackbox algorithm
    opaque_model_prediction_test, opaque_model_prediction_train, classifier, mlp_time = mlp.createInstance(X_train, X_test,
                                                                                                           y_train, y_test)

    generateReport(n_experiments=30)


if __name__ == '__main__':
    time_start = process_time()

    # dataset_name = 'wine'
    # main(dataset_name)

    dataset_name = 'ionosphere'
    main(dataset_name)

    # dataset_name = 'breast_cancer'
    # main(dataset_name)

    # dataset_name = 'digits1_7'
    # main(dataset_name)

    # dataset_name = 'digits3_9'
    # main(dataset_name)

    # dataset_name = 'banknotes'
    # main(dataset_name)

    # import matplotlib.pyplot as plt
    # fig1, ax1 = plt.subplots()
    # breast_gpx_fscore = [0.761904761904762, 0.37340153452685426, 0.822463768115942, 0.7686116700201208,
    #                      0.7482219061166429, 0.35543766578249336, 0.8293650793650793, 0.7514677103718199,
    #                      0.7329376854599406, 0.3517060367454068, 0.8556521739130434, 0.7237762237762237,
    #                      0.37340153452685426, 0.8615384615384616, 0.8541300527240774, 0.7410714285714286,
    #                      0.2872340425531915, 0.8871224165341811, 0.8699029126213592, 0.8028933092224231,
    #                      0.761904761904762, 0.37340153452685426, 0.8650646950092421, 0.761904761904762,
    #                      0.37340153452685426, 0.8463073852295409, 0.8066528066528067, 0.761904761904762,
    #                      0.761904761904762, 0.37340153452685426]
    # breast_gpx_split = [0.8807970779778823, 0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9975273768433653, 0.9820137900379085, 0.8807970779778823, 0.5,
    #                     0.9820137900379085, 0.8807970779778823, 0.5, 0.9999991684719722, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9996646498695336, 0.9975273768433653, 0.9820137900379085,
    #                     0.8807970779778823, 0.5, 0.9975273768433653, 0.8807970779778823, 0.5, 0.9996646498695336,
    #                     0.9820137900379085, 0.8807970779778823, 0.5]
    # breast_const_fscore = [0.8731884057971014, 0.8163265306122449, 0.7702702702702703, 0.33591731266149866,
    #                        0.8490230905861458, 0.7407407407407406, 0.33591731266149866, 0.8432432432432433,
    #                        0.8220064724919094, 0.807843137254902, 0.7992633517495397, 0.7702702702702703,
    #                        0.35989717223650386, 0.8553259141494435, 0.7432835820895523, 0.33591731266149866,
    #                        0.8597785977859779, 0.8447937131630648, 0.7846153846153846, 0.7586206896551724,
    #                        0.35989717223650386, 0.8492201039861351, 0.7687776141384389, 0.33591731266149866,
    #                        0.8697247706422019, 0.8496503496503497, 0.7702702702702703, 0.7702702702702703,
    #                        0.7702702702702703, 0.35989717223650386]
    # breast_const_simplicity = [0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.9895009751316033, 0.9259109423490272, 0.6236575784782976, 0.9999752619396515,
    #                            0.9998134705161004, 0.9985950204031521, 0.9895009751316033, 0.9259109423490272,
    #                            0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.999999992351647, 0.9999752619396515, 0.9985950204031521, 0.9259109423490272,
    #                            0.6236575784782976, 0.9895009751316033, 0.9259109423490272, 0.6236575784782976,
    #                            0.9985950204031521, 0.9895009751316033, 0.9259109423490272, 0.9259109423490272,
    #                            0.9259109423490272, 0.6236575784782976]
    #
    # line1, = ax1.plot(breast_gpx_split, breast_gpx_fscore, 'ro', label='gpx')
    # line2, = ax1.plot(breast_const_simplicity,breast_const_fscore , 'gs', label='constrained')
    # ax1.legend(handles=[line1, line2])
    # # ax1.plot(fscoreData, ariData, 'k+')
    # ax1.set(xlabel='comparison', ylabel='fscore',
    #         title='')
    # ax1.grid()
    #
    # fig1.savefig("pareto_results/" + dataset_name + "/breast_compare_ari.png")
    time_end = process_time()

    print(time_end - time_start)