WNJ_EPIA2023.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "IMPORTS AND CONSTANTS:"
      ],
      "metadata": {
        "id": "V4Femz5MnC56"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install gurobipy\n",
        "import gurobipy as gp\n",
        "from gurobipy import GRB\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "import pickle\n",
        "import time\n",
        "import torch\n",
        "from torch import nn\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n",
        "\n",
        "num_inst=100\n",
        "num_jobs=500\n",
        "num_ref_jobs=500\n",
        "u=0.1\n",
        "v=0.3\n",
        "fractions = [0.2]\n",
        "\n",
        "common_path = \"/content/drive/My Drive/EPIA_2023\"\n",
        "instances_path = f\"{common_path}/instances/{u}_{v}\"\n",
        "apriori_model_path = f\"{common_path}/ml_apriori_models\"\n",
        "conditional_model_path = f\"{common_path}/ml_conditional_models\""
      ],
      "metadata": {
        "id": "NAFZOlJMnHq9"
      },
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "CLASSES AND FUNCTIONS:"
      ],
      "metadata": {
        "id": "FBU5-rOOkHD3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class Instance:\n",
        "\n",
        "    def __init__(self, a_dict, a_u, a_v):\n",
        "\n",
        "        # read the parameters\n",
        "\n",
        "        self.num_jobs = len(a_dict[\"durations\"])\n",
        "        self.durations = torch.clone(a_dict[\"durations\"])\n",
        "        self.due_dates = torch.clone(a_dict[\"due_dates\"])\n",
        "        self.deadlines = torch.clone(a_dict[\"deadlines\"])\n",
        "        self.weights = torch.clone(a_dict[\"weights\"])\n",
        "        self.solution = torch.clone(a_dict[\"solution\"])\n",
        "        self.optimal_value = a_dict[\"optimal_value\"]\n",
        "        self.heuristic_value = a_dict[\"heuristic_value\"]\n",
        "        self.u = a_u\n",
        "        self.v = a_v\n",
        "\n",
        "        # scheduling params\n",
        "        self.jobs_queue = self.create_jobs_queue()\n",
        "        self.queue_index = 0\n",
        "\n",
        "        self.is_decided = torch.zeros((self.num_jobs,), dtype=torch.bool)\n",
        "        self.is_scheduled = torch.zeros((self.num_jobs,), dtype=torch.bool)\n",
        "        _, self.non_scheduled = [torch.tensor(t) for t in zip(*sorted(zip(self.deadlines, range(self.num_jobs))))]\n",
        "        self.schedule = torch.zeros((self.num_jobs,), dtype=torch.int) - 1\n",
        "        self.schedule_index = 0\n",
        "\n",
        "        self.time_moment = 0\n",
        "        self.margin = 0\n",
        "        self.cost = 0\n",
        "\n",
        "    def clear_schedule(self):\n",
        "\n",
        "        self.queue_index = 0\n",
        "\n",
        "        self.is_decided = torch.zeros((self.num_jobs,), dtype=torch.bool)\n",
        "        self.is_scheduled = torch.zeros((self.num_jobs,), dtype=torch.bool)\n",
        "        _, self.non_scheduled = [torch.tensor(t) for t in zip(*sorted(zip(self.deadlines, range(self.num_jobs))))]\n",
        "        self.schedule = torch.zeros((self.num_jobs,), dtype=torch.int) - 1\n",
        "        self.schedule_index = 0\n",
        "\n",
        "        self.time_moment = 0\n",
        "        self.margin = 0\n",
        "        self.cost = 0\n",
        "\n",
        "    def compute_apriori_features(self):\n",
        "\n",
        "        dur_sum = torch.sum(self.durations)\n",
        "        features = torch.empty((self.num_jobs, 8), dtype=torch.double)\n",
        "        features[:, 0] = self.weights / 100\n",
        "        features[:, 1] = self.durations / 100\n",
        "        features[:, 2] = (self.due_dates - self.u * dur_sum) / (self.v * dur_sum - self.u * dur_sum)\n",
        "        features[:, 3] = (self.deadlines - self.u * dur_sum) / (1.1 * dur_sum - self.u * dur_sum)\n",
        "        features[:, 4] = (self.weights / self.durations - 0.01) / (100 - 0.01)\n",
        "        features[:, 5] = (self.weights - self.durations) / 100\n",
        "        features[:, 6] = self.due_dates / self.deadlines\n",
        "        features[:, 7] = (self.deadlines - self.due_dates) / (1.1 * dur_sum - self.u * dur_sum)\n",
        "        return features\n",
        "\n",
        "    def compute_conditional_features(self, r_jobs=0):\n",
        "\n",
        "        if r_jobs > 0:\n",
        "            ref_jobs = torch.randperm(self.num_jobs)[:r_jobs]\n",
        "        else:\n",
        "            ref_jobs = torch.arange(self.num_jobs)\n",
        "\n",
        "        target_features = self.compute_apriori_features()\n",
        "        ref_features = target_features[ref_jobs]\n",
        "\n",
        "        point_part = ref_features.repeat(self.num_jobs, 1)\n",
        "        vector_part = (target_features.unsqueeze(1) - ref_features).reshape(self.num_jobs * len(ref_jobs),\n",
        "                                                                            target_features.size(dim=1))\n",
        "        return torch.cat((point_part, vector_part), dim=1)\n",
        "\n",
        "    def compute_predictions(self, a_apriori_model, a_conditional_model, a_num_ref_jobs):\n",
        "\n",
        "        a_apriori_model.eval()\n",
        "        a_conditional_model.eval()\n",
        "\n",
        "        with torch.no_grad():\n",
        "            my_softmax = nn.Softmax(dim=1)\n",
        "            apriori_features = self.compute_apriori_features()\n",
        "            apriori_probs = my_softmax(a_apriori_model(self.compute_apriori_features()))[:, 1]\n",
        "\n",
        "            ref_features = apriori_features[0:a_num_ref_jobs]\n",
        "            ref_probs = apriori_probs[0:a_num_ref_jobs]\n",
        "            probs = torch.zeros((self.num_jobs,), dtype=torch.float64)\n",
        "            for j in range(self.num_jobs):\n",
        "                conditional_features = torch.cat((ref_features, apriori_features[j] - ref_features), dim=1)\n",
        "                early_probs = my_softmax(a_conditional_model(torch.cat(\n",
        "                    (conditional_features, torch.ones((conditional_features.size(0), 1), dtype=torch.float64)),\n",
        "                    dim=1)))[:, 1]\n",
        "                tardy_probs = my_softmax(a_conditional_model(torch.cat(\n",
        "                    (conditional_features, torch.zeros((conditional_features.size(0), 1), dtype=torch.float64)),\n",
        "                    dim=1)))[:, 1]\n",
        "                probs[j] = torch.mean(early_probs * ref_probs + tardy_probs * (1 - ref_probs))\n",
        "\n",
        "        return torch.round(probs).to(torch.int), probs\n",
        "\n",
        "    def compute_early_predictions(self):\n",
        "        return torch.ones((self.num_jobs,), dtype=torch.int), torch.ones((self.num_jobs,), dtype=torch.float64)\n",
        "\n",
        "    def compute_random_predictions(self):\n",
        "        return torch.randint(high=2, size=(self.num_jobs,), dtype=torch.int), torch.full((self.num_jobs,), 0.5)\n",
        "\n",
        "    def create_jobs_queue(self):\n",
        "\n",
        "        keys = torch.cat((self.due_dates, self.deadlines))\n",
        "        values = torch.cat((torch.linspace(0, self.num_jobs - 1, self.num_jobs, dtype=torch.int),\n",
        "                            torch.linspace(0, self.num_jobs - 1, self.num_jobs, dtype=torch.int)))\n",
        "        tuples = zip(*sorted(zip(keys, values)))\n",
        "        _, ordered_jobs = [torch.tensor(t) for t in tuples]\n",
        "        return ordered_jobs\n",
        "\n",
        "    def edf_test(self, a_job):\n",
        "\n",
        "        if self.margin - self.durations[a_job] >= 0:\n",
        "            self.margin -= self.durations[a_job]\n",
        "            return True\n",
        "        else:\n",
        "            jobs_indexes = self.non_scheduled[self.non_scheduled != a_job]\n",
        "            durations = self.durations[jobs_indexes]\n",
        "            deadlines = self.deadlines[jobs_indexes]\n",
        "            my_cumsum = self.time_moment + self.durations[a_job] + torch.cumsum(durations, dim=0)\n",
        "            self.margin = torch.amin(deadlines - my_cumsum)\n",
        "            if self.margin >= 0:\n",
        "                return True\n",
        "            else:\n",
        "                return False\n",
        "\n",
        "    def improve_by_ilp(self, a_preds, a_probs, a_num_ilp_jobs):\n",
        "\n",
        "        r_jobs, fix_jobs = reduction_subset(a_probs, a_num_ilp_jobs)\n",
        "        weights, durations, due_dates, deadlines = self.reduce(r_jobs, a_preds[r_jobs])\n",
        "        ilp_preds, obj_val = solve_by_ilp(len(weights), weights, durations, due_dates, deadlines)\n",
        "        preds = torch.clone(a_preds)\n",
        "        if obj_val > -1:\n",
        "            preds[fix_jobs] = ilp_preds\n",
        "        return preds\n",
        "\n",
        "    def next_undecided_job(self):\n",
        "\n",
        "        # schedule intermediate tardy jobs\n",
        "        self.queue_index += 1\n",
        "        num_jobs = 2 * self.num_jobs\n",
        "        while self.queue_index < num_jobs:\n",
        "            job = self.jobs_queue[self.queue_index]\n",
        "            if self.is_decided[job]:\n",
        "                if not self.is_scheduled[job]:\n",
        "                    self.schedule_job(job)\n",
        "                self.queue_index += 1\n",
        "            else:\n",
        "                break\n",
        "\n",
        "    def recompute_schedule_cost(self):\n",
        "\n",
        "        durations = self.durations[self.schedule]\n",
        "        my_cumsum = torch.cumsum(durations, dim=0)\n",
        "\n",
        "        # calculating the cost\n",
        "        weights_permutation = self.weights[self.schedule]\n",
        "        is_early = my_cumsum <= self.due_dates[self.schedule]\n",
        "        weights = weights_permutation[is_early]\n",
        "        total_cost = torch.sum(weights)\n",
        "\n",
        "        # calculating the penalty\n",
        "        is_violated = my_cumsum > self.deadlines[self.schedule]\n",
        "        num_violations = len(is_violated[is_violated])\n",
        "        penalty_value = 500 * num_violations\n",
        "        flag = True\n",
        "        if num_violations > 0:\n",
        "            flag = False\n",
        "\n",
        "        return total_cost - penalty_value, flag\n",
        "\n",
        "    def reduce(self, a_jobs, a_labels):\n",
        "\n",
        "        # make \"a_jobs\" standing the first\n",
        "        jobs = torch.arange(self.num_jobs)\n",
        "        T_mask = torch.zeros(self.num_jobs, dtype=torch.bool).index_fill(0, a_jobs, True)\n",
        "        F_mask = torch.logical_not(T_mask)\n",
        "        jobs = torch.cat((jobs[T_mask], jobs[F_mask]))\n",
        "\n",
        "        # reduction\n",
        "        due_dates = self.due_dates[jobs]\n",
        "        deadlines = self.deadlines[jobs]\n",
        "        durs = self.durations[T_mask]\n",
        "        for k in range(len(a_jobs)):\n",
        "            D = due_dates[k] if a_labels[k] == 1 else deadlines[k]\n",
        "            due_dates = torch.where(due_dates <= D, torch.minimum(due_dates, D - durs[k]), due_dates - durs[k])\n",
        "            deadlines = torch.where(deadlines <= D, torch.minimum(deadlines, D - durs[k]), deadlines - durs[k])\n",
        "\n",
        "        weights = self.weights[F_mask]\n",
        "        durations = self.durations[F_mask]\n",
        "        due_dates = due_dates[len(a_jobs):]\n",
        "        deadlines = deadlines[len(a_jobs):]\n",
        "        return weights, durations, due_dates, deadlines\n",
        "\n",
        "    def render(self):\n",
        "        info = (\n",
        "            f\"RENDERING...\\n\"\n",
        "            f\"=== Given input data ===\\n\"\n",
        "            f\"Durations = {self.durations[self.non_scheduled]} \\n\"\n",
        "            f\"Due dates = {self.due_dates[self.non_scheduled]} \\n\"\n",
        "            f\"Deadlines = {self.deadlines[self.non_scheduled]} \\n\"\n",
        "            f\"Weights = {self.weights[self.non_scheduled]} \\n\"\n",
        "            f\"Solution = {self.solution[self.non_scheduled]} \\n\"\n",
        "            f\"Opt value = {self.optimal_value} \\n\"\n",
        "            f\"Heu value = {self.heuristic_value} \\n\"\n",
        "            f\"\\n\"\n",
        "            f\"=== Scheduling params ===\\n\"\n",
        "            f\"Jobs queue = {self.jobs_queue} \\n\"\n",
        "            f\"Queue index = {self.queue_index} \\n\"\n",
        "            f\"Schedule = {self.schedule} \\n\"\n",
        "            f\"Schedule index = {self.schedule_index} \\n\"\n",
        "            f\"Non scheduled jobs = {self.non_scheduled} \\n\"\n",
        "            f\"Time moment = {self.time_moment} \\n\"\n",
        "            f\"Margin = {self.margin} \\n\"\n",
        "            f\"Cost = {self.cost} \\n\"\n",
        "            f\"\\n\"\n",
        "        )\n",
        "\n",
        "        print(info)\n",
        "\n",
        "    def step(self, a_job, a_prediction):\n",
        "\n",
        "        is_early_job = (a_prediction == 1) and (self.edf_test(a_job))\n",
        "        self.is_decided[a_job] = True\n",
        "        if is_early_job:\n",
        "            # a job becomes decided and being scheduled immediately\n",
        "            self.is_scheduled[a_job] = True\n",
        "            self.schedule_job(a_job)\n",
        "\n",
        "        self.next_undecided_job()\n",
        "\n",
        "    def schedule_job(self, a_job):\n",
        "\n",
        "        self.schedule[self.schedule_index] = a_job\n",
        "        self.schedule_index += 1\n",
        "        self.non_scheduled = self.non_scheduled[self.non_scheduled != a_job]\n",
        "        self.time_moment += self.durations[a_job]\n",
        "        if self.time_moment <= self.due_dates[a_job]:\n",
        "            self.cost += self.weights[a_job]\n",
        "\n",
        "    def schedule_jobs(self, a_predictions):\n",
        "\n",
        "        num_jobs = 2 * self.num_jobs\n",
        "        while self.queue_index < num_jobs:\n",
        "            current_job = self.jobs_queue[self.queue_index]\n",
        "            pred = a_predictions[current_job]\n",
        "            self.step(current_job, pred)\n",
        "\n",
        "    def solve_by_nor(self):\n",
        "\n",
        "        self.clear_schedule()\n",
        "        _, indices = torch.sort(torch.cat((self.due_dates, self.deadlines)), stable=True)\n",
        "        non_scheduled = torch.ones((self.num_jobs,), dtype=bool)\n",
        "\n",
        "        cur_element = 0\n",
        "        cur_scheduled = 0\n",
        "        while cur_scheduled < self.num_jobs:\n",
        "            job = indices[cur_element]\n",
        "            if non_scheduled[job]:\n",
        "                self.schedule[cur_scheduled] = job\n",
        "                cur_scheduled += 1\n",
        "            cur_element += 1\n",
        "\n",
        "        self.recompute_schedule_cost()\n",
        "\n",
        "\n",
        "class NeuralNetwork_Apriori(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(NeuralNetwork_Apriori, self).__init__()\n",
        "        self.linear_stack = nn.Sequential(\n",
        "            nn.Linear(8, 32),\n",
        "            nn.Tanh(),\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(32, 32),\n",
        "            nn.Tanh(),\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(32, 32),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(32, 2)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.linear_stack(x)\n",
        "\n",
        "\n",
        "class NeuralNetwork_Conditional(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(NeuralNetwork_Conditional, self).__init__()\n",
        "        self.linear_stack = nn.Sequential(\n",
        "            nn.Linear(17, 64),\n",
        "            nn.Tanh(),\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(64, 64),\n",
        "            nn.Tanh(),\n",
        "            nn.Dropout(0.5),\n",
        "            nn.Linear(64, 64),\n",
        "            nn.Tanh(),\n",
        "            nn.Linear(64, 2)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        return self.linear_stack(x)\n",
        "\n",
        "\n",
        "def collect_stats(a_instances, a_apriori_model, a_cond_model, a_fraction):\n",
        "    print(\"Call for collecting stats. Fraction = \" + str(a_fraction) + \":\")\n",
        "    our_gap, early_gap, rand_gap, heu_gap = float(0), float(0), float(0), float(0)\n",
        "    our_opts, early_opts, rand_opts, heu_opts = float(0), float(0), float(0), float(0)\n",
        "    inst_counter = 0\n",
        "    for item in a_instances:\n",
        "        print(\"Instance \" + str(inst_counter + 1) + \"...\")\n",
        "        inst = Instance(item, u, v)\n",
        "        # compute predictions\n",
        "        our_preds, our_probs = inst.compute_predictions(a_apriori_model, a_cond_model, num_ref_jobs)\n",
        "        early_preds, early_probs = inst.compute_early_predictions()\n",
        "        rand_preds, rand_probs = inst.compute_random_predictions()\n",
        "        # call ILP\n",
        "        our_preds = inst.improve_by_ilp(our_preds, our_probs, int(a_fraction * inst.num_jobs))\n",
        "        early_preds = inst.improve_by_ilp(early_preds, early_probs, int(a_fraction * inst.num_jobs))\n",
        "        random_preds = inst.improve_by_ilp(rand_preds, rand_probs, int(a_fraction * inst.num_jobs))\n",
        "        # schedule jobs\n",
        "        inst.schedule_jobs(our_preds)\n",
        "        our_gap = our_gap + 1 - inst.cost / inst.optimal_value\n",
        "        our_opts = our_opts + 1 if inst.cost >= inst.optimal_value else our_opts\n",
        "        inst.clear_schedule()\n",
        "        inst.schedule_jobs(early_preds)\n",
        "        early_gap = early_gap + 1 - inst.cost / inst.optimal_value\n",
        "        early_opts = early_opts + 1 if inst.cost >= inst.optimal_value else early_opts\n",
        "        inst.clear_schedule()\n",
        "        inst.schedule_jobs(rand_preds)\n",
        "        rand_gap = rand_gap + 1 - inst.cost / inst.optimal_value\n",
        "        rand_opts = rand_opts + 1 if inst.cost >= inst.optimal_value else rand_opts\n",
        "        # the same for heuristic\n",
        "        heu_gap = heu_gap + 1 - inst.heuristic_value / inst.optimal_value\n",
        "        heu_opts = heu_opts + 1 if inst.heuristic_value >= inst.optimal_value else heu_opts\n",
        "        inst_counter += 1\n",
        "    our_gap = round(float((our_gap / inst_counter) * 100), 5)\n",
        "    early_gap = round(float((early_gap / inst_counter) * 100), 5)\n",
        "    rand_gap = round(float((rand_gap / inst_counter) * 100), 5)\n",
        "    heu_gap = round(float((heu_gap / inst_counter) * 100), 5)\n",
        "    our_opts = int((our_opts / inst_counter) * 100)\n",
        "    early_opts = int((early_opts / inst_counter) * 100)\n",
        "    rand_opts = int((rand_opts / inst_counter) * 100)\n",
        "    heu_opts = int((heu_opts / inst_counter) * 100)\n",
        "    return f\"{num_jobs} & {our_opts} & {heu_opts} & {rand_opts} & {early_opts} & {our_gap} & {heu_gap} & {rand_gap} & {early_gap}\"\n",
        "\n",
        "\n",
        "def load(a_num_inst, a_num_jobs, a_U, a_V):\n",
        "    input_path = os.path.join(f\"{instances_path}/{a_num_jobs}.pkl\")\n",
        "    my_file = open(input_path, \"rb\")\n",
        "    my_dict = pickle.load(my_file)\n",
        "    my_file.close()\n",
        "    instances = []\n",
        "    counter = 0\n",
        "    for item in my_dict.values():\n",
        "        if counter < a_num_inst and len(item[\"solution\"]) > 0 and item[\"heuristic_value\"] > -1:\n",
        "            instances.append(item)\n",
        "            counter += 1\n",
        "    print(str(counter) + \" instances found\")\n",
        "    return instances\n",
        "\n",
        "\n",
        "def reduction_subset(a_probs, a_num_ilp_jobs):\n",
        "    _, indices = torch.sort(torch.abs(a_probs - 0.5))\n",
        "    return torch.unique(indices[a_num_ilp_jobs:]), torch.unique(indices[:a_num_ilp_jobs])\n",
        "\n",
        "\n",
        "def solve_by_ilp(a_num_jobs, a_weights, a_durations, a_duedates, a_deadlines):\n",
        "    if a_num_jobs == 0:\n",
        "        return torch.tensor([]), -1\n",
        "    jobs = torch.arange(a_num_jobs)\n",
        "    time_moments = torch.unique(torch.cat((a_duedates, a_deadlines)))\n",
        "    m = gp.Model(name=\"Maximizing Weighted Number Of Jobs\")\n",
        "    m.Params.OutputFlag = 0\n",
        "    m.Params.FeasibilityTol = 1e-09\n",
        "    m.Params.OptimalityTol = 1e-09\n",
        "    m.Params.timelimit = 60.0\n",
        "    x = m.addVars(a_num_jobs, vtype=GRB.BINARY, name=\"x\")\n",
        "    m.setObjective(gp.quicksum(a_weights[j] * x[j] for j in range(a_num_jobs)), GRB.MAXIMIZE)\n",
        "    for t in time_moments:\n",
        "        m.addConstr(gp.quicksum(a_durations[i] for i in jobs[a_deadlines <= t])\n",
        "                    + gp.quicksum(\n",
        "            a_durations[i] * x[int(i)] for i in jobs[torch.logical_and(a_deadlines > t, a_duedates <= t)]) <= t)\n",
        "    m.optimize()\n",
        "\n",
        "    if m.Status == GRB.OPTIMAL:\n",
        "        my_vars = torch.empty((a_num_jobs,), dtype=torch.int)\n",
        "        for i in range(a_num_jobs):\n",
        "            my_vars[i] = x[i].x\n",
        "        return my_vars, int(m.objVal)\n",
        "    else:\n",
        "        return torch.tensor([]), -1"
      ],
      "metadata": {
        "id": "6MDKRMJpkKAv"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "PERFORM COMPUTATION:"
      ],
      "metadata": {
        "id": "Q5Iep3TckfAO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "_EC9DGItkQ46"
      },
      "outputs": [],
      "source": [
        "instances = load(num_inst, num_jobs, u, v)\n",
        "\n",
        "apriori_model = NeuralNetwork_Apriori().double()\n",
        "apriori_model.load_state_dict(torch.load(f\"{apriori_model_path}/{u}_{v}.pt\"))\n",
        "\n",
        "cond_model = NeuralNetwork_Conditional().double()\n",
        "cond_model.load_state_dict(torch.load(f\"{conditional_model_path}/{u}_{v}.pt\"))\n",
        "\n",
        "my_results = []\n",
        "for fract in fractions:\n",
        "    my_results.append(collect_stats(instances, apriori_model, cond_model, fract))\n",
        "for item in my_results:\n",
        "    print(item)"
      ]
    }
  ]
}