PCY_Algo_final.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Cz-flkEBpuhm",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Cz-flkEBpuhm",
    "outputId": "6947cd77-21d3-4467-9fa3-e643df767a08"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
      "Collecting pyspark\n",
      "  Downloading pyspark-3.4.0.tar.gz (310.8 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m310.8/310.8 MB\u001b[0m \u001b[31m2.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n",
      "Building wheels for collected packages: pyspark\n",
      "  Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pyspark: filename=pyspark-3.4.0-py2.py3-none-any.whl size=311317130 sha256=1ec1169df2ee12a1ce404d055c2d657ce6ae836403e2ed4fc74de7ed889f5244\n",
      "  Stored in directory: /root/.cache/pip/wheels/7b/1b/4b/3363a1d04368e7ff0d408e57ff57966fcdf00583774e761327\n",
      "Successfully built pyspark\n",
      "Installing collected packages: pyspark\n",
      "Successfully installed pyspark-3.4.0\n"
     ]
    }
   ],
   "source": [
    "!pip install pyspark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "99da018b",
   "metadata": {
    "id": "99da018b"
   },
   "outputs": [],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "from pyspark.sql.functions import col, isnan, when, count, isnull\n",
    "import sys\n",
    "import json\n",
    "from operator import add\n",
    "import string\n",
    "import csv\n",
    "from itertools import combinations\n",
    "import copy\n",
    "import math\n",
    "import time\n",
    "from functools import reduce\n",
    "import numpy as np \n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "331a6b23",
   "metadata": {
    "id": "331a6b23"
   },
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.appName(\"PCY\").getOrCreate()\n",
    "spark_df = spark.read.option(\"header\",True).csv(\"./Gro_dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e23121bd",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e23121bd",
    "outputId": "5adbc407-6665-47be-8ce3-4fca3a1a593d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38765"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "AZgM3zJhq4rE",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "AZgM3zJhq4rE",
    "outputId": "fcdaadd4-35c8-4f69-a283-97d409b926ec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----+---------------+\n",
      "|Member_number|Date|itemDescription|\n",
      "+-------------+----+---------------+\n",
      "|            0|   0|              0|\n",
      "+-------------+----+---------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Missing values\n",
    "missing_counts = spark_df.select([count(when(isnull(c) | isnan(c), c)).alias(c) for c in spark_df.columns])\n",
    "missing_counts.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "79a8da39",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "79a8da39",
    "outputId": "8ad78fdf-736c-4381-d834-9ae6ad511b99"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+----------------+\n",
      "|Member_number|      Date| itemDescription|\n",
      "+-------------+----------+----------------+\n",
      "|         1808|21-07-2015|  tropical fruit|\n",
      "|         2552|05-01-2015|      whole milk|\n",
      "|         2300|19-09-2015|       pip fruit|\n",
      "|         1187|12-12-2015|other vegetables|\n",
      "|         3037|01-02-2015|      whole milk|\n",
      "+-------------+----------+----------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25ca39a0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "25ca39a0",
    "outputId": "b223fecf-50e4-4f12-cc68-b4c35d64a048"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-4d382e29-7f36-4759-98af-504f4f1370db\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>Member_number</th>\n",
       "      <th>Date</th>\n",
       "      <th>itemDescription</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>count</td>\n",
       "      <td>38765</td>\n",
       "      <td>38765</td>\n",
       "      <td>38765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean</td>\n",
       "      <td>3003.64186766413</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stddev</td>\n",
       "      <td>1153.6110310565457</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>min</td>\n",
       "      <td>1000</td>\n",
       "      <td>01-01-2014</td>\n",
       "      <td>Instant food products</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25%</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>50%</td>\n",
       "      <td>3005.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>75%</td>\n",
       "      <td>4007.0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max</td>\n",
       "      <td>5000</td>\n",
       "      <td>31-10-2015</td>\n",
       "      <td>zwieback</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4d382e29-7f36-4759-98af-504f4f1370db')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-4d382e29-7f36-4759-98af-504f4f1370db button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-4d382e29-7f36-4759-98af-504f4f1370db');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "  summary       Member_number        Date        itemDescription\n",
       "0   count               38765       38765                  38765\n",
       "1    mean    3003.64186766413        None                   None\n",
       "2  stddev  1153.6110310565457        None                   None\n",
       "3     min                1000  01-01-2014  Instant food products\n",
       "4     25%              2002.0        None                   None\n",
       "5     50%              3005.0        None                   None\n",
       "6     75%              4007.0        None                   None\n",
       "7     max                5000  31-10-2015               zwieback"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary1 = spark_df.summary().toPandas()\n",
    "summary1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "amWOZzUqwIU6",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "amWOZzUqwIU6",
    "outputId": "206da036-a61c-4860-9735-48ba4b2c341b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Unique item count: 167\n"
     ]
    }
   ],
   "source": [
    "# Count the unique items in a specific column\n",
    "unique_items = spark_df.select(col(\"itemDescription\")).distinct().count()\n",
    "\n",
    "# Print the count\n",
    "print(\"Unique item count:\", unique_items)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "oOHvExexxAXh",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "oOHvExexxAXh",
    "outputId": "904364a3-4a93-4adf-a36f-4651802d5868"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+-----+\n",
      "|     itemDescription|count|\n",
      "+--------------------+-----+\n",
      "|preservation prod...|    1|\n",
      "|     kitchen utensil|    1|\n",
      "|      baby cosmetics|    3|\n",
      "|                bags|    4|\n",
      "|      toilet cleaner|    5|\n",
      "|     rubbing alcohol|    5|\n",
      "|      frozen chicken|    5|\n",
      "|     make up remover|    5|\n",
      "|      salad dressing|    6|\n",
      "|              whisky|    8|\n",
      "|             liqueur|    9|\n",
      "|         decalcifier|    9|\n",
      "|          hair spray|    9|\n",
      "|    organic products|   10|\n",
      "|specialty vegetables|   11|\n",
      "|       frozen fruits|   11|\n",
      "|               cream|   12|\n",
      "|               honey|   13|\n",
      "|   cooking chocolate|   15|\n",
      "|         ready soups|   15|\n",
      "+--------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n",
      "+------------------+-----+\n",
      "|   itemDescription|count|\n",
      "+------------------+-----+\n",
      "|         beverages|  251|\n",
      "|pickled vegetables|  134|\n",
      "|    snack products|   27|\n",
      "|           vinegar|   51|\n",
      "|    red/blush wine|  157|\n",
      "|          cat food|  177|\n",
      "|            dishes|  135|\n",
      "|          cake bar|   93|\n",
      "|   organic sausage|   22|\n",
      "|           napkins|  335|\n",
      "|              meat|  254|\n",
      "|        white wine|  176|\n",
      "| frozen vegetables|  425|\n",
      "|       frozen fish|  102|\n",
      "|             honey|   13|\n",
      "|       frankfurter|  580|\n",
      "| canned vegetables|   82|\n",
      "|           mustard|   92|\n",
      "|     cream cheese |  358|\n",
      "|    kitchen towels|   30|\n",
      "+------------------+-----+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Group the data by the column and count the occurrences in Ascending and descending order\n",
    "value_counts = spark_df.groupBy(col(\"itemDescription\")).count()\n",
    "\n",
    "# Sort the result in ascending order\n",
    "sorted_counts = value_counts.orderBy(col(\"count\"))\n",
    "\n",
    "# Show the sorted result\n",
    "sorted_counts.show()\n",
    "\n",
    "# Show the result\n",
    "value_counts.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "JR258Sj2x7ZB",
   "metadata": {
    "id": "JR258Sj2x7ZB"
   },
   "source": [
    "**We have 167 unique items from above we can assume the minsupport as 25**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6a1bc14",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "d6a1bc14",
    "outputId": "7d81ab61-bb5b-47d7-f06e-5ce3b25c793a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+----------------+\n",
      "|Member_number|      Date| itemDescription|\n",
      "+-------------+----------+----------------+\n",
      "|         1808|21-07-2015|  tropical fruit|\n",
      "|         2552|05-01-2015|      whole milk|\n",
      "|         2300|19-09-2015|       pip fruit|\n",
      "|         1187|12-12-2015|other vegetables|\n",
      "|         3037|01-02-2015|      whole milk|\n",
      "+-------------+----------+----------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spark_df.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7cdb2d2f",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7cdb2d2f",
    "outputId": "148de445-93f1-4620-a98e-665622e195f1"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38006"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Distinct number of records\n",
    "spark_df.distinct().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95aa1cba",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "95aa1cba",
    "outputId": "5b56e691-ac2e-4502-cd56-f5793660942b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38765"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark_df.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "62a33ddb",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "62a33ddb",
    "outputId": "323db063-3fd8-449c-e409-961511d5d27c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+------------------+\n",
      "|   itemDescription|\n",
      "+------------------+\n",
      "|         beverages|\n",
      "|pickled vegetables|\n",
      "|    snack products|\n",
      "|           vinegar|\n",
      "|    red/blush wine|\n",
      "|          cat food|\n",
      "|            dishes|\n",
      "|          cake bar|\n",
      "|   organic sausage|\n",
      "|           napkins|\n",
      "+------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# DF of distinct items\n",
    "items = spark_df.select('itemDescription').distinct()\n",
    "items.show(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1989a391",
   "metadata": {
    "id": "1989a391"
   },
   "source": [
    "#DATA PREPROCESSING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5622e121",
   "metadata": {
    "id": "5622e121"
   },
   "outputs": [],
   "source": [
    "#Grouping all the items into baskets of each member_number\n",
    "from pyspark.sql import functions as F\n",
    "df_grouped = spark_df.groupBy(['Member_number', 'date']).agg(F.collect_list(\"itemDescription\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0337a0e0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "0337a0e0",
    "outputId": "d7d492d4-1357-448d-ff65-d300e83026af"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+-----------------------------+\n",
      "|Member_number|      date|collect_list(itemDescription)|\n",
      "+-------------+----------+-----------------------------+\n",
      "|         1000|15-03-2015|         [sausage, whole m...|\n",
      "|         1000|24-06-2014|         [whole milk, past...|\n",
      "|         1000|24-07-2015|         [canned beer, mis...|\n",
      "|         1000|25-11-2015|         [sausage, hygiene...|\n",
      "|         1000|27-05-2015|         [soda, pickled ve...|\n",
      "|         1001|02-05-2015|          [frankfurter, curd]|\n",
      "|         1001|07-02-2014|         [sausage, whole m...|\n",
      "|         1001|12-12-2014|           [whole milk, soda]|\n",
      "|         1001|14-04-2015|          [beef, white bread]|\n",
      "|         1001|20-01-2015|         [frankfurter, sod...|\n",
      "|         1002|09-02-2014|         [frozen vegetable...|\n",
      "|         1002|26-04-2014|         [butter, whole milk]|\n",
      "|         1002|26-04-2015|         [tropical fruit, ...|\n",
      "|         1002|30-08-2015|         [butter milk, spe...|\n",
      "|         1003|10-02-2015|         [sausage, rolls/b...|\n",
      "|         1003|15-10-2014|         [root vegetables,...|\n",
      "|         1003|19-01-2014|         [frozen meals, de...|\n",
      "|         1003|27-02-2014|         [rolls/buns, roll...|\n",
      "|         1004|01-05-2014|         [dish cleaner, cl...|\n",
      "|         1004|01-07-2014|         [canned beer, fro...|\n",
      "+-------------+----------+-----------------------------+\n",
      "only showing top 20 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_grouped.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dMSgURhNwCJn",
   "metadata": {
    "id": "dMSgURhNwCJn"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "Kd2qqfDDsCZe",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 300
    },
    "id": "Kd2qqfDDsCZe",
    "outputId": "4aa93d38-e680-40aa-b4f7-c2046efeb671"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "  <div id=\"df-3f01db2b-124e-4ac9-8686-df40b6ac53bf\">\n",
       "    <div class=\"colab-df-container\">\n",
       "      <div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>summary</th>\n",
       "      <th>Member_number</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>count</td>\n",
       "      <td>14963</td>\n",
       "      <td>14963</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>mean</td>\n",
       "      <td>3006.840807324734</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>stddev</td>\n",
       "      <td>1152.9474711338305</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>min</td>\n",
       "      <td>1000</td>\n",
       "      <td>01-01-2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25%</td>\n",
       "      <td>2005.0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>50%</td>\n",
       "      <td>3010.0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>75%</td>\n",
       "      <td>4015.0</td>\n",
       "      <td>None</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>max</td>\n",
       "      <td>5000</td>\n",
       "      <td>31-10-2015</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>\n",
       "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3f01db2b-124e-4ac9-8686-df40b6ac53bf')\"\n",
       "              title=\"Convert this dataframe to an interactive table.\"\n",
       "              style=\"display:none;\">\n",
       "        \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
       "       width=\"24px\">\n",
       "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
       "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
       "  </svg>\n",
       "      </button>\n",
       "      \n",
       "  <style>\n",
       "    .colab-df-container {\n",
       "      display:flex;\n",
       "      flex-wrap:wrap;\n",
       "      gap: 12px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert {\n",
       "      background-color: #E8F0FE;\n",
       "      border: none;\n",
       "      border-radius: 50%;\n",
       "      cursor: pointer;\n",
       "      display: none;\n",
       "      fill: #1967D2;\n",
       "      height: 32px;\n",
       "      padding: 0 0 0 0;\n",
       "      width: 32px;\n",
       "    }\n",
       "\n",
       "    .colab-df-convert:hover {\n",
       "      background-color: #E2EBFA;\n",
       "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
       "      fill: #174EA6;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert {\n",
       "      background-color: #3B4455;\n",
       "      fill: #D2E3FC;\n",
       "    }\n",
       "\n",
       "    [theme=dark] .colab-df-convert:hover {\n",
       "      background-color: #434B5C;\n",
       "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
       "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
       "      fill: #FFFFFF;\n",
       "    }\n",
       "  </style>\n",
       "\n",
       "      <script>\n",
       "        const buttonEl =\n",
       "          document.querySelector('#df-3f01db2b-124e-4ac9-8686-df40b6ac53bf button.colab-df-convert');\n",
       "        buttonEl.style.display =\n",
       "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
       "\n",
       "        async function convertToInteractive(key) {\n",
       "          const element = document.querySelector('#df-3f01db2b-124e-4ac9-8686-df40b6ac53bf');\n",
       "          const dataTable =\n",
       "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
       "                                                     [key], {});\n",
       "          if (!dataTable) return;\n",
       "\n",
       "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
       "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
       "            + ' to learn more about interactive tables.';\n",
       "          element.innerHTML = '';\n",
       "          dataTable['output_type'] = 'display_data';\n",
       "          await google.colab.output.renderOutput(dataTable, element);\n",
       "          const docLink = document.createElement('div');\n",
       "          docLink.innerHTML = docLinkHtml;\n",
       "          element.appendChild(docLink);\n",
       "        }\n",
       "      </script>\n",
       "    </div>\n",
       "  </div>\n",
       "  "
      ],
      "text/plain": [
       "  summary       Member_number        date\n",
       "0   count               14963       14963\n",
       "1    mean   3006.840807324734        None\n",
       "2  stddev  1152.9474711338305        None\n",
       "3     min                1000  01-01-2014\n",
       "4     25%              2005.0        None\n",
       "5     50%              3010.0        None\n",
       "6     75%              4015.0        None\n",
       "7     max                5000  31-10-2015"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#After combining all the items to particular bill\n",
    "summary2 = df_grouped.summary().toPandas()\n",
    "summary2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83af924b",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "83af924b",
    "outputId": "cc2250de-e9ab-472c-8b3e-2ee6c1dea941"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Member_number: string (nullable = true)\n",
      " |-- date: string (nullable = true)\n",
      " |-- collect_list(itemDescription): array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_grouped.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "100df950",
   "metadata": {
    "id": "100df950"
   },
   "outputs": [],
   "source": [
    "df_grouped = df_grouped.withColumnRenamed(\"collect_list(itemDescription)\", \"items\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eb7cdac8",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "eb7cdac8",
    "outputId": "15e32145-2df7-4923-ed16-f6d65c7907c8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- Member_number: string (nullable = true)\n",
      " |-- date: string (nullable = true)\n",
      " |-- items: array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_grouped.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b62e2003",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "b62e2003",
    "outputId": "0f32b83b-9c40-4488-bd30-34a39244c87b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------------+----------+--------------------+\n",
      "|Member_number|      date|               items|\n",
      "+-------------+----------+--------------------+\n",
      "|         1000|15-03-2015|[sausage, whole m...|\n",
      "|         1000|24-06-2014|[whole milk, past...|\n",
      "|         1000|24-07-2015|[canned beer, mis...|\n",
      "|         1000|25-11-2015|[sausage, hygiene...|\n",
      "|         1000|27-05-2015|[soda, pickled ve...|\n",
      "|         1001|02-05-2015| [frankfurter, curd]|\n",
      "|         1001|07-02-2014|[sausage, whole m...|\n",
      "|         1001|12-12-2014|  [whole milk, soda]|\n",
      "|         1001|14-04-2015| [beef, white bread]|\n",
      "|         1001|20-01-2015|[frankfurter, sod...|\n",
      "+-------------+----------+--------------------+\n",
      "only showing top 10 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "df_grouped.show(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "786a16f3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "786a16f3",
    "outputId": "6c078027-c1d7-4e0e-d7ce-237f1a5ccda6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NUmber of unique customers :\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3898"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"NUmber of unique customers :\")\n",
    "df_grouped.select('Member_number').distinct().count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68370770",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "68370770",
    "outputId": "4a465cb9-3448-48e7-b567-2377d978d20a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14963"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_grouped.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24b353f3",
   "metadata": {
    "id": "24b353f3"
   },
   "outputs": [],
   "source": [
    "# Saving the transactions as csv file\n",
    "df_grouped.toPandas().loc[:, [\"items\"]].to_csv(\"transactions.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8cab2d9",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "e8cab2d9",
    "outputId": "76ee7665-ad69-4eef-a598-ae622ad30dcf"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- date: string (nullable = true)\n",
      " |-- items: array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Dropping member_number and date columns\n",
    "df = df_grouped.drop(\"Member_number\")\n",
    "df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab5d0c27",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "ab5d0c27",
    "outputId": "068a4676-1a72-4d10-f328-b29ff2123aec"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- items: array (nullable = false)\n",
      " |    |-- element: string (containsNull = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "transactions_with_header = df.drop(\"date\")\n",
    "transactions_with_header.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "528463b3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "528463b3",
    "outputId": "191fb1d2-f0b9-4361-8bf2-0ede2d7a29a3"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|               items|\n",
      "+--------------------+\n",
      "|[sausage, whole m...|\n",
      "|[whole milk, past...|\n",
      "|[canned beer, mis...|\n",
      "|[sausage, hygiene...|\n",
      "|[soda, pickled ve...|\n",
      "+--------------------+\n",
      "only showing top 5 rows\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#Total items \n",
    "transactions_with_header.show(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "416213d3",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "416213d3",
    "outputId": "040442c9-7db3-4444-9eb7-906ec9e17aac"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14963"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transactions_with_header.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9934aef",
   "metadata": {
    "id": "b9934aef"
   },
   "outputs": [],
   "source": [
    "transactions_with_header.toPandas().loc[:, [\"items\"]].to_csv('transactions.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "864f05a6",
   "metadata": {
    "id": "864f05a6"
   },
   "source": [
    "## PCY Implementation Using RDD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7192ddc0",
   "metadata": {
    "id": "7192ddc0"
   },
   "outputs": [],
   "source": [
    "def openfile():\n",
    "    file_name = \"transactions.csv\"\n",
    "    minsupport = 20\n",
    "    nbuckets=20\n",
    "    \n",
    "    sc = spark.sparkContext\n",
    "    data = sc.textFile(file_name)\n",
    "#     data = data.map(lambda line : ''.join(line.split(',')))\n",
    "    newfile = data.collect()\n",
    "    \n",
    "    buckets=[]\n",
    "    for x in newfile:\n",
    "       buckets.append([y.strip() for y in x.split(',')])\n",
    "\n",
    "    size1freqset(minsupport,nbuckets,buckets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10eb910d",
   "metadata": {
    "id": "10eb910d"
   },
   "outputs": [],
   "source": [
    "def size1freqset(minsupport,nbuckets,buckets):\n",
    "\n",
    "    candidatelist1=[]\n",
    "    finalist1=[]\n",
    "\n",
    "    for i in range(0, len(buckets)):\n",
    "        for b in buckets[i]:\n",
    "            candidatelist1.append(b)\n",
    "\n",
    "    candidatelist1=sorted(set(candidatelist1))\n",
    "    # print(candidatelist1)\n",
    "    \n",
    "    lala=0\n",
    "    for k in candidatelist1:\n",
    "        for i in range(0, len(buckets)):\n",
    "            for j in buckets[i]:\n",
    "                if(k==j):\n",
    "                    lala+=1\n",
    "        if lala>=minsupport:\n",
    "            finalist1.append(k)\n",
    "        lala=0\n",
    "\n",
    "    itemiddic={}\n",
    "    counter=1\n",
    "    for c in candidatelist1:\n",
    "        itemiddic[c]=counter\n",
    "        counter+=1\n",
    "\n",
    "    if finalist1:\n",
    "        print(\"Frequent Itemsets of size 1\")\n",
    "        with open(\"output1.txt\", \"w\") as f:\n",
    "            f.write(\"Frequent Itemsets of size 1 \\n\" + \"Total number of itemsets : \" + str(len(finalist1)) + '\\n'.join(finalist1))\n",
    "        for x in finalist1:\n",
    "            print(x)\n",
    "        size2freqset(minsupport,nbuckets,itemiddic,buckets,finalist1)\n",
    "    else:\n",
    "        print(\"That's all folks!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e80ab35",
   "metadata": {
    "id": "2e80ab35"
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "def size2freqset(minsupport,nbuckets,itemiddic,buckets,finalist1):\n",
    "\n",
    "    k=3\n",
    "\n",
    "    countofbuckets=[0]*nbuckets\n",
    "    bitmap=[0]*nbuckets\n",
    "    pairs=[]\n",
    "\n",
    "    \"\"\"PCY Pass 1\"\"\"\n",
    "    for i in range(0,len(buckets)):\n",
    "        for x in range(0,len(buckets[i])-1):\n",
    "            for y in range(x+1,len(buckets[i])):\n",
    "                if(buckets[i][x]<buckets[i][y]):\n",
    "                    countofbuckets[int(str(itemiddic[buckets[i][x]])+str(itemiddic[buckets[i][y]]))%nbuckets]+=1\n",
    "                    if ([buckets[i][x],buckets[i][y]] not in pairs):\n",
    "                        pairs.append(sorted([buckets[i][x],buckets[i][y]]))\n",
    "                else:\n",
    "                    countofbuckets[int(str(itemiddic[buckets[i][y]])+str(itemiddic[buckets[i][x]]))%nbuckets]+=1\n",
    "                    if ([buckets[i][y],buckets[i][x]] not in pairs):\n",
    "                        pairs.append(sorted([buckets[i][y],buckets[i][x]]))\n",
    "\n",
    "    pairs=sorted(pairs)\n",
    "\n",
    "    for x in range(0,len(countofbuckets)):\n",
    "        if countofbuckets[x]>=minsupport:\n",
    "            bitmap[x]=1\n",
    "        else:\n",
    "            bitmap[x]=0\n",
    "\n",
    "    prunedpairs=[]\n",
    "\n",
    "    \"\"\"Checking condition 1 of PCY Pass 2\"\"\"\n",
    "\n",
    "    for i in range(0,len(pairs)):\n",
    "        for j in range(0,len(pairs[i])-1):\n",
    "            if(pairs[i][j] in finalist1 and pairs[i][j+1] in finalist1):\n",
    "                prunedpairs.append(pairs[i])\n",
    "\n",
    "    candidatelist2=[]\n",
    "\n",
    "    \"\"\"Checking condition 2 of PCY Pass 2\"\"\"\n",
    "    for i in range(0, len(prunedpairs)):\n",
    "        for j in range(0,len(prunedpairs[i])-1):\n",
    "            if bitmap[int(str(itemiddic[prunedpairs[i][j]])+str(itemiddic[prunedpairs[i][j+1]]))%nbuckets]==1:\n",
    "                candidatelist2.append(prunedpairs[i])\n",
    "\n",
    "    \"\"\"Appending frequent items of size 2 to finallist2\"\"\"\n",
    "    finalist2=[]\n",
    "    p=0\n",
    "    for c in range(0,len(candidatelist2)):\n",
    "        for b in range(0,len(buckets)):\n",
    "            if set(candidatelist2[c]).issubset(set(buckets[b])):\n",
    "                p+=1\n",
    "        if p>=minsupport and p!=0:\n",
    "            finalist2.append(sorted(candidatelist2[c]))\n",
    "        p=0\n",
    "\n",
    "    finalist2=sorted(finalist2)\n",
    "\n",
    "    if finalist2:\n",
    "        print(\"\\nFrequent Itemsets of size 2\")\n",
    "        sys.set_int_max_str_digits(5016)\n",
    "        with open(\"output2.txt\", \"w\") as f:\n",
    "            f.write(\"Frequent Itemsets of size 2 \\n\" + \"Total number of itemsets : \" + str(len(finalist1)) + '\\n'.join([','.join(map(str, item)) for item in finalist2])) #(finalist2))\n",
    "        for b in finalist2:\n",
    "            print(','.join(b))\n",
    "        sizekfreqset(minsupport,nbuckets,itemiddic,buckets,finalist2,k)\n",
    "    else:\n",
    "        print(\"End\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86da403b",
   "metadata": {
    "id": "86da403b"
   },
   "outputs": [],
   "source": [
    "def sizekfreqset(minsupport,nbuckets,itemiddic,buckets,prevout,k):\n",
    "\n",
    "\t\"\"\"Candidate list\"\"\"\n",
    "\t\n",
    "\tbucketcountk=[0]*nbuckets\n",
    "\tbitmapofsizk=[0]*nbuckets\n",
    "\tkcombination=[]\n",
    "\tprevout=prevout\n",
    "\t\n",
    "\t#k combinations\n",
    "\tfor a in prevout:\n",
    "\t\tfor b in prevout:\n",
    "\t\t\tif(a!=b):\n",
    "\t\t\t\tif set(a) & set(b) and len(list(set(a) & set(b))) >= k-2:\n",
    "\t\t\t\t\tkcombination.append(sorted(set(a)|set(b)))\n",
    "\n",
    "\t# print(\"checking combination of size k\")\n",
    "\tkcombination=sorted(kcombination)\n",
    "\n",
    "\t\"\"\"PCY Pass 1 Hashing\"\"\"\n",
    "\n",
    "\thashingstring=\"\"\n",
    "\n",
    "\tfor i in range(0,len(kcombination)):\n",
    "\t\tfor x in kcombination[i]:\n",
    "\t\t\thashingstring+=str(itemiddic[x])\n",
    "\t\tbucketcountk[int(hashingstring)%nbuckets]+=1\n",
    "\t\thashingstring=\"\"\n",
    "\n",
    "\tfor x in range(0,len(bucketcountk)):\n",
    "\t\tif bucketcountk[x]>=minsupport:\n",
    "\t\t\tbitmapofsizk[x]=1\n",
    "\t\telse:\n",
    "\t\t\tbitmapofsizk[x]=0\n",
    "\n",
    "\t\"\"\"Condition 1 is automatically satisfied\"\"\"\n",
    "\n",
    "\t\"\"\"Checking condition 2 of PCY Pass 2\"\"\"\n",
    "\thashingstring=\"\"\n",
    "\tklist=[]\n",
    "\t\n",
    "\tfor i in range(0, len(kcombination)):\n",
    "\t\tfor j in kcombination[i]:\n",
    "\t\t\thashingstring+=str(itemiddic[j])\n",
    "\t\tif bitmapofsizk[int(hashingstring)%nbuckets]==1:\n",
    "\t\t\tklist.append(kcombination[i])\n",
    "\t\tteststring=\"\"\n",
    "\n",
    "\t\"\"\"Creating Candidate List of Size k\"\"\"\n",
    "\tcandidatelistk=[]\n",
    "\tcount=1\n",
    "\tfor i in range(1, len(klist)):\n",
    "\t\tif klist[i]==klist[i-1]:\n",
    "\t\t\tcount+=1\n",
    "\t\telse:\n",
    "\t\t\t# print(\"ELement is: \",klist[i-1],\" and count is: \",count)\n",
    "\t\t\tif count>=k:\n",
    "\t\t\t\tcandidatelistk.append(klist[i-1])\n",
    "\t\t\tcount=1\n",
    "\tif count>=k:\n",
    "\t\tcandidatelistk.append(klist[i-1])\n",
    "\t\t# print(\"ELement is: \",klist[i-1],\" and count is: \",count)\n",
    "\t\n",
    "\t\"\"\"Appending frequent items of size k to output\"\"\"\n",
    "\toutput=[]\n",
    "\tcounter=0\n",
    "\tfor c in range(0,len(candidatelistk)):\n",
    "\t\tfor b in range(0,len(buckets)):\n",
    "\t\t\tif set(candidatelistk[c]).issubset(set(buckets[b])):\n",
    "\t\t\t\tcounter+=1\n",
    "\t\tif counter>=minsupport and counter!=0:\n",
    "\t\t\toutput.append(candidatelistk[c])\n",
    "\t\tcounter=0\n",
    "\n",
    "\tif output:\n",
    "\t\tprint(\"\\nFrequent Itemsets of size \",k)\n",
    "\t\tfor b in output:\n",
    "\t\t\tprint(','.join(b))\n",
    "\t\tk+=1\n",
    "\t\tsizekfreqset(minsupport,nbuckets,itemiddic,buckets,output,k)\n",
    "\t\t# print(\"\\n\")\n",
    "\telse:\n",
    "\t\tprint(\"End\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5af150fd",
   "metadata": {
    "colab": {
     "background_save": true,
     "base_uri": "https://localhost:8080/"
    },
    "id": "5af150fd",
    "outputId": "d438583b-875c-4a1d-c440-672d149d6580"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Frequent Itemsets of size 1\n",
      "\"['UHT-milk'\n",
      "\"['baking powder'\n",
      "\"['beef'\n",
      "\"['berries'\n",
      "\"['beverages'\n",
      "\"['bottled beer'\n",
      "\"['bottled water'\n",
      "\"['brown bread'\n",
      "\"['butter milk'\n",
      "\"['butter'\n",
      "\"['candy'\n",
      "\"['canned beer'\n",
      "\"['canned fish'\n",
      "\"['cat food'\n",
      "\"['chewing gum'\n",
      "\"['chicken'\n",
      "\"['chocolate'\n",
      "\"['citrus fruit'\n",
      "\"['coffee'\n",
      "\"['condensed milk'\n",
      "\"['cream cheese '\n",
      "\"['curd'\n",
      "\"['dessert'\n",
      "\"['detergent'\n",
      "\"['dishes'\n",
      "\"['domestic eggs'\n",
      "\"['finished products'\n",
      "\"['fish'\n",
      "\"['frankfurter'\n",
      "\"['frozen dessert'\n",
      "\"['frozen fish'\n",
      "\"['frozen meals'\n",
      "\"['frozen vegetables'\n",
      "\"['fruit/vegetable juice'\n",
      "\"['grapes'\n",
      "\"['ham'\n",
      "\"['hamburger meat'\n",
      "\"['hard cheese'\n",
      "\"['herbs'\n",
      "\"['hygiene articles'\n",
      "\"['ice cream'\n",
      "\"['liquor'\n",
      "\"['liver loaf'\n",
      "\"['long life bakery product'\n",
      "\"['margarine'\n",
      "\"['meat'\n",
      "\"['misc. beverages'\n",
      "\"['napkins'\n",
      "\"['newspapers'\n",
      "\"['oil'\n",
      "\"['onions'\n",
      "\"['other vegetables'\n",
      "\"['packaged fruit/vegetables'\n",
      "\"['pasta'\n",
      "\"['pastry'\n",
      "\"['pet care'\n",
      "\"['photo/film'\n",
      "\"['pickled vegetables'\n",
      "\"['pip fruit'\n",
      "\"['pork'\n",
      "\"['pot plants'\n",
      "\"['processed cheese'\n",
      "\"['red/blush wine'\n",
      "\"['rolls/buns'\n",
      "\"['root vegetables'\n",
      "\"['salt'\n",
      "\"['salty snack'\n",
      "\"['sausage'\n",
      "\"['seasonal products'\n",
      "\"['semi-finished bread'\n",
      "\"['shopping bags'\n",
      "\"['sliced cheese'\n",
      "\"['soda'\n",
      "\"['soft cheese'\n",
      "\"['specialty bar'\n",
      "\"['specialty chocolate'\n",
      "\"['spread cheese'\n",
      "\"['sugar'\n",
      "\"['tropical fruit'\n",
      "\"['turkey'\n",
      "\"['waffles'\n",
      "\"['whipped/sour cream'\n",
      "\"['white bread'\n",
      "\"['white wine'\n",
      "\"['whole milk'\n",
      "\"['yogurt'\n",
      "\"['zwieback'\n",
      "'Instant food products']\"\n",
      "'UHT-milk'\n",
      "'UHT-milk']\"\n",
      "'baking powder'\n",
      "'baking powder']\"\n",
      "'beef'\n",
      "'beef']\"\n",
      "'berries'\n",
      "'berries']\"\n",
      "'beverages'\n",
      "'beverages']\"\n",
      "'bottled beer'\n",
      "'bottled beer']\"\n",
      "'bottled water'\n",
      "'bottled water']\"\n",
      "'brandy']\"\n",
      "'brown bread'\n",
      "'brown bread']\"\n",
      "'butter milk'\n",
      "'butter milk']\"\n",
      "'butter'\n",
      "'butter']\"\n",
      "'cake bar'\n",
      "'cake bar']\"\n",
      "'candles']\"\n",
      "'candy'\n",
      "'candy']\"\n",
      "'canned beer'\n",
      "'canned beer']\"\n",
      "'canned fish'\n",
      "'canned fish']\"\n",
      "'canned vegetables']\"\n",
      "'cat food'\n",
      "'cat food']\"\n",
      "'cereals']\"\n",
      "'chewing gum'\n",
      "'chewing gum']\"\n",
      "'chicken'\n",
      "'chicken']\"\n",
      "'chocolate marshmallow']\"\n",
      "'chocolate'\n",
      "'chocolate']\"\n",
      "'citrus fruit'\n",
      "'citrus fruit']\"\n",
      "'cling film/bags']\"\n",
      "'coffee'\n",
      "'coffee']\"\n",
      "'condensed milk'\n",
      "'condensed milk']\"\n",
      "'cream cheese '\n",
      "'cream cheese ']\"\n",
      "'curd cheese']\"\n",
      "'curd'\n",
      "'curd']\"\n",
      "'dental care']\"\n",
      "'dessert'\n",
      "'dessert']\"\n",
      "'detergent'\n",
      "'detergent']\"\n",
      "'dish cleaner']\"\n",
      "'dishes'\n",
      "'dishes']\"\n",
      "'dog food']\"\n",
      "'domestic eggs'\n",
      "'domestic eggs']\"\n",
      "'female sanitary products']\"\n",
      "'flour'\n",
      "'flour']\"\n",
      "'flower (seeds)']\"\n",
      "'frankfurter'\n",
      "'frozen dessert'\n",
      "'frozen dessert']\"\n",
      "'frozen fish'\n",
      "'frozen fish']\"\n",
      "'frozen meals'\n",
      "'frozen meals']\"\n",
      "'frozen potato products']\"\n",
      "'frozen vegetables'\n",
      "'frozen vegetables']\"\n",
      "'fruit/vegetable juice'\n",
      "'fruit/vegetable juice']\"\n",
      "'grapes'\n",
      "'grapes']\"\n",
      "'ham'\n",
      "'ham']\"\n",
      "'hamburger meat'\n",
      "'hamburger meat']\"\n",
      "'hard cheese'\n",
      "'hard cheese']\"\n",
      "'herbs'\n",
      "'herbs']\"\n",
      "'house keeping products']\"\n",
      "'hygiene articles'\n",
      "'hygiene articles']\"\n",
      "'ice cream'\n",
      "'ice cream']\"\n",
      "'instant coffee']\"\n",
      "'liquor (appetizer)']\"\n",
      "'liquor'\n",
      "'liquor']\"\n",
      "'long life bakery product'\n",
      "'long life bakery product']\"\n",
      "'margarine'\n",
      "'margarine']\"\n",
      "'mayonnaise']\"\n",
      "'meat'\n",
      "'meat']\"\n",
      "'misc. beverages'\n",
      "'misc. beverages']\"\n",
      "'mustard'\n",
      "'mustard']\"\n",
      "'napkins'\n",
      "'napkins']\"\n",
      "'newspapers'\n",
      "'newspapers']\"\n",
      "'oil'\n",
      "'oil']\"\n",
      "'onions'\n",
      "'onions']\"\n",
      "'other vegetables'\n",
      "'other vegetables']\"\n",
      "'packaged fruit/vegetables'\n",
      "'packaged fruit/vegetables']\"\n",
      "'pasta'\n",
      "'pasta']\"\n",
      "'pastry'\n",
      "'pastry']\"\n",
      "'pet care']\"\n",
      "'photo/film']\"\n",
      "'pickled vegetables'\n",
      "'pickled vegetables']\"\n",
      "'pip fruit'\n",
      "'pip fruit']\"\n",
      "'popcorn']\"\n",
      "'pork'\n",
      "'pork']\"\n",
      "'pot plants'\n",
      "'pot plants']\"\n",
      "'processed cheese'\n",
      "'processed cheese']\"\n",
      "'red/blush wine'\n",
      "'red/blush wine']\"\n",
      "'rice']\"\n",
      "'roll products '\n",
      "'roll products ']\"\n",
      "'rolls/buns'\n",
      "'rolls/buns']\"\n",
      "'root vegetables'\n",
      "'root vegetables']\"\n",
      "'salt']\"\n",
      "'salty snack'\n",
      "'salty snack']\"\n",
      "'sauces']\"\n",
      "'sausage'\n",
      "'sausage']\"\n",
      "'seasonal products'\n",
      "'seasonal products']\"\n",
      "'semi-finished bread'\n",
      "'semi-finished bread']\"\n",
      "'shopping bags'\n",
      "'shopping bags']\"\n",
      "'sliced cheese'\n",
      "'sliced cheese']\"\n",
      "'soda'\n",
      "'soda']\"\n",
      "'soft cheese'\n",
      "'soft cheese']\"\n",
      "'softener']\"\n",
      "'soups']\"\n",
      "'sparkling wine']\"\n",
      "'specialty bar'\n",
      "'specialty bar']\"\n",
      "'specialty cheese']\"\n",
      "'specialty chocolate'\n",
      "'specialty chocolate']\"\n",
      "'spices']\"\n",
      "'spread cheese'\n",
      "'spread cheese']\"\n",
      "'sugar'\n",
      "'sugar']\"\n",
      "'sweet spreads']\"\n",
      "'tropical fruit'\n",
      "'tropical fruit']\"\n",
      "'vinegar']\"\n",
      "'waffles'\n",
      "'waffles']\"\n",
      "'whipped/sour cream'\n",
      "'whipped/sour cream']\"\n",
      "'white bread'\n",
      "'white bread']\"\n",
      "'white wine'\n",
      "'white wine']\"\n",
      "'whole milk'\n",
      "'whole milk']\"\n",
      "'yogurt'\n",
      "'yogurt']\"\n",
      "'zwieback']\"\n",
      "\n",
      "Frequent Itemsets of size 2\n",
      "\"['beef','whole milk'\n",
      "\"['bottled beer','whole milk']\"\n",
      "\"['chicken','whole milk'\n",
      "\"['citrus fruit','other vegetables']\"\n",
      "\"['citrus fruit','rolls/buns']\"\n",
      "\"['citrus fruit','soda']\"\n",
      "\"['citrus fruit','whole milk'\n",
      "\"['citrus fruit','whole milk']\"\n",
      "\"['citrus fruit','yogurt']\"\n",
      "\"['frankfurter','other vegetables'\n",
      "\"['frankfurter','other vegetables']\"\n",
      "\"['frankfurter','rolls/buns']\"\n",
      "\"['frankfurter','soda']\"\n",
      "\"['frankfurter','whole milk'\n",
      "\"['frankfurter','whole milk']\"\n",
      "\"['other vegetables','bottled water']\"\n",
      "\"['other vegetables','newspapers']\"\n",
      "\"['other vegetables','other vegetables'\n",
      "\"['other vegetables','other vegetables']\"\n",
      "\"['other vegetables','rolls/buns'\n",
      "\"['other vegetables','rolls/buns']\"\n",
      "\"['other vegetables','shopping bags']\"\n",
      "\"['other vegetables','soda'\n",
      "\"['other vegetables','soda']\"\n",
      "\"['other vegetables','whipped/sour cream']\"\n",
      "\"['other vegetables','whole milk'\n",
      "\"['other vegetables','whole milk']\"\n",
      "\"['other vegetables','yogurt'\n",
      "\"['other vegetables','yogurt']\"\n",
      "\"['pip fruit','other vegetables']\"\n",
      "\"['pip fruit','pastry']\"\n",
      "\"['pip fruit','rolls/buns']\"\n",
      "\"['pip fruit','soda']\"\n",
      "\"['pip fruit','whole milk'\n",
      "\"['pip fruit','whole milk']\"\n",
      "\"['pork','other vegetables']\"\n",
      "\"['pork','rolls/buns']\"\n",
      "\"['pork','whole milk'\n",
      "\"['pork','whole milk']\"\n",
      "\"['rolls/buns','fruit/vegetable juice']\"\n",
      "\"['rolls/buns','other vegetables']\"\n",
      "\"['rolls/buns','rolls/buns']\"\n",
      "\"['rolls/buns','shopping bags']\"\n",
      "\"['rolls/buns','soda']\"\n",
      "\"['rolls/buns','whole milk'\n",
      "\"['rolls/buns','whole milk']\"\n",
      "\"['rolls/buns','yogurt']\"\n",
      "\"['root vegetables','rolls/buns']\"\n",
      "\"['root vegetables','shopping bags']\"\n",
      "\"['root vegetables','soda']\"\n",
      "\"['root vegetables','whole milk']\"\n",
      "\"['sausage','bottled beer'\n",
      "\"['sausage','bottled water']\"\n",
      "\"['sausage','other vegetables'\n",
      "\"['sausage','other vegetables']\"\n",
      "\"['sausage','pastry'\n",
      "\"['sausage','rolls/buns'\n",
      "\"['sausage','rolls/buns']\"\n",
      "\"['sausage','soda'\n",
      "\"['sausage','soda']\"\n",
      "\"['sausage','whole milk'\n",
      "\"['sausage','whole milk']\"\n",
      "\"['sausage','yogurt']\"\n",
      "\"['soda','other vegetables']\"\n",
      "\"['soda','rolls/buns']\"\n",
      "\"['soda','shopping bags']\"\n",
      "\"['soda','soda']\"\n",
      "\"['soda','whole milk']\"\n",
      "\"['tropical fruit','other vegetables'\n",
      "\"['tropical fruit','other vegetables']\"\n",
      "\"['tropical fruit','rolls/buns']\"\n",
      "\"['tropical fruit','soda']\"\n",
      "\"['tropical fruit','whole milk'\n",
      "\"['tropical fruit','whole milk']\"\n",
      "\"['tropical fruit','yogurt']\"\n",
      "\"['whole milk','bottled beer']\"\n",
      "\"['whole milk','bottled water']\"\n",
      "\"['whole milk','fruit/vegetable juice']\"\n",
      "\"['whole milk','margarine']\"\n",
      "\"['whole milk','newspapers']\"\n",
      "\"['whole milk','other vegetables'\n",
      "\"['whole milk','other vegetables']\"\n",
      "\"['whole milk','pastry']\"\n",
      "\"['whole milk','rolls/buns'\n",
      "\"['whole milk','rolls/buns']\"\n",
      "\"['whole milk','root vegetables']\"\n",
      "\"['whole milk','shopping bags']\"\n",
      "\"['whole milk','soda'\n",
      "\"['whole milk','soda']\"\n",
      "\"['whole milk','whipped/sour cream']\"\n",
      "\"['whole milk','whole milk'\n",
      "\"['whole milk','whole milk']\"\n",
      "\"['whole milk','yogurt'\n",
      "\"['whole milk','yogurt']\"\n",
      "\"['yogurt','rolls/buns']\"\n",
      "\"['yogurt','whole milk']\"\n",
      "\"['yogurt','yogurt']\"\n",
      "'UHT-milk','UHT-milk'\n",
      "'beef','beef'\n",
      "'bottled water','bottled water'\n",
      "'bottled water']\",'whole milk'\n",
      "'brown bread','brown bread'\n",
      "'cake bar','cake bar'\n",
      "'canned beer','canned beer'\n",
      "'chewing gum','chewing gum'\n",
      "'chicken','chicken'\n",
      "'chocolate','chocolate'\n",
      "'citrus fruit','citrus fruit'\n",
      "'citrus fruit','whole milk'\n",
      "'coffee','coffee'\n",
      "'cream cheese ','cream cheese '\n",
      "'curd','curd'\n",
      "'dessert','dessert'\n",
      "'domestic eggs','domestic eggs'\n",
      "'frankfurter','frankfurter'\n",
      "'frozen vegetables','frozen vegetables'\n",
      "'fruit/vegetable juice','fruit/vegetable juice'\n",
      "'fruit/vegetable juice','whole milk'\n",
      "'grapes','grapes'\n",
      "'herbs','herbs'\n",
      "'hygiene articles','hygiene articles'\n",
      "'margarine','margarine'\n",
      "'newspapers','newspapers'\n",
      "'other vegetables','other vegetables'\n",
      "'other vegetables','rolls/buns'\n",
      "'other vegetables','rolls/buns']\"\n",
      "'other vegetables','sausage'\n",
      "'other vegetables','soda'\n",
      "'other vegetables','whole milk'\n",
      "'other vegetables','yogurt'\n",
      "'packaged fruit/vegetables','packaged fruit/vegetables'\n",
      "'pastry','whole milk'\n",
      "'pip fruit','pip fruit'\n",
      "'pork','pork'\n",
      "'rolls/buns','rolls/buns'\n",
      "'rolls/buns','rolls/buns']\"\n",
      "'rolls/buns','root vegetables'\n",
      "'rolls/buns','shopping bags']\"\n",
      "'rolls/buns','soda'\n",
      "'rolls/buns','soda']\"\n",
      "'rolls/buns','whole milk'\n",
      "'rolls/buns','yogurt'\n",
      "'rolls/buns']\",'whole milk'\n",
      "'root vegetables','root vegetables'\n",
      "'root vegetables','whole milk'\n",
      "'sausage','sausage'\n",
      "'sausage','whole milk'\n",
      "'sausage','yogurt'\n",
      "'shopping bags','shopping bags'\n",
      "'shopping bags']\",'whole milk'\n",
      "'soda','soda'\n",
      "'soda','whole milk'\n",
      "'soda']\",'whole milk'\n",
      "'soft cheese','soft cheese'\n",
      "'specialty chocolate','specialty chocolate'\n",
      "'tropical fruit','tropical fruit'\n",
      "'tropical fruit','whole milk'\n",
      "'tropical fruit','yogurt'\n",
      "'whipped/sour cream','whipped/sour cream'\n",
      "'whole milk','whole milk'\n",
      "'whole milk','whole milk']\"\n",
      "'whole milk','yogurt'\n",
      "'yogurt','yogurt'\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-75-5e8bb430cf83>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mopenfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-63-1719b1a1199d>\u001b[0m in \u001b[0;36mopenfile\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m        \u001b[0mbuckets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0my\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m     \u001b[0msize1freqset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminsupport\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnbuckets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbuckets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-64-aa4dfbed6308>\u001b[0m in \u001b[0;36msize1freqset\u001b[0;34m(minsupport, nbuckets, buckets)\u001b[0m\n\u001b[1;32m     33\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfinalist1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0msize2freqset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminsupport\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnbuckets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mitemiddic\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbuckets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfinalist1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"That's all folks!\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-73-28f9ba7aa808>\u001b[0m in \u001b[0;36msize2freqset\u001b[0;34m(minsupport, nbuckets, itemiddic, buckets, finalist1)\u001b[0m\n\u001b[1;32m     66\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mb\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mfinalist2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     67\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m','\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mb\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 68\u001b[0;31m         \u001b[0msizekfreqset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mminsupport\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mnbuckets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mitemiddic\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mbuckets\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mfinalist2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     69\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"End\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-74-ce6462589e73>\u001b[0m in \u001b[0;36msizekfreqset\u001b[0;34m(minsupport, nbuckets, itemiddic, buckets, prevout, k)\u001b[0m\n\u001b[1;32m     43\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mkcombination\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     44\u001b[0m                         \u001b[0mhashingstring\u001b[0m\u001b[0;34m+=\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitemiddic\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 45\u001b[0;31m                 \u001b[0;32mif\u001b[0m \u001b[0mbitmapofsizk\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhashingstring\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0mnbuckets\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     46\u001b[0m                         \u001b[0mklist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkcombination\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m                 \u001b[0mteststring\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mValueError\u001b[0m: Exceeds the limit (5016) for integer string conversion: value has 5025 digits; use sys.set_int_max_str_digits() to increase the limit"
     ]
    }
   ],
   "source": [
    "openfile()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8P_tNAorAB8h",
   "metadata": {
    "colab": {
     "background_save": true
    },
    "id": "8P_tNAorAB8h"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}