pre-ass.ipynb 10.24.38 pm copy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 计算文件各种类型、属性的数量\n",
    "### 测试有多少个值未知"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "\n",
    "import math\n",
    "from collections import defaultdict\n",
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(filename, headers):\n",
    "    count_class = {class_name:0 for class_name in list(filter(lambda x : x[0] == \"class\", headers))[0][1]}\n",
    "    count_nominal = {att:defaultdict(int) for att,att_type in headers if att != \"class\" and att_type < 2}\n",
    "    count_numeric = defaultdict(int)\n",
    "    count_numeric_unknown = defaultdict(int)\n",
    "    with open(filename,\"r\") as file:\n",
    "        for line in file:\n",
    "            \n",
    "            cels = line.strip().split(\",\")\n",
    "            if not cels or not cels[0] :\n",
    "                continue               \n",
    "            assert len(cels) == len(headers)\n",
    "            \n",
    "            i = 0\n",
    "            for name,data_type in headers:\n",
    "                val = cels[i]\n",
    "                if name == \"class\":\n",
    "                    assert val in data_type\n",
    "                    count_class[val] += 1\n",
    "                elif data_type < 2:\n",
    "                    count_nominal[name][val] += 1\n",
    "                else:\n",
    "                    assert val.isdigit() or val == \"?\" \n",
    "                    if val.isdigit():\n",
    "                        count_numeric[name] += 1\n",
    "                    else:\n",
    "                        count_numeric_unknown[name] += 1 \n",
    "                i+=1\n",
    "    \n",
    "    total = reduce(lambda x, y: x+y, count_class.values())\n",
    "    print(\"total instances: \", total)\n",
    "    print(\"{0}:\".format(\"class\"), \" / \".join([\"{0} {1}\".format(key, val) for key,val in sorted(count_class.items())]))\n",
    "    print(\"--------------------attributes--------------------\")\n",
    "    for name,data_type in headers:\n",
    "        if name != \"class\":\n",
    "            if data_type < 2:\n",
    "                assert total == reduce(lambda x, y: x+y, count_nominal[name].values())\n",
    "                print(\"{0}:\\n   \".format(name), \"\\n    \".join([\"{0} {1}\".format(key, val) for key,val in sorted(count_nominal[name].items())]))\n",
    "            else:\n",
    "                assert total == count_numeric[name] + count_numeric_unknown[name]\n",
    "                print(\"{0}: known {1} / unkown {2}\".format(name, count_numeric[name], count_numeric_unknown[name]))\n",
    "            print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#headers顺序按文件顺序， 是第几个就放第几个\n",
    "#class也按顺序,第一列就摆第一个\n",
    "#0=nominal, 1=ordinal, 2=numeric\n",
    "\n",
    "headers = (\n",
    "    (\"age\", 2),\n",
    "    (\"workclass\", 0),\n",
    "    (\"fnlwgt\", 2),\n",
    "    (\"education\", 1),\n",
    "    (\"education-num\", 1),\n",
    "    (\"marital-status\", 0),\n",
    "    (\"occupation\", 0),\n",
    "    (\"relationship\", 0),\n",
    "    (\"race\", 0),\n",
    "    (\"sex\", 0),\n",
    "    (\"capital-gain\", 2),\n",
    "    (\"capital-loss\", 2),\n",
    "    (\"hours-per-week\", 2),\n",
    "    (\"native-country\", 0),\n",
    "    (\"class\", (\"<=50K\" , \">50K\"))\n",
    ")\n",
    "\n",
    "validate(\"datasets/adult.data\", headers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class ClassCounter:\n",
    "    def __init__(self, subject):\n",
    "        self.subject = subject\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        self.__classes_count = [0] * class_num\n",
    "        self.__classes_prob = [0] * class_num\n",
    "    \n",
    "    def add(self, class_index):\n",
    "        self.__classes_count[class_index] += 1\n",
    "    \n",
    "    def run(self):\n",
    "        total = sum(self.__classes_count)\n",
    "        for index, val in enumerate(self.__classes_count):\n",
    "            self.__classes_prob[index] = val/total\n",
    "    \n",
    "    def get(self):\n",
    "        return tuple(self.__classes_prob)\n",
    "    \n",
    "class AttrCounter:\n",
    "    def __init__(self, name):\n",
    "        self.name = name\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        return\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        return \n",
    "    \n",
    "    def run(self):\n",
    "        return \n",
    "    \n",
    "    def get(self, attr):\n",
    "        return 0\n",
    "    \n",
    "class NomCounter(AttrCounter):\n",
    "    __smooth_alpha = 1\n",
    "    def __init__(self, name):\n",
    "        super().__init__(name)\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        self.__attr_vals = set()\n",
    "        self.__attr_count = [defaultdict(int) for i in range(class_num)]\n",
    "        self.__attr_prob = [defaultdict(float) for i in range(class_num)]      \n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self.__attr_vals.add(attr)\n",
    "        self.__attr_count[class_index][attr] += 1\n",
    "    \n",
    "    def run(self):\n",
    "        dimension = len(self.__attr_vals)\n",
    "        for index, attrs in enumerate(self.__attr_count):\n",
    "            total = sum(attrs.values())\n",
    "            for val in self.__attr_vals:\n",
    "                self.__attr_prob[index][val] = self._laplace_smooth(attrs[val], total, dimension)\n",
    "#         print(self.__attr_vals)\n",
    "#         print(self.__attr_count)\n",
    "#         print(self.__attr_prob)\n",
    "#         print(\"____________\")\n",
    "    \n",
    "    def get(self, attr): \n",
    "        return tuple(prob[attr] for prob in self.__attr_prob)\n",
    "\n",
    "    def _laplace_smooth(self, num, total, dimension):\n",
    "        return (num+self.__smooth_alpha)/(total+dimension*self.__smooth_alpha)\n",
    "    \n",
    "class NumCounter(AttrCounter):\n",
    "    def __init__(self, name):\n",
    "        super().__init__(name)\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        self.__attr_data = [[] for i in range(class_num)]\n",
    "        self.__mean = [0] * class_num\n",
    "        self.__deviation = [0] * class_num\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self.__attr_data[class_index].append(float(attr))\n",
    "    \n",
    "    def run(self):\n",
    "        for index, data in enumerate(self.__attr_data):\n",
    "            mean = sum(data)/len(data)\n",
    "            deviation = math.sqrt(sum([(x - mean)**2 for x in data])/(len(data)-1))\n",
    "            self.__mean[index] = mean\n",
    "            self.__deviation[index] = deviation\n",
    "#         print(self.name, self.__mean, self.__deviation)\n",
    "    \n",
    "    def get(self, attr):\n",
    "        return tuple(self._normal_distribution(float(attr), self.__mean[i], self.__deviation[i]) for i in range(len(self.__mean)))\n",
    "    \n",
    "    def _normal_distribution(self, val, mean, deviation):\n",
    "        return 1/(deviation*math.sqrt(2*math.pi))*np.exp(-((val-mean)/deviation)**2/2) if deviation > 0 else 0\n",
    "    \n",
    "class Num2NomCounter(NomCounter):  \n",
    "    def __init__(self, name, bins):\n",
    "        assert(bins >= 2)\n",
    "        super().__init__(name)\n",
    "        self.__bins = bins\n",
    "\n",
    "    def prepare(self, class_num):\n",
    "        self.__split = [0] * (self.__bins-1)\n",
    "        self.__attr_data = []\n",
    "        self.__attr_count = [[0]*self.__bins for i in range(class_num)]\n",
    "        self.__attr_prob = [[0]*self.__bins for i in range(class_num)]\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self.__attr_data.append((float(attr), class_index))\n",
    "    \n",
    "    def run(self):\n",
    "        self.__attr_data.sort()\n",
    "        gaps = self.__split_integer(len(self.__attr_data),self.__bins)\n",
    "        cur = 0\n",
    "        for i in range(self.__bins-1):\n",
    "            npos = sum(gaps[:i+1])\n",
    "            self.__split[i] = self.__attr_data[npos][0]\n",
    "            for j in range(cur, npos):\n",
    "                self.__attr_count[self.__attr_data[j][1]][i] += 1\n",
    "            cur = npos\n",
    "        for j in range(cur, len(self.__attr_data)):\n",
    "            self.__attr_count[self.__attr_data[j][1]][i+1] += 1           \n",
    "        for index,attrs in enumerate(self.__attr_count):\n",
    "            total = sum(attrs)\n",
    "            for val, num in enumerate(attrs):\n",
    "                self.__attr_prob[index][val] = self._laplace_smooth(num, total, self.__bins)\n",
    "        # print(self.__split)\n",
    "        # print(self.__attr_count)\n",
    "        # print(self.__attr_prob)\n",
    "        # print(\"____________\")\n",
    "    \n",
    "    def get(self, attr):\n",
    "        attr = float(attr)\n",
    "        for i in range(self.__bins):\n",
    "            if i >= len(self.__split) or attr < self.__split[i]:\n",
    "                break\n",
    "        return tuple(prob[i] for prob in self.__attr_prob)\n",
    "    \n",
    "    def __split_integer(self, m, n):\n",
    "        quotient = int(m / n)\n",
    "        remainder = m % n\n",
    "        if remainder > 0:\n",
    "            return [quotient] * (n - remainder) + [quotient + 1] * remainder\n",
    "        if remainder < 0:\n",
    "            return [quotient - 1] * -remainder + [quotient] * (n + remainder)\n",
    "        return [quotient] * n\n",
    "    \n",
    "class Nom2NumCounter(NumCounter):\n",
    "    def __init__(self, name, mapper):\n",
    "        super().__init__(name)\n",
    "        self.__mapper = mapper\n",
    "    \n",
    "    def add(self, attr, class_index):\n",
    "        self._attr_data[class_index].append(self.__mapper(attr))\n",
    "\n",
    "class Model:\n",
    "    def __init__(self, classes, *counters):\n",
    "        self.__classes = {classname:index for index,classname in enumerate(classes)}\n",
    "        self.__counters = counters\n",
    "        self.__filters = ()\n",
    "        self.__dump_firt_column = False\n",
    "        for index, counter in enumerate(counters):\n",
    "            if isinstance(counter, ClassCounter):\n",
    "                self.__classpos = index\n",
    "            counter.prepare(len(classes))\n",
    "            \n",
    "    def set_filters(self, filters):\n",
    "        self.__filters = filters\n",
    "        \n",
    "    def set_dump_firt_column(self, enabled):\n",
    "        self.__dump_firt_column = enabled\n",
    "            \n",
    "    def preprocess(self, lines):\n",
    "        for line in lines:\n",
    "            inputs = line.strip().split(\",\")[1:] if self.__dump_firt_column else line.strip().split(\",\")\n",
    "            if not inputs or not inputs[0]:\n",
    "                continue\n",
    "            class_index = self.__classes[inputs[self.__classpos]]\n",
    "            for i, val in enumerate(inputs):\n",
    "                if val not in self.__filters:\n",
    "                    if i == self.__classpos:\n",
    "                        self.__counters[i].add(class_index)\n",
    "                    else:\n",
    "                        self.__counters[i].add(val, class_index)\n",
    "                \n",
    "    def train(self):\n",
    "        for counter in self.__counters:\n",
    "            counter.run()\n",
    "\n",
    "    def test(self, lines):\n",
    "        res = []\n",
    "        for line in lines:\n",
    "            inputs = line.strip().split(\",\")[1:] if self.__dump_firt_column else line.strip().split(\",\")\n",
    "            if not inputs or not inputs[0]:\n",
    "                continue\n",
    "            real = self.__classes[inputs[self.__classpos]]\n",
    "            predict = []\n",
    "            for i, val in enumerate(inputs):\n",
    "                if val not in self.__filters:\n",
    "                    if i == self.__classpos:\n",
    "                        predict.append(np.log2(np.array(self.__counters[i].get())))\n",
    "                    else:\n",
    "                        predict.append(np.log2(np.array(self.__counters[i].get(val))))\n",
    "                        \n",
    "                            \n",
    "            res.append((real, np.argmax(reduce(lambda x, y: x+y, predict))))\n",
    "        return res\n",
    "\n",
    "def evaluate(result, classes):\n",
    "    grid = np.zeros((len(classes), len(classes)),dtype=int)\n",
    "    for instance in result:\n",
    "        grid[instance[0]][instance[1]] += 1\n",
    "    print(grid)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "age [13.744745762711865, 12.278732394366198, 13.153749999999997] [0.4621253596612943, 0.5379642302964676, 0.5302413139918748]\n",
      "Malic acid [2.0106779661016954, 1.932676056338028, 3.3337500000000007] [0.6885488598764588, 1.0155687467009085, 1.0879057081324806]\n",
      "Ash [2.455593220338984, 2.244788732394365, 2.4370833333333333] [0.2271659820517139, 0.31546726864369473, 0.1846901756805375]\n",
      "Alcalinity of ash [17.037288135593222, 20.238028169014086, 21.416666666666668] [2.5463224533110727, 3.3497704056582367, 2.2581609287519444]\n",
      "Magnesium [106.33898305084746, 94.54929577464789, 99.3125] [10.498949322623579, 16.753497487641514, 10.890472607606213]\n",
      "Total phenols [2.8401694915254234, 2.2588732394366198, 1.6787500000000002] [0.3389613523154669, 0.5453610843043734, 0.3569708552380176]\n",
      "Flavanoids [2.982372881355932, 2.080845070422536, 0.7814583333333331] [0.3974936086367577, 0.7057007590815028, 0.293504065601863]\n",
      "Nonflavanoid phenols [0.29, 0.363661971830986, 0.44749999999999995] [0.07004924376268554, 0.12396127778601694, 0.12413959198367105]\n",
      "Proanthocyanins [1.8993220338983055, 1.6302816901408452, 1.1535416666666667] [0.4121092272635884, 0.6020677983440211, 0.408835864601428]\n",
      "Color intensity [5.528305084745763, 3.08661971830986, 7.396249979166668] [1.2385728056814824, 0.9249292539153958, 2.3109421491810918]\n",
      "Hue [1.0620338983050848, 1.0562816901408452, 0.6827083333333334] [0.11648264131468111, 0.20293680811654444, 0.11444109482348182]\n",
      "OD280/OD315 of diluted wines [3.1577966101694916, 2.785352112676055, 1.6835416666666658] [0.3570765773013912, 0.4965734904204311, 0.2721114413706684]\n",
      "Proline [1115.7118644067796, 519.5070422535211, 629.8958333333334] [221.52076658684322, 157.21122035923852, 115.09704315911695]\n",
      "[[58  1  0]\n",
      " [ 0 70  1]\n",
      " [ 0  0 48]]\n"
     ]
    }
   ],
   "source": [
    "# filename = \"datasets/adult.data\"\n",
    "# classes = (\"<=50K\" , \">50K\")\n",
    "# model = Model(\n",
    "#     classes,\n",
    "#     NumCounter(\"age\"),\n",
    "#     NomCounter(\"workclass\"),\n",
    "#     NumCounter(\"fnlwgt\"),\n",
    "#     NomCounter(\"education\"),\n",
    "#     NomCounter(\"education-num\"),\n",
    "#     NomCounter(\"marital-status\"),\n",
    "#     NomCounter(\"occupation\"),\n",
    "#     NomCounter(\"relationship\"),\n",
    "#     NomCounter(\"race\"),\n",
    "#     NomCounter(\"sex\"),\n",
    "#     NumCounter(\"capital-gain\"),\n",
    "#     NumCounter(\"capital-loss\"),\n",
    "#     NumCounter(\"hours-per-week\"),\n",
    "#     NomCounter(\"native-country\"),\n",
    "#     ClassCounter(\"salary\")\n",
    "# )\n",
    "# model.set_filters((\"?\"))\n",
    "\n",
    "# filename = \"datasets/car.data\"\n",
    "# classes = (\"unacc\" , \"acc\", \"good\", \"vgood\")\n",
    "# model = Model(\n",
    "#     classes,\n",
    "#     NomCounter(\"buying\"),\n",
    "#     NomCounter(\"maint\"),\n",
    "#     NomCounter(\"doors\"),\n",
    "#     NomCounter(\"persons\"),\n",
    "#     NomCounter(\"lug_boot\"),\n",
    "#     NomCounter(\"safety\"),\n",
    "#     ClassCounter(\"Car Evaluation\")\n",
    "# )\n",
    "\n",
    "filename = \"datasets/wine.data\"\n",
    "classes = (\"1\" , \"2\", \"3\")\n",
    "model = Model(\n",
    "    classes,\n",
    "    ClassCounter(\"Alcohol\"),\n",
    "    NumCounter(\"age\"),\n",
    "    NumCounter(\"Malic acid\"),\n",
    "    NumCounter(\"Ash\"),\n",
    "    NumCounter(\"Alcalinity of ash\"),\n",
    "    NumCounter(\"Magnesium\"),\n",
    "    NumCounter(\"Total phenols\"),\n",
    "    NumCounter(\"Flavanoids\"),\n",
    "    NumCounter(\"Nonflavanoid phenols\"),\n",
    "    NumCounter(\"Proanthocyanins\"),\n",
    "    NumCounter(\"Color intensity\"),\n",
    "    NumCounter(\"Hue\"),\n",
    "    NumCounter(\"OD280/OD315 of diluted wines\"),\n",
    "    NumCounter(\"Proline\")\n",
    ")\n",
    "\n",
    "with open(filename,\"r\") as file:\n",
    "    lines = file.readlines()\n",
    "    model.preprocess(lines)\n",
    "    model.train()\n",
    "    result = model.test(lines)\n",
    "    evaluate(result, classes)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}