4_layer_digit_recognizer.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "XYh1UU9jAoU_"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "HseArtX4zVWk"
      },
      "outputs": [],
      "source": [
        "data = pd.read_csv('https://raw.githubusercontent.com/prathameshks/ML-From-Scratch/main/MNIST_DATA/train.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vUvwmY73zVWk",
        "outputId": "b53a211b-bca4-4889-c13f-1a0a2419a80d"
      },
      "outputs": [],
      "source": [
        "data = np.array(data)\n",
        "m, n = data.shape\n",
        "np.random.shuffle(data) # shuffle before splitting into test and training sets\n",
        "\n",
        "data_test = data[0:1000].T\n",
        "Y_test = data_test[0]\n",
        "X_test = data_test[1:n]\n",
        "X_test = X_test / 255.\n",
        "\n",
        "data_train = data[1000:m].T\n",
        "Y_train = data_train[0]\n",
        "X_train = data_train[1:n]\n",
        "X_train = X_train / 255.\n",
        "_,m_train = X_train.shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bv8JLIOhzVWn",
        "outputId": "107ed409-aac7-477b-d325-441b37b98475"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "X_train.shape = (784, 41000)\n",
            "X_test.shape = (784, 1000)\n",
            "Y_train.shape = (41000,)\n",
            "Y_test.shape = (1000,)\n"
          ]
        }
      ],
      "source": [
        "print(f\"{X_train.shape = }\")\n",
        "print(f\"{X_test.shape = }\")\n",
        "print(f\"{Y_train.shape = }\")\n",
        "print(f\"{Y_test.shape = }\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "ssN6456HzVWn"
      },
      "outputs": [],
      "source": [
        "class NeuralNetwork:\n",
        "    def __init__(self):\n",
        "        self.w1 = np.random.rand(16, 784) - 0.5\n",
        "        self.b1 = np.random.rand(16, 1) - 0.5\n",
        "        self.w2 = np.random.rand(16, 16) - 0.5\n",
        "        self.b2 = np.random.rand(16, 1) - 0.5 \n",
        "        self.w3 = np.random.rand(10, 16) - 0.5\n",
        "        self.b3 = np.random.rand(10, 1) - 0.5        \n",
        "\n",
        "    def ReLU(self, x):\n",
        "        return np.maximum(0, x)\n",
        "\n",
        "    def softMax(self, x):\n",
        "        # limit overflow\n",
        "        exp_x = np.exp(np.maximum(x, -60))  # Clip values to prevent overflow\n",
        "        sum_exp_x = np.sum(exp_x, axis=0)\n",
        "        return exp_x / sum_exp_x\n",
        "\n",
        "    def one_hot(self,y):\n",
        "        one_hot_y = np.zeros((y.size , y.max() + 1))\n",
        "        one_hot_y[np.arange(y.size) , y] = 1\n",
        "        one_hot_y = one_hot_y.T\n",
        "        return one_hot_y\n",
        "\n",
        "    def deriv_ReLU(self,x):\n",
        "        return x>0\n",
        "    \n",
        "    def forward_prop(self, X):\n",
        "        z1 = self.w1.dot(X) + self.b1\n",
        "        a1 = self.ReLU(z1)\n",
        "        z2 = self.w2.dot(a1) + self.b2\n",
        "        a2 = self.ReLU(z2)\n",
        "        z3 = self.w3.dot(a2) + self.b3\n",
        "        a3 = self.softMax(z3)\n",
        "\n",
        "        return z1,a1,z2,a2,z3,a3\n",
        "\n",
        "    def back_prop(self, z1, a1, z2, a2,z3,a3,x, y):\n",
        "        one_hot_y = self.one_hot(y)\n",
        "        dz3 = a3 - one_hot_y\n",
        "        dw3 = 1/m * dz3.dot(a2.T)\n",
        "        db3 = 1/m * np.sum(dz3)\n",
        "        \n",
        "        dz2 = self.w3.T.dot(dz3) * self.deriv_ReLU(z2)\n",
        "        dw2 = 1/ m * dz2.dot(a1.T)\n",
        "        db2 = 1 / m * np.sum(dz2)\n",
        "\n",
        "        dz1 = self.w2.T.dot(dz2) * self.deriv_ReLU(z1)\n",
        "        dw1 = 1 / m * dz1.dot(x.T)\n",
        "        db1 = 1 / m * np.sum(dz1)\n",
        "\n",
        "        return dw1,db1,dw2,db2,dw3,db3\n",
        "\n",
        "    def update_params(self, dw1,db1,dw2,db2,dw3,db3,alpha):\n",
        "        self.w1 = self.w1 - alpha*dw1\n",
        "        self.b1 = self.b1 - alpha*db1\n",
        "        self.w2 = self.w2 - alpha*dw2\n",
        "        self.b2 = self.b2 - alpha*db2\n",
        "        self.w3 = self.w3 - alpha*dw3 \n",
        "        self.b3 = self.b3 - alpha*db3\n",
        "\n",
        "    def get_predictions(self, a):\n",
        "        return np.argmax(a, 0)\n",
        "\n",
        "    def get_accuracy(self, predictions, y):\n",
        "        # print(predictions, y)\n",
        "        return np.sum(predictions == y) / y.size\n",
        "\n",
        "    def gradient_decent(self,x,y,iterations,alpha):\n",
        "        for i in range(iterations):\n",
        "            z1,a1,z2,a2,z3,a3 = self.forward_prop(x)\n",
        "            dw1,db1,dw2,db2,dw3,db3 = self.back_prop(z1,a1,z2,a2,z3,a3,x,y)\n",
        "            self.update_params(dw1,db1,dw2,db2,dw3,db3,alpha)\n",
        "            if(i % 50 == 0):\n",
        "                print(\"Iteration: \",i)\n",
        "                predictions = self.get_predictions(a3)\n",
        "                print(\"Accuracy: \",self.get_accuracy(predictions,y))\n",
        "    \n",
        "\n",
        "    def make_predictions(self,x):\n",
        "        _,_,_,_,_,a3 = self.forward_prop(x)\n",
        "        predictions = self.get_predictions(a3)\n",
        "        return predictions\n",
        "\n",
        "    def show_training_predictions(self,index,x,y):\n",
        "        cur_img = x[:,index,None]\n",
        "        np.save(\"img.txt\",cur_img)\n",
        "        prediction = self.make_predictions(cur_img)\n",
        "        print(\"Prediction: \", prediction)\n",
        "        print(\"Label: \", y[index])\n",
        "        cur_img = cur_img.reshape((28,28))\n",
        "        plt.imshow(cur_img, cmap='gray')\n",
        "        plt.show()\n",
        "\n",
        "    def save(self,file=\"4_l_digit_model.npz\"):\n",
        "        np.savez(file, w1=self.w1, b1=self.b1, w2=self.w2, b2=self.b2,w3 = self.w3,b3 = self.b3)\n",
        "        \n",
        "    def load(self,file=\"4_l_digit_model.npz\"):\n",
        "        npz = np.load(file)\n",
        "        self.w1 = npz['w1']\n",
        "        self.b1 = npz['b1']\n",
        "        self.w2 = npz['w2']\n",
        "        self.b2 = npz['b2']\n",
        "        self.w3 = npz['w3']\n",
        "        self.b3 = npz['b3']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "GVackV-p1ieR"
      },
      "outputs": [],
      "source": [
        "# get testing accuracy\n",
        "\n",
        "def get_accuracy(predictions, y):\n",
        "    # print(predictions, y)\n",
        "    return np.sum(predictions == y) / y.size\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "model = NeuralNetwork()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.load()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MFG9F7LYzVWo",
        "outputId": "b68a8bba-a9e8-4729-bb8b-374e87e7dd6a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Iteration:  0\n",
            "Accuracy:  0.9233658536585366\n",
            "Iteration:  50\n",
            "Accuracy:  0.9236585365853659\n",
            "Iteration:  100\n",
            "Accuracy:  0.9238780487804878\n"
          ]
        }
      ],
      "source": [
        "# training\n",
        "model.gradient_decent(X_train, Y_train, 101, 0.05)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.save()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vRNl6bdK1udN",
        "outputId": "a1c496b3-4f46-4aa6-e205-169995672d0a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Accuracy:  0.914\n"
          ]
        }
      ],
      "source": [
        "# test accuracy\n",
        "predictions = model.make_predictions(X_test)\n",
        "print(\"Accuracy: \",get_accuracy(predictions,Y_test))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 465
        },
        "id": "kJlvXRXFzVWp",
        "outputId": "032e8149-0b98-4783-ce88-2380c05890dc"
      },
      "outputs": [],
      "source": [
        "model.show_training_predictions(104,X_train,Y_train)"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.11"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}