3_4_Activation_function_and_Weight_initializer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/skj092/experiments/blob/main/Activation_function_and_Weight_initializer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RK3JOifWzsDi"
   },
   "source": [
    "# Objective : \n",
    "\n",
    "**To Study different activation functions and weight initializers**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "emc1KI4y0X75",
    "outputId": "0aa49176-0911-4cf1-bf54-22ef24ba1838"
   },
   "outputs": [],
   "source": [
    "# !pip install mlflow -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "HZhYnGSP0AoG",
    "outputId": "b55dbe5f-205f-42a5-83fa-d77adbea8a08"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\_distributor_init.py:30: UserWarning: loaded more than 1 DLL from .libs:\n",
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\.libs\\libopenblas.GK7GX5KEQ4F6UYO3P26ULGBQYHGQO7J4.gfortran-win_amd64.dll\n",
      "c:\\users\\sonu.ramkumar.jha\\desktop\\experiments\\env\\lib\\site-packages\\numpy\\.libs\\libopenblas.WCDJNK7YVMPZQ2ME2ZZHJJRJ3JIKNDB7.gfortran-win_amd64.dll\n",
      "  warnings.warn(\"loaded more than 1 DLL from .libs:\"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.5.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2021/06/14 14:33:16 WARNING mlflow.utils.autologging_utils: You are using an unsupported version of tensorflow. If you encounter errors during autologging, try upgrading / downgrading tensorflow to a supported version, or try upgrading MLflow.\n"
     ]
    }
   ],
   "source": [
    "# TensorFlow and tf.keras\n",
    "import tensorflow as tf\n",
    "\n",
    "# Helper libraries\n",
    "import numpy as np\n",
    "from tensorflow.keras import initializers\n",
    "from tensorflow.python.keras import activations\n",
    "\n",
    "print(tf.__version__)\n",
    "\n",
    "# downloading fashion_mnist data\n",
    "fashion_mnist = tf.keras.datasets.fashion_mnist\n",
    "\n",
    "(train_images, train_labels), (test_images, test_labels) = fashion_mnist.load_data()\n",
    "\n",
    "class_names = ['T-shirt/top', 'Trouser', 'Pullover', 'Dress', 'Coat',\n",
    "               'Sandal', 'Shirt', 'Sneaker', 'Bag', 'Ankle boot']\n",
    "\n",
    "train_images = train_images / 255.0\n",
    "\n",
    "test_images = test_images / 255.0              \n",
    "\n",
    "\n",
    "import mlflow.tensorflow\n",
    "mlflow.tensorflow.autolog()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d7m7ONXi3uoA"
   },
   "source": [
    "# With Default Weight Initializer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2rkDVMV41rdj"
   },
   "source": [
    "## 1. relu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "6slPJNcx0s-V",
    "outputId": "64068951-6161-45b4-e655-226d94bb242d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 13:52 - loss: 2.2553 - accuracy: 0.1458WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0026s vs `on_train_batch_end` time: 0.1468s). Check your callbacks.\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.4906 - accuracy: 0.8268\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3733 - accuracy: 0.8659\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3345 - accuracy: 0.8781\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3091 - accuracy: 0.8856\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2942 - accuracy: 0.8923\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2774 - accuracy: 0.8985\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2664 - accuracy: 0.9021\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2545 - accuracy: 0.9067\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2458 - accuracy: 0.9095\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2376 - accuracy: 0.9102\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmp_97355bl\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3374 - accuracy: 0.8844\n",
      "test_loss 0.33737125992774963\n",
      "test_accuracy 0.8844000101089478\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.relu\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "\n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tSAQyFF-15G3"
   },
   "source": [
    "## 2. Softmax"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7IjieM_X0w0R",
    "outputId": "acc490d1-d22d-4b57-c5a3-5c2e3fa55a2d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_1 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 11:57 - loss: 2.2983 - accuracy: 0.0833WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0016s vs `on_train_batch_end` time: 0.1273s). Check your callbacks.\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 1.3811 - accuracy: 0.6191\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.8298 - accuracy: 0.6755\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.7209 - accuracy: 0.6961\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.6663 - accuracy: 0.7296\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.6276 - accuracy: 0.7440\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.6063 - accuracy: 0.7484\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5805 - accuracy: 0.7735\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5377 - accuracy: 0.8164\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 5s 3ms/step - loss: 0.5001 - accuracy: 0.8327\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4772 - accuracy: 0.8399\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmp4fdechfl\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.5113 - accuracy: 0.8312\n",
      "test_loss 0.511297881603241\n",
      "test_accuracy 0.8312000036239624\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.softmax\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('initializer', model.layers[1].kernel_initializer)\n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2YZlBOw22Bd7"
   },
   "source": [
    "## 3. Sigmoid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Rp_IfneF2HBK",
    "outputId": "ac0a27ab-458c-4250-c4e9-0c93b6103de4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_2 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 12:02 - loss: 2.3812 - accuracy: 0.2083WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0024s vs `on_train_batch_end` time: 0.1278s). Check your callbacks.\n",
      "1875/1875 [==============================] - 5s 2ms/step - loss: 0.5405 - accuracy: 0.8150\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3920 - accuracy: 0.8594\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3544 - accuracy: 0.8731\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3295 - accuracy: 0.8811\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3103 - accuracy: 0.8872\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2956 - accuracy: 0.8915\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2816 - accuracy: 0.8972\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2699 - accuracy: 0.8997\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.2586 - accuracy: 0.9049\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2513 - accuracy: 0.9067\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmpzgdah6pi\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3318 - accuracy: 0.8810\n",
      "test_loss 0.33182984590530396\n",
      "test_accuracy 0.8809999823570251\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.sigmoid\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('initializer', model.layers[1].kernel_initializer)\n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "\n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HOXH0hMj2Nhy"
   },
   "source": [
    "## 4. tanh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "liCWUr-A2VQp",
    "outputId": "48bc84c3-6f9d-4747-f24e-2d6ff873efe4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_3\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_3 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 6:05 - loss: 2.1505 - accuracy: 0.2708WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0018s vs `on_train_batch_end` time: 0.0645s). Check your callbacks.\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.4769 - accuracy: 0.8298\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3697 - accuracy: 0.8636\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3352 - accuracy: 0.8766\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3118 - accuracy: 0.8856\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2957 - accuracy: 0.8904\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2820 - accuracy: 0.8951\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2707 - accuracy: 0.8992\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2580 - accuracy: 0.9035\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.2522 - accuracy: 0.9067\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2438 - accuracy: 0.9087\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmpzxwj0c23\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3351 - accuracy: 0.8817\n",
      "test_loss 0.3351154327392578\n",
      "test_accuracy 0.8816999793052673\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('initializer', model.layers[1].kernel_initializer)\n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "\n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "6sAswU_92YbZ"
   },
   "source": [
    "## 5. selu"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "pTr0GwDa2a7B",
    "outputId": "f3f54393-7453-4cd3-ed05-075940445d6d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_4\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_4 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 8:54 - loss: 2.3942 - accuracy: 0.1562WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0020s vs `on_train_batch_end` time: 0.0949s). Check your callbacks.\n",
      "1875/1875 [==============================] - 5s 2ms/step - loss: 0.4923 - accuracy: 0.8239\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 5s 2ms/step - loss: 0.3857 - accuracy: 0.8605\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3516 - accuracy: 0.8716\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3244 - accuracy: 0.8805\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3056 - accuracy: 0.8871\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2914 - accuracy: 0.8913\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2743 - accuracy: 0.8983\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.2648 - accuracy: 0.9011\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.2540 - accuracy: 0.9056\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2459 - accuracy: 0.9074\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmpzazq355f\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3284 - accuracy: 0.8836\n",
      "test_loss 0.3283739984035492\n",
      "test_accuracy 0.8835999965667725\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.selu\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation),\n",
    "tf.keras.layers.Dense(10)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('initializer', model.layers[1].kernel_initializer)\n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "EAmHslRC2n7h"
   },
   "source": [
    "# With Weight Initializer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pOvN2p-x2sa9"
   },
   "source": [
    "## 1. relu + he_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hC_nZhfo205b",
    "outputId": "9b9d6069-1613-41e6-96db-7bebc11752f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_5\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_5 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 8:39 - loss: 2.2752 - accuracy: 0.1979WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0024s vs `on_train_batch_end` time: 0.0922s). Check your callbacks.\n",
      "1875/1875 [==============================] - 5s 2ms/step - loss: 0.4947 - accuracy: 0.8256\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3712 - accuracy: 0.8656\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3310 - accuracy: 0.8791\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3073 - accuracy: 0.8869\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2890 - accuracy: 0.8934\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2763 - accuracy: 0.8983\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2650 - accuracy: 0.9014\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2525 - accuracy: 0.9059\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2432 - accuracy: 0.9086\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2362 - accuracy: 0.9116\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmpxz2bdo7s\\model\\data\\model\\assets\n",
      "313/313 - 1s - loss: 0.3388 - accuracy: 0.8834\n",
      "test_loss 0.3388063609600067\n",
      "test_accuracy 0.883400022983551\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.relu\n",
    "initializer = tf.keras.initializers.he_normal\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation, kernel_initializer=initializer),\n",
    "tf.keras.layers.Dense(10,kernel_initializer=initializer)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "\n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PokAZdFS27Hi"
   },
   "source": [
    "## 2. softmax + glorot_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "_OxHguIq3EEv",
    "outputId": "c8721295-5acb-41ce-8e42-9decca81cc84"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_6\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_6 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_13 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 22:31 - loss: 2.2999 - accuracy: 0.1458WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0028s vs `on_train_batch_end` time: 0.2407s). Check your callbacks.\n",
      "1875/1875 [==============================] - 5s 3ms/step - loss: 1.3332 - accuracy: 0.6831\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.7282 - accuracy: 0.7452\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.6123 - accuracy: 0.7541\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5751 - accuracy: 0.7590\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5573 - accuracy: 0.7642\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5467 - accuracy: 0.7682\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5384 - accuracy: 0.7737\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.5292 - accuracy: 0.7833\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.5191 - accuracy: 0.8019\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5068 - accuracy: 0.8156\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmp0_21czwo\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.5652 - accuracy: 0.8086\n",
      "test_loss 0.5651732683181763\n",
      "test_accuracy 0.8086000084877014\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.softmax\n",
    "initializer = tf.keras.initializers.glorot_normal\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation, kernel_initializer=initializer),\n",
    "tf.keras.layers.Dense(10,kernel_initializer=initializer)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "TJ6vDPgk3LBu"
   },
   "source": [
    "## 3. sigmoid + glorot_mormal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fpVAA_HA3SBU",
    "outputId": "332345f0-bec4-4b06-a284-3a32aca8aa30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_7\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_7 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_14 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 8:41 - loss: 2.3433 - accuracy: 0.1250WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0018s vs `on_train_batch_end` time: 0.0924s). Check your callbacks.\n",
      "1875/1875 [==============================] - 4s 2ms/step - loss: 0.5434 - accuracy: 0.8130\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3899 - accuracy: 0.8587\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3529 - accuracy: 0.8721\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3288 - accuracy: 0.8802\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3113 - accuracy: 0.8865\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2942 - accuracy: 0.8932\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2822 - accuracy: 0.8976\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2705 - accuracy: 0.9007\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - ETA: 0s - loss: 0.2615 - accuracy: 0.90 - 2s 1ms/step - loss: 0.2611 - accuracy: 0.9042\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2518 - accuracy: 0.9074\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmpsswux0ma\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3363 - accuracy: 0.8765\n",
      "test_loss 0.3362782597541809\n",
      "test_accuracy 0.8765000104904175\n"
     ]
    }
   ],
   "source": [
    "initializer = tf.keras.initializers.glorot_normal\n",
    "activation = tf.keras.activations.sigmoid\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation, kernel_initializer=initializer),\n",
    "tf.keras.layers.Dense(10,kernel_initializer=initializer)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "AQAk535u3VkC"
   },
   "source": [
    "## 4. tanh + glorot_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7gKVi1YH3cKK",
    "outputId": "91c0db1b-1162-41a6-c40b-210ab971e2fe"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_8 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_17 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 6:11 - loss: 2.2522 - accuracy: 0.1979WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0016s vs `on_train_batch_end` time: 0.0658s). Check your callbacks.\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4740 - accuracy: 0.8293\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.3700 - accuracy: 0.8655\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3343 - accuracy: 0.8774\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.3118 - accuracy: 0.8845\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2953 - accuracy: 0.8906\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2798 - accuracy: 0.8968\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2674 - accuracy: 0.9000\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.2588 - accuracy: 0.9032\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2497 - accuracy: 0.9067: \n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2408 - accuracy: 0.9100\n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmputt80oli\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3369 - accuracy: 0.8767\n",
      "test_loss 0.33686619997024536\n",
      "test_accuracy 0.8766999840736389\n"
     ]
    }
   ],
   "source": [
    "initializer = tf.keras.initializers.glorot_normal\n",
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation, kernel_initializer=initializer),\n",
    "tf.keras.layers.Dense(10,kernel_initializer=initializer)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "HPdn9AFa3g7c"
   },
   "source": [
    "## 5. selu + lucan_normal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "cD1OySlD3k3_",
    "outputId": "5c8c10b3-a3a1-4da4-88d1-76d61449f2b1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_9\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten_9 (Flatten)          (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_18 (Dense)             (None, 128)               100480    \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 101,770\n",
      "Trainable params: 101,770\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Epoch 1/10\n",
      "   3/1875 [..............................] - ETA: 8:32 - loss: 2.2926 - accuracy: 0.1250WARNING:tensorflow:Callback method `on_train_batch_end` is slow compared to the batch time (batch time: 0.0018s vs `on_train_batch_end` time: 0.0906s). Check your callbacks.\n",
      "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4778 - accuracy: 0.8292\n",
      "Epoch 2/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3683 - accuracy: 0.8666\n",
      "Epoch 3/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3371 - accuracy: 0.8763\n",
      "Epoch 4/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3134 - accuracy: 0.8837\n",
      "Epoch 5/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2969 - accuracy: 0.8902\n",
      "Epoch 6/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2811 - accuracy: 0.8960: 0s - loss: 0.2811 - accuracy: 0.89\n",
      "Epoch 7/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2713 - accuracy: 0.8989: 0s - loss: 0.2702 - accuracy\n",
      "Epoch 8/10\n",
      "1875/1875 [==============================] - 3s 1ms/step - loss: 0.2627 - accuracy: 0.9016\n",
      "Epoch 9/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2516 - accuracy: 0.9061\n",
      "Epoch 10/10\n",
      "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2438 - accuracy: 0.9086: 0s - loss: 0.2453 - \n",
      "INFO:tensorflow:Assets written to: C:\\Users\\SONURA~1.JHA\\AppData\\Local\\Temp\\tmp6txl7hmh\\model\\data\\model\\assets\n",
      "313/313 - 0s - loss: 0.3431 - accuracy: 0.8759\n",
      "test_loss 0.34306737780570984\n",
      "test_accuracy 0.8758999705314636\n"
     ]
    }
   ],
   "source": [
    "activation = tf.keras.activations.selu\n",
    "initializer = tf.keras.initializers.lecun_normal\n",
    "\n",
    "initializer = tf.keras.initializers.glorot_normal\n",
    "activation = tf.keras.activations.tanh\n",
    "\n",
    "model = tf.keras.Sequential([\n",
    "tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "tf.keras.layers.Dense(128, activation=activation, kernel_initializer=initializer),\n",
    "tf.keras.layers.Dense(10,kernel_initializer=initializer)\n",
    "])\n",
    "\n",
    "model.compile(optimizer='adam',loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),metrics=['accuracy'])\n",
    "\n",
    "# model summary\n",
    "model.summary()\n",
    "\n",
    "with mlflow.start_run():\n",
    "\n",
    "    model.fit(train_images, train_labels, epochs=10)\n",
    "\n",
    "    test_loss, test_acc = model.evaluate(test_images,  test_labels, verbose=2)\n",
    "\n",
    "    print('test_loss', test_loss)\n",
    "    print('test_accuracy', test_acc)\n",
    "\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_loss', test_loss)\n",
    "    mlflow.tensorflow.mlflow.log_metric('test_acc', test_acc)  \n",
    "    mlflow.tensorflow.mlflow.log_param('activation', activation)\n",
    "    mlflow.tensorflow.mlflow.log_param('initializer',model.layers[1].kernel_initializer)\n",
    "    \n",
    "del model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "MCjnkWJC3qOl"
   },
   "source": [
    "# Results : \n",
    "\n",
    "|Act Func|initializer|train_loss|train_accuracy|test_loss|test_accuracy|\n",
    "|----|----|----|----|----|----|\n",
    "|relu|GlorotUniform|0.2378 |0.9109\t|0.3364 |0.8807|\n",
    "|relu|he_normal|0.2423 |0.9089\t|0.3358 |0.8867|\n",
    "|softmax|GlorotUniform|0.4847 |0.8383\t|0.5149 |0.8275|\n",
    "|softmax|glorot_normal|0.4730 |0.8419\t|0.5077 |0.8318|\n",
    "|sigmoid|GlorotUniform|0.2503 |0.9078\t|0.3451 |0.8767|\n",
    "|sigmoid|glorot_normal|0.2517 |0.9063\t|0.3325 |0.8791|\n",
    "|tanh|GlorotUniform|0.2396 |0.9101\t|0.3354 |0.8812|\n",
    "|tanh|glorot_normal|0.2433 |0.9086\t|0.3297 |0.8811|\n",
    "|selu|GlorotUniform|0.2470 |0.9064\t|0.3470 |0.8791|\n",
    "|selu|lucan_normal|0.2416 |0.9112\t|0.3346 |0.8812|"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "a1tY6I0f4Dsl"
   },
   "source": [
    "**Observations**\n",
    "- By Default Dense layer have `Glorot Uniform` weight initializer.\n",
    "- `Softmax` performed `worst` with both the initializers.\n",
    "- Default weight initializer (GlorotUniform) performed worse in every activation function.\n",
    "- Order of model according to test_loss is : `tanh > selu > sigmoid > relu > softmax`.\n",
    "- Order of model according to test_accuracy is : `relu > tanh > selu > sigmoid > softmax`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZMu9Qq8v9DLb"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "authorship_tag": "ABX9TyMuLdgewwTxN4rzfigyGS1N",
   "collapsed_sections": [],
   "include_colab_link": true,
   "name": "Activation_function_and_Weight_initializer.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}