Collab 14 - Entrenamiento Distribuido.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<a href=\"https://colab.research.google.com/github/joanby/tensorflow2/blob/master/Collab%2014%20-%20Entrenamiento%20Distribuido.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7eeB9h7f-5Pk"
   },
   "source": [
    "## Paso 1: Instalación de dependencias y configuración del entorno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 496
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 45662,
     "status": "ok",
     "timestamp": 1565190923137,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "wZML5kPN-k8a",
    "outputId": "0fcb37d1-b3ea-4a45-85ea-06ef5668e5bf"
   },
   "outputs": [],
   "source": [
    "#!pip install tensorflow-gpu==2.3.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "lSKxj2vSED3K"
   },
   "source": [
    "## Paso 2: Importar las dependencias del proyecto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SJgY-K01ELTU"
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import numpy as np\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1040,
     "status": "ok",
     "timestamp": 1565191066373,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "sRUOhsd8EMht",
    "outputId": "0deacce3-68b5-4c65-dd01-e7875ca7eb09"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.3.0'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "26ptHcr0EI0a"
   },
   "source": [
    "## Paso 3: Pre procesado del dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hczKVjTGEjAH"
   },
   "source": [
    "### Cargar el dataset del MNIST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1745,
     "status": "ok",
     "timestamp": 1565191155374,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "G1Ha87twDSHS",
    "outputId": "0af79b94-2805-44eb-b6a2-12887e630788"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 0s 0us/step\n"
     ]
    }
   ],
   "source": [
    "(X_train, y_train), (X_test, y_test) = tf.keras.datasets.mnist.load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7Rx1xqx0Ek2Y"
   },
   "source": [
    "### Normalización de las imágenes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "iiCH6RVDEmdS"
   },
   "outputs": [],
   "source": [
    "X_train = X_train / 255.\n",
    "X_test = X_test / 255."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1714,
     "status": "ok",
     "timestamp": 1565191250087,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "dKO-LUwRF_VN",
    "outputId": "40c577fa-d389-4131-81cf-8f0bc230696d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cFay5qJlEm9j"
   },
   "source": [
    "### Redimensionar el dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jKSk6QhDEpTR"
   },
   "outputs": [],
   "source": [
    "X_train = X_train.reshape(-1, 28*28)\n",
    "X_test = X_test.reshape(-1, 28*28)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2491,
     "status": "ok",
     "timestamp": 1565191341125,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "tY5ZlIXbGT6z",
    "outputId": "31578489-842b-4f3a-82c4-ba7c153c59ee"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 784)"
      ]
     },
     "execution_count": 8,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4C4z3xaBHCkj"
   },
   "source": [
    "## Paso 4: Entrenamiento distribuido"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Go7pPmPNHF4i"
   },
   "source": [
    "### Definir un modelo normal (no distribuido)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y0-yHacPGVbC"
   },
   "outputs": [],
   "source": [
    "model_normal = tf.keras.models.Sequential()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "cZXEmKu2Hnwa"
   },
   "outputs": [],
   "source": [
    "model_normal.add(tf.keras.layers.Dense(units=128, activation='relu', input_shape=(784,)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "D8x-c_2WH2xi"
   },
   "outputs": [],
   "source": [
    "model_normal.add(tf.keras.layers.Dropout(rate=0.2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "VyRrtur0H_F7"
   },
   "outputs": [],
   "source": [
    "model_normal.add(tf.keras.layers.Dense(units=10, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4bw4pGh7IID6"
   },
   "outputs": [],
   "source": [
    "model_normal.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "QxNAPpg_Ixd1"
   },
   "source": [
    "### Definir una estrategia distribuida"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "7eqaEeJ8IY5c"
   },
   "outputs": [],
   "source": [
    "distribute = tf.distribute.MirroredStrategy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Yq18oQJyLvR8"
   },
   "source": [
    "### Definir un modelo distribuido"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G1PqCVZHJWDS"
   },
   "outputs": [],
   "source": [
    "with distribute.scope():\n",
    "  model_distributed = tf.keras.models.Sequential()\n",
    "  model_distributed.add(tf.keras.layers.Dense(units=128, activation='relu', input_shape=(784,)))\n",
    "  model_distributed.add(tf.keras.layers.Dropout(rate=0.2))\n",
    "  model_distributed.add(tf.keras.layers.Dense(units=10, activation='softmax'))\n",
    "  model_distributed.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['sparse_categorical_accuracy'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yq-5J0hmNJPh"
   },
   "source": [
    "### Comparar la velocidad de entrenamiento normal vs distribuida"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 153541,
     "status": "ok",
     "timestamp": 1565192598404,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "0DYLCQftM99q",
    "outputId": "6712a2cc-5081-4d04-a27b-490130711f43"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "2400/2400 [==============================] - 16s 6ms/step - loss: 0.2813 - sparse_categorical_accuracy: 0.9178\n",
      "Epoch 2/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.1362 - sparse_categorical_accuracy: 0.9597\n",
      "Epoch 3/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.1035 - sparse_categorical_accuracy: 0.9683\n",
      "Epoch 4/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0852 - sparse_categorical_accuracy: 0.9739\n",
      "Epoch 5/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0730 - sparse_categorical_accuracy: 0.9769\n",
      "Epoch 6/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0651 - sparse_categorical_accuracy: 0.9790\n",
      "Epoch 7/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0576 - sparse_categorical_accuracy: 0.9814\n",
      "Epoch 8/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0518 - sparse_categorical_accuracy: 0.9829\n",
      "Epoch 9/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0469 - sparse_categorical_accuracy: 0.9845\n",
      "Epoch 10/10\n",
      "2400/2400 [==============================] - 15s 6ms/step - loss: 0.0443 - sparse_categorical_accuracy: 0.9855\n",
      "El entrenamiento distribuido ha tardado: 152.52110266685486\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "model_distributed.fit(X_train, y_train, epochs=10, batch_size=25)\n",
    "print(\"El entrenamiento distribuido ha tardado: {}\".format(time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 374
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 297781,
     "status": "ok",
     "timestamp": 1565192745275,
     "user": {
      "displayName": "Juan Gabriel Gomila Salas",
      "photoUrl": "https://lh4.googleusercontent.com/-RJ38mzk8Fog/AAAAAAAAAAI/AAAAAAAAA_g/VV8PcNMAUrk/s64/photo.jpg",
      "userId": "03106941341701838274"
     },
     "user_tz": -120
    },
    "id": "Ub9tQLLTNYTt",
    "outputId": "e8982ff6-e9eb-4d2f-8f68-6108c4774a38"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 12s 202us/sample - loss: 0.2892 - sparse_categorical_accuracy: 0.9165\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 12s 197us/sample - loss: 0.1401 - sparse_categorical_accuracy: 0.9575\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 12s 197us/sample - loss: 0.1075 - sparse_categorical_accuracy: 0.9674\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 12s 196us/sample - loss: 0.0873 - sparse_categorical_accuracy: 0.9728\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 12s 196us/sample - loss: 0.0734 - sparse_categorical_accuracy: 0.9764\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 12s 197us/sample - loss: 0.0653 - sparse_categorical_accuracy: 0.9791\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 12s 195us/sample - loss: 0.0579 - sparse_categorical_accuracy: 0.9816\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 12s 195us/sample - loss: 0.0521 - sparse_categorical_accuracy: 0.9828\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 12s 195us/sample - loss: 0.0480 - sparse_categorical_accuracy: 0.9844\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 12s 195us/sample - loss: 0.0444 - sparse_categorical_accuracy: 0.9851\n",
      "El entrenamiento normal ha tardado: 145.67900371551514\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "model_normal.fit(X_train, y_train, epochs=10, batch_size=25)\n",
    "print(\"El entrenamiento normal ha tardado: {}\".format(time.time() - start_time))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1CxbQ8JMOF4q"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "Collab 14 - Entrenamiento Distribuido.ipynb",
   "provenance": [
    {
     "file_id": "1RABY1ldyCwtHgXIYx8xzyLd0kI3IfKRk",
     "timestamp": 1565177507295
    }
   ],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}