Python72-深度学习LSTM实现.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "原文：https://blog.csdn.net/q383700092/article/details/77992378"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 LSTM之keras实现"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.random.seed(2017)\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import np_utils\n",
    "from keras.models import Sequential\n",
    "from keras.layers import LSTM,Activation,Dense\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(60000, 28, 28)\n",
      "<class 'numpy.ndarray'>\n"
     ]
    }
   ],
   "source": [
    "(X_train, y_train),(X_test, y_test)= mnist.load_data()\n",
    "print(X_train.shape)\n",
    "print(type(X_train))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 参数\n",
    "learning_rate=0.001\n",
    "epochs=2\n",
    "batch_size=128\n",
    "n_input=28\n",
    "n_step=28\n",
    "n_hidden=128\n",
    "n_classes=10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(60000, 28, 28)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 归一化\n",
    "X_train=X_train.reshape(-1,n_step,n_input)/255\n",
    "X_test=X_test.reshape(-1,n_step,n_input)/255\n",
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_train = np_utils.to_categorical(y_train, num_classes=10)\n",
    "y_test = np_utils.to_categorical(y_test, num_classes=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 构建模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model=Sequential()\n",
    "model.add(LSTM(n_hidden,\n",
    "               batch_input_shape=(None,n_step,n_input),\n",
    "               unroll=True))\n",
    "model.add(Dense(n_classes))\n",
    "model.add(Activation('softmax'))\n",
    "# optimizer\n",
    "adam=Adam(lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "lstm_5 (LSTM)                (None, 128)               80384     \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                1290      \n",
      "_________________________________________________________________\n",
      "activation_3 (Activation)    (None, 10)                0         \n",
      "=================================================================\n",
      "Total params: 81,674\n",
      "Trainable params: 81,674\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/2\n",
      "60000/60000 [==============================] - 22s 365us/step - loss: 0.4249 - acc: 0.8666 - val_loss: 0.1806 - val_acc: 0.9455\n",
      "Epoch 2/2\n",
      "60000/60000 [==============================] - 18s 304us/step - loss: 0.1440 - acc: 0.9563 - val_loss: 0.1257 - val_acc: 0.9614\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1ba02b38>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.summary()\n",
    "model.compile(optimizer=adam,\n",
    "             loss='categorical_crossentropy',\n",
    "             metrics=['accuracy'])\n",
    "model.fit(X_train,y_train,\n",
    "         batch_size=batch_size,\n",
    "         epochs=epochs,\n",
    "         verbose=1,\n",
    "         validation_data=(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 评价"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM test score: 0.125696593118459\n",
      "LSTM test accuracy: 0.9614\n"
     ]
    }
   ],
   "source": [
    "scores=model.evaluate(X_test,y_test,verbose=0)\n",
    "print(\"LSTM test score:\",scores[0]) # loss\n",
    "print(\"LSTM test accuracy:\",scores[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2 TensorFlow实现LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.set_random_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-5-6fbbf3403c7c>:1: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\base.py:252: _internal_retry.<locals>.wrap.<locals>.wrapped_fn (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use urllib or similar directly.\n",
      "Successfully downloaded train-images-idx3-ubyte.gz 9912422 bytes.\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST\\train-images-idx3-ubyte.gz\n",
      "Successfully downloaded train-labels-idx1-ubyte.gz 28881 bytes.\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting MNIST\\train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Successfully downloaded t10k-images-idx3-ubyte.gz 1648877 bytes.\n",
      "Extracting MNIST\\t10k-images-idx3-ubyte.gz\n",
      "Successfully downloaded t10k-labels-idx1-ubyte.gz 4542 bytes.\n",
      "Extracting MNIST\\t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From E:\\ProgramData\\Anaconda3\\lib\\site-packages\\tensorflow\\contrib\\learn\\python\\learn\\datasets\\mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Datasets(train=<tensorflow.contrib.learn.python.learn.datasets.mnist.DataSet object at 0x0000000008618F28>, validation=<tensorflow.contrib.learn.python.learn.datasets.mnist.DataSet object at 0x000000000BCAB390>, test=<tensorflow.contrib.learn.python.learn.datasets.mnist.DataSet object at 0x000000000BC98D30>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mnist=input_data.read_data_sets('MNIST',one_hot=True)\n",
    "mnist"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 参数设置"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr=0.001\n",
    "training_iters=100000\n",
    "batch_size=128\n",
    "\n",
    "n_inputs=28\n",
    "n_steps=28\n",
    "n_hidden_units=128\n",
    "n_classes=10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# input\n",
    "x=tf.placeholder(tf.float32,[None,n_steps,n_inputs]) # None表示个数不定，其实这里为60000\n",
    "y=tf.placeholder(tf.float32,[None,n_classes])\n",
    "\n",
    "# weights\n",
    "weights={\n",
    "    # （28,28）\n",
    "    'in':tf.Variable(tf.random_normal([n_inputs,n_hidden_units])),\n",
    "    'out':tf.Variable(tf.random_normal([n_hidden_units,n_classes]))\n",
    "}\n",
    "\n",
    "biases={\n",
    "    # （128，）\n",
    "    'in':tf.Variable(tf.constant(0.1,shape=[n_hidden_units,])),\n",
    "    # (10,)\n",
    "    'out':tf.Variable(tf.constant(0.1,shape=[n_classes,]))\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def RNN(X,weights,biaes):\n",
    "    # hidden layer for input to cell\n",
    "    ########################################\n",
    "    X=tf.reshape(X,[-1,n_inputs])\n",
    "    \n",
    "    #X_in = (128 batch * 28 steps, 128 hidden)\n",
    "    X_in=tf.matmul(X,weights['in'])+biases['in']\n",
    "    # X_in ==> (128 batch, 28 steps, 128 hidden)\n",
    "    X_in=tf.reshape(X_in,[-1,n_steps,n_hidden_units])\n",
    "    \n",
    "    # cell\n",
    "    ##########################################\n",
    "    \n",
    "    # basic LSTM Cell\n",
    "    cell=tf.contrib.rnn.BasicLSTMCell(n_hidden_units)\n",
    "    # lstm cell is divided into two parts (c_state, h_state)\n",
    "    init_state=cell.zero_state(batch_size,dtype=tf.float32)\n",
    "    \n",
    "    # You have 2 options for following step.\n",
    "    # 1: tf.nn.rnn(cell, inputs);\n",
    "    # 2: tf.nn.dynamic_rnn(cell, inputs).\n",
    "    # If use option 1, you have to modified the shape of X_in, go and check out this:\n",
    "    # https://github.com/aymericdamien/TensorFlow-Examples/blob/master/examples/3_NeuralNetworks/recurrent_network.py\n",
    "    # In here, we go for option 2.\n",
    "    # dynamic_rnn receive Tensor (batch, steps, inputs) or (steps, batch, inputs) as X_in.\n",
    "    # Make sure the time_major is changed accordingly.\n",
    "    outputs, final_state = tf.nn.dynamic_rnn(cell, X_in, initial_state=init_state, time_major=False)\n",
    "    \n",
    "    # hidden layer for output as the final results\n",
    "    #############################################\n",
    "    # results = tf.matmul(final_state[1], weights['out']) + biases['out']\n",
    "\n",
    "    # # or\n",
    "    # unpack to list [(batch, outputs)..] * steps #交换维度\n",
    "    outputs = tf.unstack(tf.transpose(outputs, [1,0,2]))\n",
    "    results = tf.matmul(outputs[-1], weights['out']) + biases['out']    # shape = (128, 10)\n",
    "\n",
    "    return results\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-12-48e1ee901d2f>:2: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "0.2265625\n",
      "0.6484375\n",
      "0.8203125\n",
      "0.875\n",
      "0.8359375\n",
      "0.875\n",
      "0.8515625\n",
      "0.8671875\n",
      "0.875\n",
      "0.921875\n",
      "0.890625\n",
      "0.8984375\n",
      "0.9296875\n",
      "0.9296875\n",
      "0.9140625\n",
      "0.890625\n",
      "0.9375\n",
      "0.9375\n",
      "0.953125\n",
      "0.953125\n",
      "0.9375\n",
      "0.9453125\n",
      "0.9140625\n",
      "0.96875\n",
      "0.984375\n",
      "0.9609375\n",
      "0.9609375\n",
      "0.9375\n",
      "0.953125\n",
      "0.953125\n",
      "0.9453125\n",
      "0.9609375\n",
      "0.9453125\n",
      "0.9453125\n",
      "0.9765625\n",
      "0.9296875\n",
      "0.9453125\n",
      "0.96875\n",
      "0.96875\n",
      "0.96875\n"
     ]
    }
   ],
   "source": [
    "pred = RNN(x, weights, biases)\n",
    "cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y))\n",
    "train_op = tf.train.AdamOptimizer(lr).minimize(cost)\n",
    "\n",
    "correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    init = tf.global_variables_initializer()\n",
    "    sess.run(init)\n",
    "    step = 0\n",
    "    while step * batch_size < training_iters:\n",
    "        batch_xs, batch_ys = mnist.train.next_batch(batch_size)\n",
    "        batch_xs = batch_xs.reshape([batch_size, n_steps, n_inputs])\n",
    "        sess.run([train_op], feed_dict={\n",
    "            x: batch_xs,\n",
    "            y: batch_ys,\n",
    "        })\n",
    "        if step % 20 == 0:\n",
    "            print(sess.run(accuracy, feed_dict={\n",
    "            x: batch_xs,\n",
    "            y: batch_ys,\n",
    "            }))\n",
    "        step += 1"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}