classification_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from keras.models import Sequential\n",
    "from keras.layers import (Dense, CuDNNLSTM, Dropout, Conv1D, Conv2D, Reshape, Activation, MaxPooling2D, Flatten,\n",
    "                        BatchNormalization)\n",
    "import os\n",
    "import tensorflow as tf\n",
    "import keras\n",
    "from keras.optimizers import Adam\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.losses import huber_loss\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import datetime\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import seaborn\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给ret分类\n",
    "def label_ret(ret):\n",
    "    '''\n",
    "    class 0 = [-inf, -0.1]\n",
    "    class 1 = [-0.1, 0.1] unprofitable\n",
    "    class 2 = [0.1, inf]\n",
    "    '''\n",
    "    label = None\n",
    "    if ret < -0.1:\n",
    "        label = 0\n",
    "    elif -0.1 <= ret and ret <= 0.1:\n",
    "        label = 1\n",
    "    else:\n",
    "        label = 2\n",
    "        \n",
    "    return label\n",
    "def label_ret_bi(ret):\n",
    "    label = None\n",
    "    if ret <= 0:\n",
    "        label = 0\n",
    "    else:\n",
    "        label = 1\n",
    "    return label\n",
    "def label_ret2(ret):\n",
    "    if -0.1 < ret and ret < 0.1:\n",
    "        return 0\n",
    "    else:\n",
    "        return 1\n",
    "    \n",
    "def label_ret3(ret):\n",
    "    if -1 <= ret and ret <= 1:\n",
    "        return 0\n",
    "    elif 1 < ret:\n",
    "        return 1\n",
    "    elif ret < -1:\n",
    "        return 2\n",
    "    else:\n",
    "        raise ValueError\n",
    "        \n",
    "def generate_sequence(X_df, y_series, seq_length):\n",
    "    assert (X_df.index == y_series.index).all()\n",
    "    dataX = list()\n",
    "    dataY = list()\n",
    "    index = list()\n",
    "    for i in range(0, X_df.shape[0] - seq_length + 1):\n",
    "        dataX.append(X_df[i:i+seq_length])\n",
    "        dataY.append(y_series[i+seq_length-1])\n",
    "        index.append(y_series.index[i+seq_length-1])\n",
    "        \n",
    "    return dataX, dataY, pd.Index(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "variety = 'RB'\n",
    "factor_store = pd.HDFStore('/home/data/vb/training_x_150.h5', mode='r')\n",
    "factor_df = factor_store.get(variety)\n",
    "y_store = pd.HDFStore('/home/data/vb/training_y_reg_150.h5', mode='r')\n",
    "y_series = y_store.get(variety)\n",
    "helper_df = pd.read_parquet('/home/data/vb/training_helper_150_{}.parquet'.format(variety))\n",
    "\n",
    "# 对ret做分类\n",
    "ret_y_series = np.exp(y_series) - 1 # 获得回报的原始收益\n",
    "\n",
    "label_y_series = ret_y_series.transform(label_ret_bi).rename('Y_label') # 分类标签\n",
    "ret_label_df = pd.concat([ret_y_series, label_y_series], axis=1) # 合并log ret和label\n",
    "assert (factor_df.index == label_y_series.index).all() # 确认数据和标签索引一样\n",
    "\n",
    "\n",
    "# 对齐日期 去掉na\n",
    "pd.options.mode.use_inf_as_na = True\n",
    "df = helper_df.join(factor_df, how='inner').join(label_y_series, how='inner')\n",
    "df.dropna(inplace=True)\n",
    "\n",
    "# 再次得到 factor_df, label_y_series, helper_df\n",
    "factor_df = df[factor_df.columns]\n",
    "label_y_series = df[label_y_series.name]\n",
    "helper_df = df[helper_df.columns]\n",
    "\n",
    "assert (factor_df.index == label_y_series.index).all() and \\\n",
    "        (label_y_series.index == helper_df.index).all()     # 确认数据和标签索引一样\n",
    "\n",
    "# train val test split\n",
    "factor_df_train, factor_df_test, label_y_series_train, label_y_series_test = \\\n",
    "train_test_split(factor_df, label_y_series, test_size=0.2, shuffle=False)\n",
    "factor_df_train, factor_df_val, label_y_series_train, label_y_series_val = \\\n",
    "train_test_split(factor_df_train, label_y_series_train, test_size=0.1, shuffle=False)\n",
    "\n",
    "\n",
    "# normalize data 在这里会丢失dataframe\n",
    "scaler = StandardScaler()\n",
    "scaler.fit(factor_df_train)\n",
    "factor_train_normalized = scaler.transform(factor_df_train)\n",
    "factor_val_normalized = scaler.transform(factor_df_val)\n",
    "factor_test_normalized = scaler.transform(factor_df_test)\n",
    "\n",
    "\n",
    "# 将dataframe的index和columns加回去\n",
    "factor_df_train_normalized = pd.DataFrame(factor_train_normalized, \n",
    "                                          index=factor_df_train.index, columns=factor_df_train.columns)\n",
    "factor_df_val_normalized = pd.DataFrame(factor_val_normalized, \n",
    "                                        index=factor_df_val.index, columns=factor_df_val.columns)\n",
    "factor_df_test_normalized = pd.DataFrame(factor_test_normalized, \n",
    "                                         index=factor_df_test.index, columns=factor_df_test.columns)\n",
    "del factor_train_normalized\n",
    "del factor_val_normalized\n",
    "del factor_test_normalized\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 给lstm制造时间序列数据\n",
    "\n",
    "NUM_CLASSES = 2\n",
    "seq_length = 100\n",
    "X_train, y_train, index_train = generate_sequence(factor_df_train_normalized, label_y_series_train, seq_length)\n",
    "X_val, y_val, index_val = generate_sequence(factor_df_val_normalized, label_y_series_val, seq_length)\n",
    "X_test, y_test, index_test = generate_sequence(factor_df_test_normalized, label_y_series_test, seq_length)\n",
    "\n",
    "X_train = np.array([factor_seq_df.values for factor_seq_df in X_train]) # 将list 转换为ndarray\n",
    "X_val = np.array([factor_seq_df.values for factor_seq_df in X_val]) # 将list 转换为ndarray\n",
    "X_test = np.array([factor_seq_df.values for factor_seq_df in X_test]) # 将list 转换为ndarray\n",
    "\n",
    "y_cat_train = keras.utils.to_categorical(y_train, num_classes=NUM_CLASSES) # 标签转换为one hot\n",
    "y_cat_val = keras.utils.to_categorical(y_val, num_classes=NUM_CLASSES) # 标签转换为one hot\n",
    "y_cat_test = keras.utils.to_categorical(y_test, num_classes=NUM_CLASSES) # 标签转换为one hot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 176, 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape[1:] + (1,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_gpu_option():\n",
    "    os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\" \n",
    "    config = tf.ConfigProto()\n",
    "    config.gpu_options.allow_growth=True\n",
    "    sess = tf.Session(config=config)\n",
    "    keras.backend.tensorflow_backend.set_session(sess)\n",
    "    \n",
    "    return sess\n",
    "    \n",
    "def create_lstm_model(input_shape, num_classes):\n",
    "    NUM_NEURONS = 1\n",
    "    MULTIPLIER = 1\n",
    "    model = Sequential()\n",
    "    model.add(CuDNNLSTM(64*MULTIPLIER, input_shape=input_shape, return_sequences=True))\n",
    "    model.add(CuDNNLSTM(32*MULTIPLIER, return_sequences=False))\n",
    "    model.add(Dense(16*MULTIPLIER, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))  \n",
    "    model.summary()\n",
    "    \n",
    "    return model\n",
    "\n",
    "def create_lstm_model_dropout(input_shape, num_classes):\n",
    "    NUM_NEURONS = 1\n",
    "    MULTIPLIER = 8\n",
    "    model = Sequential()\n",
    "    model.add(CuDNNLSTM(64*MULTIPLIER, input_shape=input_shape, return_sequences=True))\n",
    "    model.add(Dropout(0.25))\n",
    "    model.add(CuDNNLSTM(32*MULTIPLIER, return_sequences=False))\n",
    "    model.add(Dense(16*MULTIPLIER, activation='relu'))\n",
    "    model.add(Dense(num_classes, activation='softmax'))  \n",
    "    model.summary()\n",
    "    \n",
    "    return model\n",
    "\n",
    "def create_cnn_model(input_shape, num_classes):\n",
    "    model = Sequential()\n",
    "    model.add(Reshape(input_shape+(1,), input_shape=input_shape))\n",
    "    model.add(Conv2D(32, (5, 5), padding='same', activation='relu'))\n",
    "\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    model.add(Dropout(0.5))\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    \n",
    "    model.summary()\n",
    "    \n",
    "    return model\n",
    "\n",
    "def create_cnn_conv1d_model(input_shape, num_classes):\n",
    "    model = Sequential()\n",
    "    model.add(Conv1D(filters=20, kernel_size=30, activation='relu', input_shape=input_shape))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Conv1D(filters=30, kernel_size=30, activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Conv1D(filters=30, kernel_size=30, activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Flatten())\n",
    "    model.add(Dense(1024, activation='relu'))\n",
    "    model.add(BatchNormalization())\n",
    "    model.add(Dense(num_classes, activation='softmax'))\n",
    "    model.summary()\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import backend as K\n",
    "def Precision(y_true, y_pred):\n",
    "    \"\"\"精确率\"\"\"\n",
    "    tp= K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))  # true positives\n",
    "    pp= K.sum(K.round(K.clip(y_pred, 0, 1))) # predicted positives\n",
    "    precision = tp/ (pp+ K.epsilon())\n",
    "    return precision\n",
    "    \n",
    "def Recall(y_true, y_pred):\n",
    "    \"\"\"召回率\"\"\"\n",
    "    tp = K.sum(K.round(K.clip(y_true * y_pred, 0, 1))) # true positives\n",
    "    pp = K.sum(K.round(K.clip(y_true, 0, 1))) # possible positives\n",
    "    recall = tp / (pp + K.epsilon())\n",
    "    return recall\n",
    " \n",
    "def F1(y_true, y_pred):\n",
    "    \"\"\"F1-score\"\"\"\n",
    "    precision = Precision(y_true, y_pred)\n",
    "    recall = Recall(y_true, y_pred)\n",
    "    f1 = 2 * ((precision * recall) / (precision + recall + K.epsilon()))\n",
    "    return f1    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_shape:  (100, 176)\n",
      "Model: \"sequential_8\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv1d_22 (Conv1D)           (None, 71, 20)            105620    \n",
      "_________________________________________________________________\n",
      "batch_normalization_29 (Batc (None, 71, 20)            80        \n",
      "_________________________________________________________________\n",
      "conv1d_23 (Conv1D)           (None, 42, 30)            18030     \n",
      "_________________________________________________________________\n",
      "batch_normalization_30 (Batc (None, 42, 30)            120       \n",
      "_________________________________________________________________\n",
      "conv1d_24 (Conv1D)           (None, 13, 30)            27030     \n",
      "_________________________________________________________________\n",
      "batch_normalization_31 (Batc (None, 13, 30)            120       \n",
      "_________________________________________________________________\n",
      "flatten_8 (Flatten)          (None, 390)               0         \n",
      "_________________________________________________________________\n",
      "dense_15 (Dense)             (None, 1024)              400384    \n",
      "_________________________________________________________________\n",
      "batch_normalization_32 (Batc (None, 1024)              4096      \n",
      "_________________________________________________________________\n",
      "dense_16 (Dense)             (None, 2)                 2050      \n",
      "=================================================================\n",
      "Total params: 557,530\n",
      "Trainable params: 555,322\n",
      "Non-trainable params: 2,208\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sess = set_gpu_option()\n",
    "input_shape = (X_train.shape[1], X_train.shape[2])\n",
    "print('input_shape: ', input_shape)\n",
    "model = create_cnn_conv1d_model(input_shape=input_shape ,num_classes=NUM_CLASSES)\n",
    "adam = keras.optimizers.Adam(learning_rate=1e-3)\n",
    "model.compile(optimizer=adam, loss='categorical_crossentropy', metrics=['accuracy', Precision, Recall, F1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "majority_label_train: \n",
      " [0.50641656 0.49358344]\n",
      "majority_label_val: \n",
      " [0.5216285 0.4783715]\n",
      "majority_label_test: \n",
      " [0.51504767 0.4849523 ]\n"
     ]
    }
   ],
   "source": [
    "majority_label_train = np.sum(y_cat_train, axis=0)/np.sum(y_cat_train)\n",
    "print('majority_label_train: \\n', majority_label_train)\n",
    "majority_label_val = np.sum(y_cat_val, axis=0)/np.sum(y_cat_val)\n",
    "print('majority_label_val: \\n', majority_label_val)\n",
    "majority_label_test = np.sum(y_cat_test, axis=0)/np.sum(y_cat_test)\n",
    "print('majority_label_test: \\n', majority_label_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "log_dir = \"logs/fit/\" + datetime.datetime.now().strftime(\"%Y%m%d-%H%M%S\")\n",
    "tensorboard_callback = keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1, write_grads=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 53845 samples, validate on 5895 samples\n",
      "Epoch 1/10\n",
      "53845/53845 [==============================] - 25s 456us/step - loss: 0.8555 - accuracy: 0.4984 - Precision: 0.4984 - Recall: 0.4984 - F1: 0.4984 - val_loss: 1.1560 - val_accuracy: 0.5282 - val_Precision: 0.5284 - val_Recall: 0.5284 - val_F1: 0.5284\n",
      "Epoch 2/10\n",
      "53845/53845 [==============================] - 24s 442us/step - loss: 0.7465 - accuracy: 0.5161 - Precision: 0.5161 - Recall: 0.5161 - F1: 0.5161 - val_loss: 1.2122 - val_accuracy: 0.4877 - val_Precision: 0.4875 - val_Recall: 0.4875 - val_F1: 0.4875\n",
      "Epoch 3/10\n",
      "53845/53845 [==============================] - 24s 442us/step - loss: 0.7041 - accuracy: 0.5702 - Precision: 0.5702 - Recall: 0.5702 - F1: 0.5702 - val_loss: 1.7589 - val_accuracy: 0.4772 - val_Precision: 0.4770 - val_Recall: 0.4770 - val_F1: 0.4770\n",
      "Epoch 4/10\n",
      "53845/53845 [==============================] - 24s 442us/step - loss: 0.6576 - accuracy: 0.6305 - Precision: 0.6305 - Recall: 0.6305 - F1: 0.6305 - val_loss: 1.6006 - val_accuracy: 0.4755 - val_Precision: 0.4753 - val_Recall: 0.4753 - val_F1: 0.4753\n",
      "Epoch 5/10\n",
      "53845/53845 [==============================] - 24s 447us/step - loss: 0.6088 - accuracy: 0.6768 - Precision: 0.6768 - Recall: 0.6768 - F1: 0.6768 - val_loss: 1.1754 - val_accuracy: 0.4989 - val_Precision: 0.4992 - val_Recall: 0.4992 - val_F1: 0.4992\n",
      "Epoch 6/10\n",
      "53845/53845 [==============================] - 24s 442us/step - loss: 0.5738 - accuracy: 0.7084 - Precision: 0.7085 - Recall: 0.7085 - F1: 0.7085 - val_loss: 2.1925 - val_accuracy: 0.4746 - val_Precision: 0.4744 - val_Recall: 0.4744 - val_F1: 0.4744\n",
      "Epoch 7/10\n",
      "53845/53845 [==============================] - 24s 444us/step - loss: 0.5456 - accuracy: 0.7271 - Precision: 0.7271 - Recall: 0.7271 - F1: 0.7271 - val_loss: 2.1488 - val_accuracy: 0.4765 - val_Precision: 0.4763 - val_Recall: 0.4763 - val_F1: 0.4763\n",
      "Epoch 8/10\n",
      "53845/53845 [==============================] - 24s 441us/step - loss: 0.5196 - accuracy: 0.7466 - Precision: 0.7466 - Recall: 0.7466 - F1: 0.7466 - val_loss: 2.0213 - val_accuracy: 0.4777 - val_Precision: 0.4775 - val_Recall: 0.4775 - val_F1: 0.4775\n",
      "Epoch 9/10\n",
      "53845/53845 [==============================] - 24s 442us/step - loss: 0.4966 - accuracy: 0.7623 - Precision: 0.7623 - Recall: 0.7623 - F1: 0.7623 - val_loss: 2.2435 - val_accuracy: 0.4638 - val_Precision: 0.4636 - val_Recall: 0.4636 - val_F1: 0.4636\n",
      "Epoch 10/10\n",
      "53845/53845 [==============================] - 24s 443us/step - loss: 0.4753 - accuracy: 0.7757 - Precision: 0.7758 - Recall: 0.7758 - F1: 0.7758 - val_loss: 2.5394 - val_accuracy: 0.4680 - val_Precision: 0.4679 - val_Recall: 0.4679 - val_F1: 0.4679\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x7f5be87d49e8>"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(x=X_train, y=y_cat_train, epochs=10, verbose=True, batch_size=None, validation_data=(X_val, y_cat_val), \n",
    "          shuffle=False)#, callbacks=[tensorboard_callback])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14886/14886 [==============================] - 3s 203us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.210644110478686,\n",
       " 0.4905951917171478,\n",
       " 0.4897398054599762,\n",
       " 0.4897398054599762,\n",
       " 0.4897397756576538]"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(x=X_test, y=y_cat_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.5476969 , 0.45230305],\n",
       "       [0.3923277 , 0.60767233],\n",
       "       [0.37865564, 0.6213443 ],\n",
       "       ...,\n",
       "       [0.6948963 , 0.30510372],\n",
       "       [0.5194539 , 0.48054606],\n",
       "       [0.59105015, 0.40894982]], dtype=float32)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prediction = model.predict(X_test)\n",
    "prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10704"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "sum(np.argmax(prediction, axis=1) != 0)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}