NEC_Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import unicode_literals, print_function, division\n",
    "from io import open\n",
    "import unicodedata\n",
    "import string\n",
    "import re\n",
    "import random\n",
    "import math\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "class C_L(nn.Module):\n",
    "    def __init__(self, input_dim=1, hidden_dim=256, layer_dim=8):\n",
    "        super(C_L, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.layer_dim = layer_dim\n",
    "        self.input_dim = input_dim\n",
    "\n",
    "        # stacked LSTM\n",
    "        self.lstm = nn.LSTM(self.input_dim, self.hidden_dim, self.layer_dim, dropout=0.4, bidirectional=False, batch_first=True)\n",
    "\n",
    "    def forward(self, x, s_h, s_c):\n",
    "        # Initialize hidden and cell state\n",
    "        h0 = s_h\n",
    "        c0 = s_c\n",
    "        out, (hn, cn) = self.lstm(x, (h0,c0))\n",
    "        return out, hn, cn\n",
    "\n",
    "\n",
    "class C_F(nn.Module):\n",
    "    def __init__(self, hidden_dim=64, output_dim=24):\n",
    "        super(C_F, self).__init__()\n",
    "        self.hidden_dim = hidden_dim\n",
    "        self.output_dim = output_dim\n",
    "        self.bn1 = nn.BatchNorm1d(num_features=self.hidden_dim)\n",
    "        self.L_out1 = nn.Linear(self.hidden_dim, self.output_dim) \n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def forward(self, x, id=-1):\n",
    "        linear_out = self.L_out1(self.bn1(x[:,id,:]))\n",
    "        linear_out = self.sigmoid(linear_out)\n",
    "        return linear_out\n",
    "\n",
    "# prepare train and val set\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.utils.data import DataLoader\n",
    "class RnnDataset(TensorDataset):\n",
    "    def __init__(self, data, label):\n",
    "        self.data = data\n",
    "        self.label = label\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        return self.data[index],self.label[index] \n",
    "    \n",
    "    def __len__(self):\n",
    "        return len(self.data)\n",
    "\n",
    "def read_dataset(sensor_id=4004,start_point='1966-12-18 14:30:00',train_point='2014-01-20 9:30:00'):\n",
    "\n",
    "    if(str(sensor_id)[0]=='4'):\n",
    "        trainX = pd.read_csv('./data'+'/reservoir_stor_'+str(sensor_id)+'_sof24.tsv', sep='\\t')\n",
    "    elif(str(sensor_id)[0]=='6'):\n",
    "        trainX = pd.read_csv('./data'+'/raingauge_byhour_'+str(sensor_id)+'_sof.tsv', sep='\\t')\n",
    "    else:\n",
    "        print(\"Error: the support sensor type is 4*** or 6****.\")\n",
    "        sys.exit()\n",
    "        \n",
    "    start_num = trainX[trainX[\"TSC_TSTAMP_UTC\"]==start_point].index.values[0]\n",
    "    print(\"for sensor \", sensor_id, \"start_num is: \", start_num)\n",
    "    idx_num = 0\n",
    "    print(len(trainX))\n",
    "    print(trainX[:3])\n",
    "\n",
    "    val_skips = []\n",
    "    \n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_validation_timestamps_24avg.tsv', sep='\\t')\n",
    "    val_points = val_set[\"Hold Out Start\"]\n",
    "    for val_i in val_points:\n",
    "        valskip = trainX[trainX[\"TSC_TSTAMP_UTC\"]==val_i].index.values[0] - start_num\n",
    "        val_skips.append(valskip)\n",
    "    \n",
    "    # id of test_start and test_end, save it in the last two values in val_skips\n",
    "    test_start = trainX[trainX[\"TSC_TSTAMP_UTC\"]=='2017-01-01 14:30:00'].index.values[0] - start_num \n",
    "    test_end = trainX[trainX[\"TSC_TSTAMP_UTC\"]=='2018-12-31 14:30:00'].index.values[0] - start_num\n",
    "    val_skips.append(test_start)\n",
    "    val_skips.append(test_end)   \n",
    "        \n",
    "    print(\"val_skips is: \", val_skips)\n",
    "\n",
    "    #foot label of train_end\n",
    "    train_end = trainX[trainX[\"TSC_TSTAMP_UTC\"]==train_point].index.values[0] - start_num \n",
    "    print(\"train_end is : \", train_end)\n",
    "\n",
    "    #the whole dataset\n",
    "    train_x = trainX[start_num:] \n",
    "    print(len(train_x))\n",
    "    #train_end = len(train_x) # the whole data set? It's your choice\n",
    "    sensor_data = np.array(train_x[\"TSC_VALUE_F\"])\n",
    "    time_data = [math.floor(int(np.array(train_x[\"TSC_TSTAMP_UTC\"])[i][5:7])-1) for i in range(len(train_x))]\n",
    "    \n",
    "    return sensor_data, train_end, val_skips, time_data, start_num\n",
    " \n",
    "def diff_norm_dataset(sensor_data0):\n",
    "    a = sensor_data0\n",
    "    b = a[:-1]\n",
    "    a = a[1:] - b\n",
    "    c = np.array([0]+a.tolist())\n",
    "    mean = c.mean()\n",
    "    print(\"mean is: \", mean)\n",
    "    std = c.std()\n",
    "    print(\"std is \", std)\n",
    "    c = (c - mean) / std\n",
    "\n",
    "    extream_num = 0\n",
    "    print(\"extream boundary is: \", std * alpha_extreams)\n",
    "    total_num = len(c)\n",
    "    for i in range(total_num):\n",
    "        if(abs(c[i]) > alpha_extreams):\n",
    "            extream_num += 1\n",
    "    print(\" the total norm data number is: \", total_num)\n",
    "    print(\" the number of extreams is: \", extream_num, \" the ratio is: \", extream_num/total_num)\n",
    "\n",
    "    return c, mean, std\n",
    "\n",
    "def std_norm_dataset(sensor_data0):\n",
    "\n",
    "    c = sensor_data0\n",
    "    mean = c.mean()\n",
    "    print(\"mean is: \", mean)\n",
    "    std = c.std()\n",
    "    print(\"std is \", std)\n",
    "    c = (c - mean) / std\n",
    "  \n",
    "    return c, mean, std\n",
    "\n",
    "\n",
    "def split_dataset(norm_data,train_end, input_dim=24, output_size=24, train_days=90, predict_days=3, train_volum=50000):\n",
    "\n",
    "    en_seq_len = train_days\n",
    "    de_seq_len = predict_days\n",
    "\n",
    "    DATA = []\n",
    "    Label = []\n",
    "    print(\"norm data size:\", len(norm_data))\n",
    "\n",
    "    train_size = train_end + 1 - (train_days * input_dim + output_size * predict_days)\n",
    "    print(\"Available train size is: \", train_size)\n",
    "    \n",
    "    # step 1, randomly choose train data from normed data\n",
    "    norm_num = train_volum * norm_percent\n",
    "    extreme_num = train_volum * (1 - norm_percent)\n",
    "    extreme_total = 0\n",
    "    norm_total = 0\n",
    "    \n",
    "    # avoid points in validation set and whole test set years\n",
    "    val_skips_extend = []\n",
    "    l = len(val_skips)\n",
    "    for i in range(l-2):\n",
    "        for j in range(output_size*predict_days):\n",
    "            val_skips_extend.append(val_skips[i]+j)\n",
    "    \n",
    "    test_start = val_skips[l-2]\n",
    "    print(\"test set start ID is: \", test_start)\n",
    "    test_end = val_skips[l-1]\n",
    "    print(\"test set end ID is: \", test_end)\n",
    "    \n",
    "    while(extreme_total<extreme_num or norm_total<norm_num):\n",
    "        i = random.randint(0, train_size)\n",
    "    \n",
    "        if (i+train_days not in val_skips_extend ) and ((i+train_days)<test_start or (i+train_days)>test_end): \n",
    "            data0 = np.array(norm_data[i:(i+train_days*input_dim)]).reshape(train_days,-1)\n",
    "            label0 = np.array(class_label[(i+train_days*input_dim):(i+train_days*input_dim+output_size*predict_days)]) \n",
    "            label1 = np.array(norm_data[(i+train_days*input_dim):(i+train_days*input_dim+output_size*predict_days)]) \n",
    "\n",
    "            # step 2, over sampling extreme points\n",
    "            if(label0.sum()>=1 and extreme_total<extreme_num):\n",
    "                extreme_total += 1;\n",
    "                DATA.append(data0)\n",
    "                Label.append(label0)\n",
    "\n",
    "            if(label0.sum()<1 and norm_total<norm_num):\n",
    "                norm_total += 1;\n",
    "                DATA.append(data0)\n",
    "                Label.append(label0) \n",
    "    \n",
    "    dataset1 = RnnDataset(DATA, Label)\n",
    "    data_loader = DataLoader(dataset1, \n",
    "                         batch_size,\n",
    "                         shuffle=True,\n",
    "                         num_workers=2,\n",
    "                         pin_memory=True,\n",
    "                         collate_fn=lambda x: x)\n",
    "    return data_loader\n",
    "\n",
    "#initiate models\n",
    "def create_model(input_dim, hidden_dim, layer_dim, output_dim):\n",
    "    encoder = C_L(input_dim, hidden_dim, layer_dim)\n",
    "    outlinear = C_F(hidden_dim, predict_days) \n",
    "\n",
    "    encoder = encoder.to(device)\n",
    "    outlinear = outlinear.to(device)\n",
    "\n",
    "    criterion = nn.BCELoss(reduction='sum')\n",
    "    criterion1 = nn.MSELoss(reduction='sum')\n",
    "    encoder_optimizer = torch.optim.SGD(encoder.parameters(),0.001)  \n",
    "    outlinear_optimizer = torch.optim.Adam(outlinear.parameters(),0.0005) \n",
    "    return encoder, outlinear, encoder_optimizer, outlinear_optimizer, criterion, criterion1\n",
    "\n",
    "def train_loop(max_accuracy):\n",
    "    num_epochs = 100 #early stop or reach to num_epochs\n",
    "    early_stop = 0\n",
    "    old_val_acc = 0\n",
    "    for epoch in range(num_epochs):\n",
    "        print_loss_total = 0  # Reset every epoch\n",
    "        encoder.train()\n",
    "        outlinear.train()\n",
    "        for i, batch in enumerate(dataloader):\n",
    "            x_train = [TrainData for TrainData, _ in batch]\n",
    "            y_train = [TrainLabel for _, TrainLabel in batch]\n",
    "            x_train = torch.from_numpy(np.array(x_train, np.float32)).to(device)\n",
    "            y_train = torch.from_numpy(np.array(y_train,np.float32)).to(device)\n",
    "\n",
    "            # Clear gradients w.r.t. parameters\n",
    "            encoder_optimizer.zero_grad()\n",
    "            outlinear_optimizer.zero_grad()\n",
    "            loss = 0\n",
    "\n",
    "            # Forward pass\n",
    "            s_h = torch.zeros(layer_dim, x_train.size(0), hidden_dim).to(device)\n",
    "            s_c = torch.zeros(layer_dim, x_train.size(0), hidden_dim).to(device)\n",
    "            encoder_out, _, _ = encoder(x_train, s_h, s_c)\n",
    "            e_out = outlinear(encoder_out)\n",
    "        \n",
    "            # C loss parameters  alpha=2  beta=0.5\n",
    "            loss = 0.5 * criterion(e_out**2, y_train) + 0.5 * criterion1(e_out, y_train)\n",
    "        \n",
    "            loss.backward()\n",
    "            encoder_optimizer.step()\n",
    "            outlinear_optimizer.step()\n",
    "            print_loss_total += loss.item()\n",
    "\n",
    "        encoder.eval()\n",
    "        outlinear.eval()\n",
    "        val_acc, max_accuracy = generate_val_accuracy(max_accuracy, epoch)\n",
    "        print('-----------Epoch: {}. train_Loss>: {:.6f}. --------------------'.format(epoch, print_loss_total)) \n",
    "        print('-----------Epoch: {}. val_f1_score>: {:.6f}. --------------------'.format(epoch, val_acc)) \n",
    "\n",
    "        if val_acc == 1:\n",
    "            break\n",
    "\n",
    "        #early stop\n",
    "        if(val_acc < old_val_acc):\n",
    "            early_stop += 1\n",
    "        else:\n",
    "            early_stop = 0\n",
    "        if(early_stop >= 4):     # C early stop is 4\n",
    "            break\n",
    "        old_val_acc = val_acc\n",
    "    return max_accuracy\n",
    "\n",
    "def inference_test(encoder, outlinear, x_test):\n",
    "    \n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    y_predict = []\n",
    "    with torch.no_grad():\n",
    "        x_test = torch.from_numpy(np.array(x_test, np.float32)).to(device)\n",
    "        s_h = torch.zeros(layer_dim, x_test.size(0), hidden_dim).to(device)\n",
    "        s_c = torch.zeros(layer_dim, x_test.size(0), hidden_dim).to(device)\n",
    "        encoder_out, encoder_h, encoder_c = encoder(x_test, s_h, s_c)\n",
    "        d_out = outlinear(encoder_out)\n",
    "        y_predict.extend(d_out[0])\n",
    "        y_predict = [y_predict[i].item() for i in range(len(y_predict))]\n",
    "        y_predict = np.array(y_predict).reshape(1,-1) \n",
    "        \n",
    "    return y_predict\n",
    "\n",
    "def test(test_point):\n",
    "    \n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    #test_data\n",
    "    if(str(sensor_id)[0]=='4'):\n",
    "        trainX = pd.read_csv('./data'+'/reservoir_stor_'+str(sensor_id)+'_sof24.tsv', sep='\\t')\n",
    "    elif (str(sensor_id)[0]=='6'):\n",
    "        trainX = pd.read_csv('./data'+'/raingauge_byhour_'+str(sensor_id)+'_sof.tsv', sep='\\t')\n",
    "    else:\n",
    "        print(\"Error: the support sensor type is 4*** or 6****.\")\n",
    "        sys.exit()\n",
    "        \n",
    "    #foot label of test_data\n",
    "    point = trainX[trainX[\"TSC_TSTAMP_UTC\"]==test_point].index.values[0]\n",
    "    test_point = point - start_num\n",
    "    x = trainX[point-train_days*input_dim:point][\"TSC_TSTAMP_UTC\"]\n",
    "    y = class_label[test_point:test_point+predict_days*output_dim]\n",
    "\n",
    "    #inference\n",
    "    norm_data = sensor_data_norm\n",
    "    x_test = np.array(norm_data[test_point-train_days*input_dim:test_point], np.float32).reshape(train_days, -1)\n",
    "    x_test = [x_test]\n",
    "    y_predict = inference_test(encoder=encoder,outlinear=outlinear,x_test=x_test)\n",
    "    y_predict = np.array(y_predict.tolist())[0]\n",
    "    \n",
    "    return y_predict,y\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import f1_score\n",
    "def generate_val_accuracy(max_accuracy, epoch):\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    #test_data\n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_validation_timestamps_24avg.tsv',sep='\\t')\n",
    "    val_points = val_set[\"Hold Out Start\"]\n",
    "\n",
    "    total = 0\n",
    "    val_loss_list = []\n",
    "    val_pred_list = []\n",
    "    val_pred_lists_print = []\n",
    "    ground_truth_lists = []\n",
    "    for i in range(len(val_points)):\n",
    "        val_pred_list_print = []\n",
    "        ground_truth_list = []\n",
    "        val_point = val_points[i]\n",
    "        test_predict, ground_truth = test(test_point=val_point)\n",
    "        test_predict = [1 if test_predict[i].item()>=0.5 else 0 for i in range(len(test_predict))]  \n",
    "    \n",
    "        for j in range(len(test_predict)):\n",
    "            val_pred_list_print.append(test_predict[j])\n",
    "            ground_truth_list.append(ground_truth[j])\n",
    "    \n",
    "        val_pred_lists_print.append(val_pred_list_print)\n",
    "        ground_truth_lists.append(ground_truth_list)\n",
    "    \n",
    "    val_acc = f1_score(np.array(ground_truth_lists).reshape(1,-1).squeeze(), np.array(val_pred_lists_print).reshape(1,-1).squeeze(), average='micro')\n",
    "    new_max_accuracy = max_accuracy\n",
    "                  \n",
    "    if (val_acc > max_accuracy) :\n",
    "        aa = pd.DataFrame(data=val_pred_lists_print)\n",
    "        aa.to_csv('./val/'+str(sensor_id)+norm+'classification.tsv', sep='\\t')\n",
    "        #save_model\n",
    "        torch.save(outlinear, outlinear_name) \n",
    "        torch.save(encoder, encoder_name)\n",
    "        new_max_accuracy = val_acc\n",
    "    print(\"val max F1 score: \", new_max_accuracy)\n",
    "\n",
    "    return val_acc, new_max_accuracy\n",
    "\n",
    "def generate_test(path, filename):\n",
    "    test_predict = np.zeros(predict_days*output_dim)\n",
    "    #test_data\n",
    "    val_set = pd.read_csv('./data/'+str(sensor_id)+'_test_timestamps_24avg.tsv',sep='\\t')\n",
    "    val_points = val_set[\"Start\"]\n",
    "    total = 0\n",
    "    val_pred_list = []\n",
    "    val_pred_lists_print = []\n",
    "    ground_truth_lists = []\n",
    "    \n",
    "    for i in range(len(val_points)):\n",
    "        val_pred_list_print = []\n",
    "        ground_truth_list = []\n",
    "        val_point = val_points[i]\n",
    "        test_predict, ground_truth = test(test_point=val_point)\n",
    "        test_predict = [1 if test_predict[i].item() >= 0.5 else 0 for i in range(len(test_predict))]  \n",
    "    \n",
    "        for j in range(len(test_predict)):\n",
    "            val_pred_list_print.append(test_predict[j])\n",
    "            ground_truth_list.append(ground_truth[j])\n",
    "            \n",
    "        val_pred_lists_print.append(val_pred_list_print)\n",
    "        ground_truth_lists.append(ground_truth_list)\n",
    "                  \n",
    "    aa = pd.DataFrame(data = val_pred_lists_print)\n",
    "    aa.to_csv(path+filename+'.tsv', sep='\\t')\n",
    "    print(\"test classification is saved at:  \" + path + filename + '.tsv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "sensor_ids = [4009]\n",
    "\n",
    "for i in range(len(sensor_ids)) :\n",
    "    # sensor's data\n",
    "    sensor_id = sensor_ids[i]\n",
    "    \n",
    "    # dataset parameters\n",
    "    start_point = '1980-12-31 14:30:00'\n",
    "    max_accuracy = 0\n",
    "    print(\"Begin!! Sensor id is :---------------------\",sensor_id)\n",
    "\n",
    "    # training parameters\n",
    "    input_dim = 1          # univari forecasting, set 1\n",
    "    output_dim = 1           # univari forecasting, set 1\n",
    "    train_days = 15*24       # history vector length, h=360\n",
    "    train_end = 0\n",
    "    predict_days = 24*3      # forecasting vector length, f=72\n",
    "    train_point = '2019-12-31 14:30:00'\n",
    "    alpha_extreams = 1.5     # normal/extreme boundary, \\epsilon=1.5\n",
    "    print(\"alpha_extreams is: \", alpha_extreams)\n",
    "    norm_percent = 0.0       # C use oversampling, OS%=1\n",
    "    is_diff = 1              # one order difference, set 1\n",
    "    if(is_diff==0):\n",
    "        norm = \"std\"\n",
    "    else:\n",
    "        norm = \"diff\"\n",
    "\n",
    "    val_skips = []\n",
    "    sensor_data, train_end, val_skips, time_data, start_num = read_dataset(sensor_id=sensor_id,start_point=start_point,train_point=train_point)\n",
    "\n",
    "    if(is_diff==0):\n",
    "        sensor_data_norm, mean, std = std_norm_dataset(sensor_data)\n",
    "        print(\"I am using std_norm.\")\n",
    "    else:\n",
    "        sensor_data_norm, mean, std = diff_norm_dataset(sensor_data)\n",
    "        print(\"I am using diff_std_norm.\")\n",
    "    \n",
    "    class_label = []\n",
    "    for i in range(len(sensor_data_norm)):\n",
    "        if abs(sensor_data_norm[i]) > alpha_extreams :\n",
    "            class_label.append(1)\n",
    "        else:\n",
    "            class_label.append(0)\n",
    "\n",
    "    # model hyperparameters\n",
    "    hidden_dim = 1024         # C hidden = 1024\n",
    "    layer_dim = 4             # C layers = 4\n",
    "    batch_size = 64           # C batch size = 8\n",
    "    train_volum = 1000      # C volume = 100000\n",
    "    print(\"hidden dim is: \", hidden_dim)\n",
    "    print(\"layer dim is: \", layer_dim)\n",
    "    print(\"train_volum is: \", train_volum)\n",
    "\n",
    "    encoder_name = \"./model/\" + str(sensor_id) + '_' + norm + \"LSTM_classifier.txt\"  # C_LSTM model path\n",
    "    outlinear_name = \"./model/\" + str(sensor_id) + '_' + norm + \"FC_classifier.txt\"  # C_FC model path\n",
    "\n",
    "    #get train data_loader \n",
    "    dataloader = split_dataset(norm_data=sensor_data_norm, train_end=train_end, input_dim=input_dim, output_size=output_dim, train_days=train_days, predict_days=predict_days,train_volum=train_volum)\n",
    "\n",
    "    #create_model\n",
    "    encoder, outlinear, encoder_optimizer, outlinear_optimizer,criterion,criterion1 = create_model(input_dim=input_dim, hidden_dim=hidden_dim, layer_dim=layer_dim, output_dim=input_dim)\n",
    "\n",
    "    #train the model\n",
    "    train_loop(max_accuracy)    #if only inferencing, just hide this line.\n",
    "    \n",
    "    print(\"Finish!! Sensor id is :---------------------\",sensor_id)\n",
    "    \n",
    "    #generate test prediction    \n",
    "    encoder = torch.load(encoder_name)\n",
    "    outlinear = torch.load(outlinear_name)\n",
    "    encoder.eval()\n",
    "    outlinear.eval()\n",
    "    generate_test(\"./test/\", str(sensor_id) + 'classification')  # the path of generated test classification\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}