Portfolio_Construction_Notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "colab": {
      "name": "Portfolio Construction Notebook.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Im2p7dk8xC1",
        "colab_type": "text"
      },
      "source": [
        "# Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j3bj9Df_8xC3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import os\n",
        "import itertools\n",
        "#import talib\n",
        "\n",
        "from sklearn.preprocessing import MinMaxScaler\n",
        "from sklearn.metrics import mean_squared_error\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from statsmodels.graphics.tsaplots import plot_acf\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.metrics import mean_squared_error\n",
        "\n",
        "import xgboost as xgb\n",
        "from sklearn.metrics import accuracy_score\n",
        "import seaborn as sns\n",
        "import operator\n",
        "import warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "\n",
        "plt.rc('figure', figsize=(20, 8), dpi=100)\n",
        "from datetime import datetime"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CTvUe-Hw8_cc",
        "colab_type": "text"
      },
      "source": [
        "# Classes and Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JhswmZ1A9Dev",
        "colab_type": "text"
      },
      "source": [
        "Classes to train the regressor given data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5dgINEZX8xC8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "\n",
        "class Plot:\n",
        "    \n",
        "    def __init__(self, data):\n",
        "        self.data = data\n",
        "        \n",
        "    def autocorr(self, col, ticker):\n",
        "        \n",
        "        plot_acf(self.data[col], title = col + ' Autocorrelation Plot of ' + ticker)\n",
        "        plt.show()\n",
        "            \n",
        "    def featureImportance(self, feature_importance, num_features, flag, ticker):\n",
        "    \n",
        "        f = dict()\n",
        "        n = len(feature_importance)\n",
        "        for i in range (n):\n",
        "            f[X_test.columns[i]] = feature_importance[i]\n",
        "        f = sorted(f.items(), key=operator.itemgetter(1), reverse=True)\n",
        "        f = f[:num_features]\n",
        "        feature_name = list()\n",
        "        feature_values = list()\n",
        "        for i, j in f:\n",
        "            feature_name.append(i)\n",
        "            feature_values.append(j)\n",
        "        fig = plt.figure(figsize=(14,5))\n",
        "        plt.xticks(rotation='vertical')\n",
        "        plt.bar([i for i in range(len(f))], feature_values, tick_label=feature_name)\n",
        "        if flag == 1:\n",
        "            plt.title('Feature importance for EPS Prediction of ' + ticker + ' (Excluding Analyst estimate features)')\n",
        "        else:\n",
        "            plt.title('Feature importance for EPS Prediction of ' + ticker + ' (Including Analyst estimate features)')\n",
        "        plt.show()\n",
        "    \n",
        "    def lossStatsAndCurve(self, X_test, Y_test, regressor, ticker):\n",
        "        \n",
        "        \n",
        "        rmse = np.sqrt(mean_squared_error(Y_test, regressor.predict(X_test)))\n",
        "        print(\"Root Mean Squared Error: %f\" % (rmse))\n",
        "        #print (\"Regression Prediction Score: \" + str(round(regressor.score(X_test,Y_test) * 100, 2)) + \"%\")\n",
        "        eval_result = regressor.evals_result()\n",
        "        training_rounds = range(len(eval_result['validation_0']['rmse']))\n",
        "        plt.scatter(x=training_rounds,y=eval_result['validation_0']['rmse'],label='Training Error')\n",
        "        plt.scatter(x=training_rounds,y=eval_result['validation_1']['rmse'],label='Validation Error')\n",
        "        plt.xlabel('Iterations')\n",
        "        plt.ylabel('RMSE')\n",
        "        plt.title('Training Vs Validation Error of ' + ticker)\n",
        "        plt.legend()\n",
        "        plt.show()\n",
        "\n",
        "class Model:\n",
        "    def __init__(self, X_train, Y_train, X_test, Y_test):\n",
        "        \n",
        "        self.X_train = X_train\n",
        "        self.Y_train = Y_train\n",
        "        self.X_test = X_test\n",
        "        self.Y_test = Y_test\n",
        "    \n",
        "    def trainModel(self, epoch = 1000, verbose_flag = False, learning_rate = 0.01):\n",
        "        \n",
        "        regressor = xgb.XGBRegressor(colsample_bytree = 0.4, learning_rate = learning_rate, base_score=0.65, max_depth = 4, alpha = 10, n_estimators = epoch)\n",
        "        xgbModel=regressor.fit(X_train, Y_train,eval_set = [(X_train, Y_train), (X_test, Y_test)], verbose = verbose_flag)\n",
        "        return (xgbModel, regressor)\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jc62-zzG9RYA",
        "colab_type": "text"
      },
      "source": [
        "Functions to get data, either for all time or from beginning of time up until the specified date"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KJfY4fBT8xDA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def getData(data_temp, inc_analyst):\n",
        "    \n",
        "    data = data_temp.copy()\n",
        "    Y = data['EPS (diluted)']\n",
        "    \n",
        "    del data['EPS (recurring)']\n",
        "    del data['EPS (diluted)']\n",
        "    \n",
        "    if inc_analyst == False:\n",
        "        #del data['Growth (YoY%)_Analyst']\n",
        "        del data['EPS_Analyst']\n",
        "    X = data\n",
        "    \n",
        "    train_samples = int(X.shape[0] * 0.75)\n",
        "     \n",
        "    X_train = X.iloc[:train_samples]\n",
        "    X_test = X.iloc[train_samples:]\n",
        "\n",
        "    Y_train = Y.iloc[:train_samples]\n",
        "    Y_test = Y.iloc[train_samples:]\n",
        "    \n",
        "    \n",
        "    return (X_train, Y_train), (X_test, Y_test)\n",
        "\n",
        "def getDataUntilDate(data_temp, data_dates, startingDate, stoppingDate):\n",
        "    data = data_temp.copy()\n",
        "    Y = data['EPS (diluted)']\n",
        "    \n",
        "    del data['EPS (recurring)']\n",
        "    del data['EPS (diluted)']\n",
        "    \n",
        "    X = data\n",
        "    n = X.shape[0]\n",
        "    for i in range(n):\n",
        "        if (data_dates[i] > startingDate):\n",
        "            beginIndex = i\n",
        "            break\n",
        "    \n",
        "    lastIndex = beginIndex+1\n",
        "    for i in range(beginIndex, n):\n",
        "        if (data_dates[i] > stoppingDate):\n",
        "            lastIndex = i\n",
        "            break\n",
        "        \n",
        "    X_train = X.iloc[range(beginIndex,lastIndex)]\n",
        "    Y_train = Y.iloc[range(beginIndex,lastIndex)]\n",
        "    X_test = X.iloc[range(lastIndex, lastIndex+5)]\n",
        "    Y_test = Y.iloc[range(lastIndex, lastIndex+5)]\n",
        "        \n",
        "    return (X_train, Y_train), (X_test, Y_test)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PAngyUic9Uyi",
        "colab_type": "text"
      },
      "source": [
        "Functions to create portfolio weights, given the predicted EPS. The formulas and explanations / rationales of each flavor is outlined in the final report document. The functions createPortfolio1/2/3 correspond to flavor A,B,C in the report respectively. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4ZaePEpJ8xDD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def createPortfolio1(predDf, dataset, datasetDates, long_only = False, exclude_citi = False):\n",
        "    n = predDf.shape[0]\n",
        "    weightColumns = predDf.columns[1:]\n",
        "    weightDf = pd.DataFrame(columns = weightColumns)\n",
        "    k = len(weightColumns)\n",
        "\n",
        "    for i in range(n):\n",
        "        currRow = predDf.iloc[i]\n",
        "        currDate = currRow['Date']\n",
        "        proportions = np.zeros(k-1)\n",
        "        for j in range(1,k):\n",
        "            eps = currRow[weightColumns[j]]\n",
        "            df = dataset[j-1]\n",
        "            dfDates = datasetDates[j-1]\n",
        "            for ii in range(len(dfDates)):\n",
        "                if (dfDates[ii] > currDate):\n",
        "                    foundIndex = ii\n",
        "                    break\n",
        "            \n",
        "            if (foundIndex >= len(df)):\n",
        "                foundIndex = len(df) - 1\n",
        "            \n",
        "            closePrice = df.iloc[foundIndex]['Close']\n",
        "            \n",
        "            \n",
        "            proportions[j-1] = eps / closePrice\n",
        "            if (long_only and proportions[j-1] < 0):\n",
        "                proportions[j-1] = 0\n",
        "            if (exclude_citi and ((j-1) == ticker.index('Citigroup'))):\n",
        "                proportions[j-1] = 0\n",
        "        \n",
        "        if (long_only and sum(proportions) == 0):\n",
        "            proportions[:] = 1\n",
        "        proportions = proportions / sum(proportions)\n",
        "        weightDict = {}\n",
        "        weightDict['Date'] = currDate\n",
        "        for j in range(1,k):\n",
        "            weightDict[weightColumns[j]] = proportions[j-1]\n",
        "        \n",
        "        weightDf = weightDf.append(weightDict, ignore_index = True)\n",
        "        \n",
        "    return weightDf\n",
        "\n",
        "def createPortfolio2(predDf, dataset, datasetDates, long_only = False, exclude_citi = False):\n",
        "    n = predDf.shape[0]\n",
        "    weightColumns = predDf.columns[1:]\n",
        "    weightDf = pd.DataFrame(columns = weightColumns)\n",
        "    k = len(weightColumns)\n",
        "\n",
        "    for i in range(n):\n",
        "        currRow = predDf.iloc[i]\n",
        "        currDate = currRow['Date']\n",
        "        proportions = np.zeros(k-1)\n",
        "        for j in range(1,k):\n",
        "            eps = currRow[weightColumns[j]]\n",
        "            df = dataset[j-1]\n",
        "            dfDates = datasetDates[j-1]\n",
        "            for ii in range(len(dfDates)):\n",
        "                if (dfDates[ii] > currDate):\n",
        "                    foundIndex = ii\n",
        "                    break\n",
        "            if (foundIndex >= len(df)):\n",
        "                foundIndex = len(df) - 1\n",
        "            \n",
        "            analyst = df.iloc[foundIndex].get('EPS_Analyst')\n",
        "            if (analyst == None):\n",
        "                proportions[j-1] = 0\n",
        "            else:\n",
        "                proportions[j-1] = (eps - analyst) / eps\n",
        "\n",
        "            if (long_only and proportions[j-1] < 0):\n",
        "                proportions[j-1] = 0\n",
        "            if (exclude_citi and ((j-1) == ticker.index('Citigroup'))):\n",
        "                proportions[j-1] = 0\n",
        "        \n",
        "        if (long_only and sum(proportions) == 0):\n",
        "            proportions[:] = 1\n",
        "\n",
        "        proportions = proportions / sum(proportions)\n",
        "        weightDict = {}\n",
        "        weightDict['Date'] = currDate\n",
        "        for j in range(1,k):\n",
        "            weightDict[weightColumns[j]] = proportions[j-1]\n",
        "        \n",
        "        weightDf = weightDf.append(weightDict, ignore_index = True)\n",
        "        \n",
        "    return weightDf\n",
        "\n",
        "def createPortfolio3(predDf, dataset, datasetDates, long_only = False, exclude_citi = False):\n",
        "    n = predDf.shape[0]\n",
        "    weightColumns = predDf.columns[1:]\n",
        "    weightDf = pd.DataFrame(columns = weightColumns)\n",
        "    k = len(weightColumns)\n",
        "\n",
        "    for i in range(n):\n",
        "        currRow = predDf.iloc[i]\n",
        "        currDate = currRow['Date']\n",
        "        proportions = np.zeros(k-1)\n",
        "        for j in range(1,k):\n",
        "            eps = currRow[weightColumns[j]]\n",
        "            df = dataset[j-1]\n",
        "            dfDates = datasetDates[j-1]\n",
        "            for ii in range(len(dfDates)):\n",
        "                if (dfDates[ii] > currDate):\n",
        "                    foundIndex = ii\n",
        "                    break\n",
        "\n",
        "            if (foundIndex >= len(df)):\n",
        "                foundIndex = len(df) - 1\n",
        "\n",
        "            startDateToConsider = datetime(currDate.year-1, currDate.month, 1)  \n",
        "            for jj in range(len(dfDates)):\n",
        "                if (dfDates[jj] > startDateToConsider):\n",
        "                    startDateIndex = jj\n",
        "                    break\n",
        "            \n",
        "            analyst = df.iloc[foundIndex].get('EPS_Analyst')\n",
        "            if (analyst == None):\n",
        "                proportions[j-1] = 0\n",
        "            else:\n",
        "                proportions[j-1] = (eps - analyst) / analyst\n",
        "\n",
        "            if (long_only and proportions[j-1] < 0):\n",
        "                proportions[j-1] = 0\n",
        "            if (exclude_citi and ((j-1) == ticker.index('Citigroup'))):\n",
        "                proportions[j-1] = 0\n",
        "        \n",
        "        if (long_only and sum(proportions) == 0):\n",
        "            proportions[:] = 1\n",
        "        proportions = proportions / sum(proportions)\n",
        "        weightDict = {}\n",
        "        weightDict['Date'] = currDate\n",
        "        for j in range(1,k):\n",
        "            weightDict[weightColumns[j]] = proportions[j-1]\n",
        "        \n",
        "        weightDf = weightDf.append(weightDict, ignore_index = True)\n",
        "        \n",
        "    return weightDf"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mGCWSlcF9kPH",
        "colab_type": "text"
      },
      "source": [
        "Function to calculate returns and portfolio \"NAVs\" given the asset weights and asset prices."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6_YBTG8I8xDH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def calcReturns(weights, dataset, datasetDates):\n",
        "    n = weights[0].shape[0]\n",
        "    m = len(weights)\n",
        "    weightColumns = weights[0].columns[2:]\n",
        "    k = len(weightColumns)    \n",
        "\n",
        "    prices = np.zeros([n,m])\n",
        "    for j in range(m):\n",
        "        prices[0,j] = 100\n",
        "        \n",
        "    for i in range(1,n):\n",
        "        periodStartDate = weights[0].iloc[i-1].Date\n",
        "        periodEndDate = weights[0].iloc[i].Date\n",
        "        prch = np.zeros(k)\n",
        "        \n",
        "        for j in range(k):\n",
        "            # find price change of asset j between periodStartDate and periodEndDate\n",
        "            df = dataset[j]\n",
        "            dfDates = datasetDates[j]\n",
        "            for ii in range(len(dfDates)):\n",
        "                if(dfDates[ii] >= periodStartDate):\n",
        "                    startIndex = ii\n",
        "                    break\n",
        "            \n",
        "            for ii in range(startIndex, len(dfDates)):\n",
        "                if(dfDates[ii] >= periodEndDate):\n",
        "                    endIndex = ii\n",
        "                    break\n",
        "\n",
        "            if (startIndex >= len(df)):\n",
        "                startIndex = len(df) - 1\n",
        "            if (endIndex >= len(df)):\n",
        "                endIndex = len(df) - 1\n",
        "                \n",
        "            startPrice = df.iloc[startIndex]['Close']    \n",
        "            endPrice = df.iloc[endIndex]['Close']\n",
        "            prch[j] = (endPrice - startPrice) / startPrice\n",
        "        \n",
        "        for j in range(m):\n",
        "            totalPrch = 0\n",
        "            for jj in range(k):\n",
        "                totalPrch = totalPrch + weights[j].iloc[i][weightColumns[jj]] * prch[jj]\n",
        "            prevPrice = prices[i-1,j]\n",
        "            nextPrice = prevPrice * (1+totalPrch)\n",
        "            prices[i,j] = nextPrice\n",
        "    \n",
        "    return prices\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0qcEQW5v9tQw",
        "colab_type": "text"
      },
      "source": [
        "Function to calculate statistics (tracking error, alpha, information ratio) against the benchmark, given the prices of both our portfolio and benchmark index. It also plots the price movement over time."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EYiHJepk8xDN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def plotAgainstSP(prices, weight, spPrices, title = '', imageFile = ''):\n",
        "    n = prices.shape[0]\n",
        "    m = prices.shape[1]-1\n",
        "    \n",
        "    spCleanPrices = np.zeros(n)\n",
        "    spCleanPrices[0] = 100\n",
        "    \n",
        "    relatives = np.zeros([n-1,m])\n",
        "    absolutes = np.zeros([n-1,m])\n",
        "    for i in range(1,n):\n",
        "        endDate = weight.Date[i]\n",
        "        startDate = weight.Date[i-1]\n",
        "        for ii in range(len(spPrices.Date)):\n",
        "            if(spPrices.Date[ii] >= startDate):\n",
        "                startIndex = ii\n",
        "                break\n",
        "        for ii in range(startIndex,len(spPrices.Date)):\n",
        "            if(spPrices.Date[ii] >= endDate):\n",
        "                endIndex = ii\n",
        "                break\n",
        "        startSpPrice = spPrices.Price[startIndex]\n",
        "        endSpPrice = spPrices.Price[endIndex]\n",
        "        spPrch = (endSpPrice - startSpPrice) / startSpPrice\n",
        "        spCleanPrices[i] = (spPrch + 1) * spCleanPrices[i-1]\n",
        "        for j in range(m):\n",
        "            start = prices.iloc[i-1][j]\n",
        "            end = prices.iloc[i][j]\n",
        "            prch = (end - start) / start\n",
        "            relatives[i-1,j] = prch - spPrch\n",
        "            absolutes[i-1,j] = prch\n",
        "    \n",
        "    # Calculate analytics\n",
        "    trackingErrors = np.std(relatives,0)\n",
        "    alphas = np.sum(relatives,0)\n",
        "    ir = np.zeros(m)\n",
        "    \n",
        "    returns = np.sum(absolutes,0)\n",
        "    vols = np.std(absolutes,0)\n",
        "    sharpes = np.zeros(m)\n",
        "    \n",
        "    for j in range(m):\n",
        "        ir[j] = alphas[j] / trackingErrors[j]\n",
        "        sharpes[j] = returns[j] / vols[j]\n",
        "    \n",
        "    totalData = prices.copy()\n",
        "    totalData['SP_Index'] = spCleanPrices\n",
        "    totalData['Date'] = weight.Date\n",
        "    plt.figure()\n",
        "    for j in range(m):\n",
        "        plt.plot( totalData.Date, totalData[j])\n",
        "    plt.plot(totalData.Date, totalData.SP_Index)\n",
        "    plt.title(title)\n",
        "    plt.legend()\n",
        "    plt.savefig(imageFile)\n",
        "    \n",
        "    print(title)\n",
        "    print(ir)\n",
        "\n",
        "    return relatives, absolutes, trackingErrors, alphas, ir, returns, vols, sharpes, spCleanPrices\n",
        "    "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rBDeWDQK97Bw",
        "colab_type": "text"
      },
      "source": [
        "# Main Code"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pv18fBvq9-Hc",
        "colab_type": "text"
      },
      "source": [
        "These are flags to control whether we want to redo prediction, construction, and simulation steps. They should all be True, but if you have run them previously and have the excel file saved, you can set some steps to False and just reuse the previously-computed Excel file to save time. \n",
        "\n",
        "The Excel files also help us investigate why certain portfolios are doing really well / really poorly etc, by finding out abnormally high weight percentage of a single asset, etc. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b823rREn8xDQ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "doPrediction = True\n",
        "doPortfolio1 = True\n",
        "doPortfolio2 = True\n",
        "doPortfolio3 = True\n",
        "doPrices = True\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oqY-j9Ed8xDT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "# MAIN code starts here\n",
        "long_only_values = [True, False]\n",
        "exclude_citi_values = [True, False]\n",
        "financials_only = [True, False]\n",
        "combos = list(itertools.product(long_only_values, exclude_citi_values, financials_only))\n",
        "\n",
        "for c in range(len(combos)):\n",
        "    print('*** Beginning Loop ***')\n",
        "    \n",
        "    combo = combos[c]\n",
        "    long_only = combo[0]\n",
        "    exclude_citi = combo[1]\n",
        "    financials = combo[2]\n",
        "    print('Long Only: ' + str(long_only))\n",
        "    print('Exclude Citi: ' + str(exclude_citi))\n",
        "    print('Financials: ' + str(financials))\n",
        "\n",
        "\n",
        "    if (financials):    \n",
        "        ticker = ['WellsFargo', 'GoldmanSachs', 'BankOfAmerica', 'BerkshireHathaway', 'Blackrock', 'BNYMellon', 'Citigroup', 'JPMorgan', 'MorganStanley']\n",
        "    else:\n",
        "        ticker = ['WellsFargo', 'GoldmanSachs', 'BankOfAmerica', 'BerkshireHathaway', 'Blackrock', 'BNYMellon', 'Citigroup', 'JPMorgan', 'MorganStanley','Adobe', 'Apple', 'NVIDIA']\n",
        "    \n",
        "    os.chdir(\"C:\\\\Users\\\\hdharmaw\\\\OneDrive - GMO\\\\Documents\\\\4742\\\\project\\\\EventPrediction_DeepLearning\\\\FinalData\")\n",
        "\n",
        "    dataset = []\n",
        "    for i in range(len(ticker)):\n",
        "        df = pd.read_excel(ticker[i] + '_Final.xlsx')\n",
        "        print(ticker[i])\n",
        "        print('Total dataset has {} days, and {} features.'.format(df.shape[0], df.shape[1]))\n",
        "        dataset.append(df)\n",
        "    \n",
        "    datasetDates = []\n",
        "    \n",
        "    for item in dataset:\n",
        "        datasetDates.append(item['Date'])\n",
        "        del item['Date']\n",
        "\n",
        "\n",
        "    trainingStartDate = datetime(2004,1,1)\n",
        "    portfolioStartDate = datetime(2010,1,1)\n",
        "    \n",
        "    if (financials):\n",
        "        portfolioStopDate = datetime(2018,1,1)\n",
        "    else:\n",
        "        portfolioStopDate = datetime(2019,11,1)\n",
        "\n",
        "    currDate = portfolioStartDate\n",
        "\n",
        "    predColumns = ['Date']\n",
        "    for i in range(len(ticker)):\n",
        "        predColumns.append(ticker[i])\n",
        "    predDf = pd.DataFrame(columns = predColumns)\n",
        "    \n",
        "    os.chdir(\"C:\\\\Users\\\\hdharmaw\\\\OneDrive - GMO\\\\Documents\\\\4742\\\\project\\\\EventPrediction_DeepLearning\\\\PortfolioConstruction\")\n",
        "    \n",
        "    predFile = 'predictions.xlsx'\n",
        "    print('*** Beginning Prediction Phase ***')\n",
        "    if (doPrediction):\n",
        "        retrain = False\n",
        "        regressorDict = {}\n",
        "        while (currDate <= portfolioStopDate):\n",
        "            print(currDate)\n",
        "            currDateDict = {'Date': currDate}\n",
        "        \n",
        "            for i in range(len(dataset)):\n",
        "                df = dataset[i]\n",
        "                dfDates = datasetDates[i]\n",
        "    \n",
        "                (X_train, Y_train), (X_test, Y_test) = getDataUntilDate(df, dfDates, trainingStartDate, currDate)        \n",
        "                \n",
        "                if (retrain or (currDate == portfolioStartDate)):\n",
        "                    m1 = Model(X_train, Y_train, X_test, Y_test)\n",
        "                    xgbModel, regressor = m1.trainModel(verbose_flag=False)\n",
        "                    regressorDict[i] = regressor\n",
        "                else:\n",
        "                    regressor = regressorDict[i]\n",
        "                    \n",
        "                prediction = np.average(regressor.predict(X_test))\n",
        "                currDateDict[predColumns[i+1]] = prediction\n",
        "                \n",
        "            predDf = predDf.append(currDateDict, ignore_index=True)\n",
        "                \n",
        "            if (currDate.month == 12):\n",
        "                currDate = datetime(currDate.year+1, 1, 1)   \n",
        "            else:\n",
        "                currDate = datetime(currDate.year, currDate.month+1, 1)  \n",
        "        \n",
        "        predDf.to_excel(predFile)\n",
        "        \n",
        "    predDf = pd.read_excel(predFile)\n",
        "        \n",
        "    print('*** Beginning Construction Phase ***')\n",
        "    \n",
        "    combo = combos[c]\n",
        "    long_only = combo[0]\n",
        "    exclude_citi = combo[1]\n",
        "\n",
        "    weight1File = 'weights1_' + str(long_only) + '_' + str(exclude_citi) + '_' + str(financials) + '.xlsx'\n",
        "    weight2File = 'weights2_' + str(long_only) + '_' + str(exclude_citi) + '_' + str(financials) + '.xlsx'\n",
        "    weight3File = 'weights3_' + str(long_only) + '_' + str(exclude_citi) + '_' + str(financials) + '.xlsx'\n",
        "    imageFile = 'plot_' + str(long_only) + '_' + str(exclude_citi) + '_' + str(financials) + '.png'\n",
        "\n",
        "    if (doPortfolio1):\n",
        "        weight1Df = createPortfolio1(predDf, dataset, datasetDates, long_only = long_only, exclude_citi = exclude_citi)\n",
        "        weight1Df.to_excel(weight1File)\n",
        "    weight1Df = pd.read_excel(weight1File)\n",
        "        \n",
        "    if (doPortfolio2):\n",
        "        weight2Df = createPortfolio2(predDf, dataset, datasetDates, long_only = long_only, exclude_citi = exclude_citi)\n",
        "        weight2Df.to_excel(weight2File)\n",
        "    weight2Df = pd.read_excel(weight2File)\n",
        "        \n",
        "    if (doPortfolio3):\n",
        "        weight3Df = createPortfolio3(predDf, dataset, datasetDates, long_only = long_only, exclude_citi = exclude_citi)\n",
        "        weight3Df.to_excel(weight3File)\n",
        "    weight3Df = pd.read_excel(weight3File)\n",
        "    \n",
        "    weights = [weight1Df, weight2Df, weight3Df]  \n",
        "\n",
        "    print('*** Beginning Return Calc Phase ***')\n",
        "    \n",
        "    pricesFile = 'prices_' + str(long_only) + '_' + str(exclude_citi) + '_' + str(financials) + '.xlsx'\n",
        "    if (doPrices):\n",
        "        prices = calcReturns(weights, dataset, datasetDates)\n",
        "        pd.DataFrame(prices).to_excel(pricesFile)\n",
        "    prices = pd.read_excel(pricesFile)\n",
        "    \n",
        "    print('*** Beginning Analytics Phase ***')\n",
        "    \n",
        "    if (financials):\n",
        "        spPrices = pd.read_excel('SPF prices.xls')\n",
        "    else:\n",
        "        spPrices = pd.read_excel('SPX prices.xls')\n",
        "    \n",
        "    title = ('Financials ' if financials else 'All Sectors ') + ' - '\n",
        "    title = title + ('Long Only' if long_only else 'Long Short')\n",
        "    title = title + ' - '\n",
        "    title = title + ('No Citi' if exclude_citi else 'With Citi')\n",
        "    \n",
        "    relatives, absolutes, trackingErrors, alphas, ir, returns, vols, sharpes, spCleanPrices = \\\n",
        "        plotAgainstSP(prices, weights[0], spPrices, title = title, imageFile = imageFile)\n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}