process_data.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape text from FOMC statements and Calculate Yield Changes\n",
    "\n",
    "### 1. Import and mess with statements"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import PyPDF2\n",
    "import os\n",
    "import pandas as pd\n",
    "\n",
    "#https://fraser.stlouisfed.org/title/677#576435"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'FOMC_statements/19940204statement.pdf'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-cce0773ef9a9>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpdfFileObj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'FOMC_statements/19940204statement.pdf'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpdfReader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mPyPDF2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPdfFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpdfFileObj\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mpdfReader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnumPages\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'FOMC_statements/19940204statement.pdf'"
     ]
    }
   ],
   "source": [
    "pdfFileObj = open('FOMC_statements/19940204statement.pdf', 'rb')\n",
    "pdfReader = PyPDF2.PdfFileReader(pdfFileObj)\n",
    "pdfReader.numPages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Release Date: February 4, 1994\\n For immediate releaseChairman Alan Greenspan announced today that the Federal Open Market Committee decided toincrease slightly the degree of pressure on reserve positions. The action is expected to be associatedwith a small increase in short-term money market interest rates.\\nThe decision was taken to move toward a less accommodative stance in monetary policy in order tosustain and enhance the economic expansion.Chairman Greenspan decided to announce this action immediately so as to avoid anymisunderstanding of the Committee's purposes, given the fact that this is the first firming of reservemarket conditions by the Committee since early 1989.Home | Press releases\\nAccessibility\\n | Contact UsLast update: April 20, 2007\\n\""
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pageObj = pdfReader.getPage(0)\n",
    "pageObj.extractText()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "statements = []\n",
    "dates = []\n",
    "\n",
    "# loop through files\n",
    "for filename in os.listdir('FOMC_statements'):\n",
    "    if filename.endswith(\".pdf\"):\n",
    "        \n",
    "        # add date to list\n",
    "        dt = filename.replace('statement.pdf', '')\n",
    "        dates.append(dt)\n",
    "        \n",
    "        # read specific file\n",
    "        pdfFileObj = open('FOMC_statements/' + filename, 'rb')\n",
    "        pdfReader = PyPDF2.PdfFileReader(pdfFileObj)\n",
    "        \n",
    "        p = ''\n",
    "        # loop through pages\n",
    "        for i in range(pdfReader.numPages):\n",
    "            pageObj = pdfReader.getPage(i)\n",
    "            p += pageObj.extractText()\n",
    "            \n",
    "        statements.append(p)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = pd.DataFrame({'date': dates, 'text': statements})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.head()\n",
    "data.to_csv(\"fomc_statements.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Note\n",
    "\n",
    "The code above was used to scrape the pdfs, but the text was still very messy, so I manually cleaned a bunch of them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reload cleaned statements\n",
    "data = pd.read_csv(\"fomc_statements_clean.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1994-02-04</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1994-03-22</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1994-04-18</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1994-05-17</td>\n",
       "      <td>The Federal Reserve today announced two action...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994-08-16</td>\n",
       "      <td>The Federal Reserve announced today the follow...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date                                               text\n",
       "0 1994-02-04  Chairman Alan Greenspan announced today that t...\n",
       "1 1994-03-22  Chairman Alan Greenspan announced today that t...\n",
       "2 1994-04-18  Chairman Alan Greenspan announced today that t...\n",
       "3 1994-05-17  The Federal Reserve today announced two action...\n",
       "4 1994-08-16  The Federal Reserve announced today the follow..."
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Clean dates\n",
    "data.date = pd.to_datetime(data.date, format='%Y%m%d')\n",
    "data = data.sort_values('date')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Import and mess with treasury data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>1 MO</th>\n",
       "      <th>3 MO</th>\n",
       "      <th>6 MO</th>\n",
       "      <th>1 YR</th>\n",
       "      <th>2 YR</th>\n",
       "      <th>3 YR</th>\n",
       "      <th>5 YR</th>\n",
       "      <th>7 YR</th>\n",
       "      <th>10 YR</th>\n",
       "      <th>20 YR</th>\n",
       "      <th>30 YR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>7121</th>\n",
       "      <td>1990-01-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.83</td>\n",
       "      <td>7.89</td>\n",
       "      <td>7.81</td>\n",
       "      <td>7.87</td>\n",
       "      <td>7.90</td>\n",
       "      <td>7.87</td>\n",
       "      <td>7.98</td>\n",
       "      <td>7.94</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7120</th>\n",
       "      <td>1990-01-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.89</td>\n",
       "      <td>7.94</td>\n",
       "      <td>7.85</td>\n",
       "      <td>7.94</td>\n",
       "      <td>7.96</td>\n",
       "      <td>7.92</td>\n",
       "      <td>8.04</td>\n",
       "      <td>7.99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7119</th>\n",
       "      <td>1990-01-04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.84</td>\n",
       "      <td>7.90</td>\n",
       "      <td>7.82</td>\n",
       "      <td>7.92</td>\n",
       "      <td>7.93</td>\n",
       "      <td>7.91</td>\n",
       "      <td>8.02</td>\n",
       "      <td>7.98</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7118</th>\n",
       "      <td>1990-01-05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.79</td>\n",
       "      <td>7.85</td>\n",
       "      <td>7.79</td>\n",
       "      <td>7.90</td>\n",
       "      <td>7.94</td>\n",
       "      <td>7.92</td>\n",
       "      <td>8.03</td>\n",
       "      <td>7.99</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7117</th>\n",
       "      <td>1990-01-08</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.79</td>\n",
       "      <td>7.88</td>\n",
       "      <td>7.81</td>\n",
       "      <td>7.90</td>\n",
       "      <td>7.95</td>\n",
       "      <td>7.92</td>\n",
       "      <td>8.05</td>\n",
       "      <td>8.02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Date  1 MO  3 MO  6 MO  1 YR  2 YR  3 YR  5 YR  7 YR  10 YR  20 YR  \\\n",
       "7121 1990-01-02   NaN  7.83  7.89  7.81  7.87  7.90  7.87  7.98   7.94    NaN   \n",
       "7120 1990-01-03   NaN  7.89  7.94  7.85  7.94  7.96  7.92  8.04   7.99    NaN   \n",
       "7119 1990-01-04   NaN  7.84  7.90  7.82  7.92  7.93  7.91  8.02   7.98    NaN   \n",
       "7118 1990-01-05   NaN  7.79  7.85  7.79  7.90  7.94  7.92  8.03   7.99    NaN   \n",
       "7117 1990-01-08   NaN  7.79  7.88  7.81  7.90  7.95  7.92  8.05   8.02    NaN   \n",
       "\n",
       "      30 YR  \n",
       "7121   8.00  \n",
       "7120   8.04  \n",
       "7119   8.04  \n",
       "7118   8.06  \n",
       "7117   8.09  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "treas = pd.read_csv('USTREASURY-YIELD.csv')\n",
    "treas.Date = pd.to_datetime(treas.Date, format='%Y-%m-%d')\n",
    "treas = treas.sort_values('Date')\n",
    "treas.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1 MO</th>\n",
       "      <th>3 MO</th>\n",
       "      <th>6 MO</th>\n",
       "      <th>1 YR</th>\n",
       "      <th>2 YR</th>\n",
       "      <th>3 YR</th>\n",
       "      <th>5 YR</th>\n",
       "      <th>7 YR</th>\n",
       "      <th>10 YR</th>\n",
       "      <th>20 YR</th>\n",
       "      <th>30 YR</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1990-01-02</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990-01-03</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.04</td>\n",
       "      <td>0.07</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.06</td>\n",
       "      <td>0.05</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.04</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990-01-04</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>-0.04</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990-01-05</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>-0.03</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1990-01-08</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.01</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            1 MO  3 MO  6 MO  1 YR  2 YR  3 YR  5 YR  7 YR  10 YR  20 YR  \\\n",
       "date                                                                       \n",
       "1990-01-02   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN    NaN    NaN   \n",
       "1990-01-03   NaN  0.06  0.05  0.04  0.07  0.06  0.05  0.06   0.05    NaN   \n",
       "1990-01-04   NaN -0.05 -0.04 -0.03 -0.02 -0.03 -0.01 -0.02  -0.01    NaN   \n",
       "1990-01-05   NaN -0.05 -0.05 -0.03 -0.02  0.01  0.01  0.01   0.01    NaN   \n",
       "1990-01-08   NaN  0.00  0.03  0.02  0.00  0.01  0.00  0.02   0.03    NaN   \n",
       "\n",
       "            30 YR  \n",
       "date               \n",
       "1990-01-02    NaN  \n",
       "1990-01-03   0.04  \n",
       "1990-01-04   0.00  \n",
       "1990-01-05   0.02  \n",
       "1990-01-08   0.03  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "h = treas.loc[:, treas.columns != 'Date'].diff()\n",
    "h.insert(loc=0, column='date', value=treas['Date'])\n",
    "h = h.set_index('date')\n",
    "h.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/victoriaeastman/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py:3: FutureWarning: \n",
      "Passing list-likes to .loc or [] with any missing label will raise\n",
      "KeyError in the future, you can use .reindex() as an alternative.\n",
      "\n",
      "See the documentation here:\n",
      "https://pandas.pydata.org/pandas-docs/stable/indexing.html#deprecate-loc-reindex-listlike\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>1 MO</th>\n",
       "      <th>3 MO</th>\n",
       "      <th>6 MO</th>\n",
       "      <th>1 YR</th>\n",
       "      <th>2 YR</th>\n",
       "      <th>3 YR</th>\n",
       "      <th>5 YR</th>\n",
       "      <th>7 YR</th>\n",
       "      <th>10 YR</th>\n",
       "      <th>20 YR</th>\n",
       "      <th>30 YR</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1994-02-04</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.10</td>\n",
       "      <td>0.12</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.15</td>\n",
       "      <td>0.14</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994-03-22</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.08</td>\n",
       "      <td>-0.08</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>-0.09</td>\n",
       "      <td>-0.09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994-04-18</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.16</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.18</td>\n",
       "      <td>0.19</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.20</td>\n",
       "      <td>0.17</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994-05-17</th>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.01</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>-0.15</td>\n",
       "      <td>-0.17</td>\n",
       "      <td>-0.19</td>\n",
       "      <td>-0.23</td>\n",
       "      <td>-0.21</td>\n",
       "      <td>-0.19</td>\n",
       "      <td>-0.19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1994-08-16</th>\n",
       "      <td>NaN</td>\n",
       "      <td>0.02</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>-0.02</td>\n",
       "      <td>-0.05</td>\n",
       "      <td>-0.06</td>\n",
       "      <td>-0.10</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>-0.11</td>\n",
       "      <td>-0.12</td>\n",
       "      <td>-0.12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            1 MO  3 MO  6 MO  1 YR  2 YR  3 YR  5 YR  7 YR  10 YR  20 YR  \\\n",
       "date                                                                       \n",
       "1994-02-04   NaN  0.10  0.12  0.11  0.14  0.15  0.15  0.14   0.13   0.11   \n",
       "1994-03-22   NaN -0.10 -0.08 -0.08 -0.06 -0.11 -0.11 -0.12  -0.11  -0.09   \n",
       "1994-04-18   NaN  0.16  0.19  0.18  0.19  0.20  0.20  0.20   0.17   0.13   \n",
       "1994-05-17   NaN -0.01 -0.10 -0.12 -0.15 -0.17 -0.19 -0.23  -0.21  -0.19   \n",
       "1994-08-16   NaN  0.02 -0.02 -0.02 -0.05 -0.06 -0.10 -0.12  -0.11  -0.12   \n",
       "\n",
       "            30 YR  \n",
       "date               \n",
       "1994-02-04   0.06  \n",
       "1994-03-22  -0.09  \n",
       "1994-04-18   0.12  \n",
       "1994-05-17  -0.19  \n",
       "1994-08-16  -0.12  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Subset only FOMC statement dates\n",
    "dts = list(data['date'])\n",
    "g = h.loc[dts]\n",
    "g.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Write data to csv\n",
    "g.to_csv('treas_changes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = g['10 YR']\n",
    "labels = [1 if i >= 0 else 0 for i in labels]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>date</th>\n",
       "      <th>text</th>\n",
       "      <th>labels</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1994-02-04</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1994-03-22</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1994-04-18</td>\n",
       "      <td>Chairman Alan Greenspan announced today that t...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1994-05-17</td>\n",
       "      <td>The Federal Reserve today announced two action...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1994-08-16</td>\n",
       "      <td>The Federal Reserve announced today the follow...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        date                                               text  labels\n",
       "0 1994-02-04  Chairman Alan Greenspan announced today that t...       1\n",
       "1 1994-03-22  Chairman Alan Greenspan announced today that t...       0\n",
       "2 1994-04-18  Chairman Alan Greenspan announced today that t...       1\n",
       "3 1994-05-17  The Federal Reserve today announced two action...       0\n",
       "4 1994-08-16  The Federal Reserve announced today the follow...       0"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['labels']= labels\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_csv('statements_with_labels.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}