hourly_test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import database as db_lib\n",
    "import pandas as pd\n",
    "import process_data_old as pr_old\n",
    "import process_data as pr\n",
    "import datetime\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#database_name = 'bence_test'  #pw: bCCnw3b   #12345678\n",
    "#set_up_connection(db, database_name, user='bence', create_tables=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pony.orm as porm\n",
    "#import database\n",
    "from datetime import date\n",
    "#import station_names\n",
    "import getpass\n",
    "import pandas as pd\n",
    "\n",
    "from pony.orm.core import ObjectNotFound, TransactionIntegrityError\n",
    "\n",
    "\n",
    "#conn_url = 'postgresql://localhost:5432'\n",
    "db = porm.Database()\n",
    "\n",
    "class Station(db.Entity):\n",
    "    stations_id   = porm.PrimaryKey(int, auto=False)\n",
    "    von_datum     = porm.Optional(date)\n",
    "    bis_datum     = porm.Optional(date)\n",
    "    stationshoehe = porm.Optional(int)\n",
    "    geobreite     = porm.Optional(float)\n",
    "    geolaenge     = porm.Optional(float)\n",
    "    stationsname  = porm.Required(str)\n",
    "    bundesland    = porm.Optional(str)\n",
    "    measurements  = porm.Set('DailyMeasurement')\n",
    "    hourlymeasurements = porm.Set('HourlyMeasurement')\n",
    "\n",
    "    @classmethod\n",
    "    def in_city(cls, city_name):\n",
    "        return cls.select(lambda s: city_name in s.stationsname)\n",
    "\n",
    "\n",
    "class DailyMeasurement(db.Entity):\n",
    "    mess_datum  = porm.Required(date)\n",
    "    stations_id = porm.Required(int)\n",
    "    station     = porm.Optional(Station)\n",
    "    qn_3        = porm.Optional(int)  # quality level of next columns\n",
    "    fx          = porm.Optional(float)\n",
    "    fm          = porm.Optional(float)\n",
    "    qn_4        = porm.Optional(int)\n",
    "    rsk         = porm.Optional(float)\n",
    "    rskf        = porm.Optional(float)\n",
    "    sdk         = porm.Optional(float)\n",
    "    shk_tag     = porm.Optional(float)\n",
    "    nm          = porm.Optional(float)\n",
    "    vpm         = porm.Optional(float)\n",
    "    pm          = porm.Optional(float)\n",
    "    tmk         = porm.Optional(float)\n",
    "    upm         = porm.Optional(float)\n",
    "    txk         = porm.Optional(float)\n",
    "    tnk         = porm.Optional(float)\n",
    "    tgk         = porm.Optional(float)\n",
    "\n",
    "    porm.PrimaryKey(mess_datum, stations_id)\n",
    "\n",
    "    #import math\n",
    "    #def before_insert(self):\n",
    "    #    for x in self._columns_:\n",
    "    #        if isinstance(getattr(self, x), float):\n",
    "    #            if math.isnan((getattr(self, x))):\n",
    "    #                setattr(self, x, None)\n",
    "    #    self.station = Station[self.stations_id]\n",
    "\n",
    "    #def after_insert(self):\n",
    "    #    self.station = Station[self.stations_id]\n",
    "\n",
    "    #def after_update(self):\n",
    "    #    self.station = Station[self.stations_id]\n",
    "\n",
    "class DailyPrediction(db.Entity):\n",
    "    id                 = porm.PrimaryKey(int, auto=True)\n",
    "    website            = porm.Required(str)\n",
    "    city               = porm.Required(str)\n",
    "    date_of_acquisition = porm.Required(str)\n",
    "    date_for_which_weather_is_predicted = porm.Required(str)\n",
    "    temperature_max    = porm.Required(float)\n",
    "    temperature_min    = porm.Required(float)\n",
    "    wind_speed         = porm.Optional(float, nullable=True)\n",
    "    humidity           = porm.Optional(float, nullable=True)\n",
    "    precipitation_per  = porm.Optional(float, nullable=True)\n",
    "    precipitation_l    = porm.Optional(float, nullable=True)\n",
    "    wind_direction     = porm.Optional(str, 3, nullable=True)\n",
    "    condition          = porm.Optional(str, nullable=True)\n",
    "    snow               = porm.Optional(float, nullable=True)\n",
    "    UVI                = porm.Optional(int, unsigned=True)\n",
    "\n",
    "\n",
    "class HourlyPrediction(db.Entity):\n",
    "    id                  = porm.PrimaryKey(int, auto=True)\n",
    "    website             = porm.Required(str)\n",
    "    city                = porm.Required(str)\n",
    "    date_of_acquisition = porm.Required(str)\n",
    "    date_for_which_weather_is_predicted = porm.Required(str)\n",
    "    temperature         = porm.Required(float)\n",
    "    wind_speed          = porm.Optional(float)\n",
    "    humidity            = porm.Optional(float)\n",
    "    precipitation_per   = porm.Optional(float)\n",
    "    precipitation_l     = porm.Optional(float)\n",
    "    wind_direction      = porm.Optional(str, 3)\n",
    "    condition           = porm.Optional(str)\n",
    "    snow                = porm.Optional(float)\n",
    "    UVI                 = porm.Optional(int, unsigned=True)\n",
    "\n",
    "\n",
    "class DailyPeriodPrediction(db.Entity):\n",
    "    id                  = porm.PrimaryKey(int, auto=True)\n",
    "    website             = porm.Required(str)\n",
    "    city                = porm.Required(str)\n",
    "    date_of_acquisition = porm.Required(str)\n",
    "    date_for_which_weather_is_predicted = porm.Required(str)\n",
    "    temperature         = porm.Required(float)\n",
    "    wind_speed          = porm.Optional(float)\n",
    "    precipitation_per   = porm.Optional(float)\n",
    "    precipitation_l     = porm.Optional(float)\n",
    "    wind_direction      = porm.Optional(str, 3)\n",
    "    condition           = porm.Optional(str)\n",
    "\n",
    "class HourlyMeasurement(db.Entity):\n",
    "    station      = porm.Optional(Station)\n",
    "    mess_datum   = porm.Required(datetime.datetime)#date / date.datetime\n",
    "    qn_9         = porm.Optional(float, nullable=True)\n",
    "    rf_tu        = porm.Optional(float, nullable=True)\n",
    "    stations_id  = porm.Required(int, nullable=True)\n",
    "    tt_tu        = porm.Optional(float, nullable=True)\n",
    "    v_n          = porm.Optional(float, nullable=True)\n",
    "    qn_8_x       = porm.Optional(float, nullable=True)\n",
    "    v_n_i        = porm.Optional(str, nullable=True)\n",
    "    v_s1_cs      = porm.Optional(float, nullable=True)\n",
    "    v_s1_csa     = porm.Optional(str, nullable=True)\n",
    "    v_s1_hhs     = porm.Optional(float, nullable=True)\n",
    "    v_s1_ns      = porm.Optional(float, nullable=True)\n",
    "    v_s2_cs      = porm.Optional(float, nullable=True)\n",
    "    v_s2_csa     = porm.Optional(str, nullable=True)\n",
    "    v_s2_hhs     = porm.Optional(float, nullable=True)\n",
    "    v_s2_ns      = porm.Optional(float, nullable=True)\n",
    "    v_s3_cs      = porm.Optional(float, nullable=True)\n",
    "    v_s3_csa     = porm.Optional(str, nullable=True)\n",
    "    v_s3_hhs     = porm.Optional(float, nullable=True)\n",
    "    v_s3_ns      = porm.Optional(float, nullable=True)\n",
    "    v_s4_cs      = porm.Optional(float, nullable=True)\n",
    "    v_s4_csa     = porm.Optional(str, nullable=True)\n",
    "    v_s4_hhs     = porm.Optional(float, nullable=True)\n",
    "    v_s4_ns      = porm.Optional(float, nullable=True)\n",
    "    r1           = porm.Optional(float, nullable=True)\n",
    "    q_n_8_y      = porm.Optional(float, nullable=True)\n",
    "    rs_ind       = porm.Optional(float, nullable=True)\n",
    "    wrtr         = porm.Optional(float, nullable=True)\n",
    "    p            = porm.Optional(float, nullable=True)\n",
    "    p0           = porm.Optional(float, nullable=True)\n",
    "    qn_2         = porm.Optional(float, nullable=True)\n",
    "    v_te002      = porm.Optional(float, nullable=True)\n",
    "    v_te005      = porm.Optional(float, nullable=True)\n",
    "    v_te010      = porm.Optional(float, nullable=True)\n",
    "    v_te020      = porm.Optional(float, nullable=True)\n",
    "    v_te050      = porm.Optional(float, nullable=True)\n",
    "    v_te100      = porm.Optional(float, nullable=True)\n",
    "    qn_7         = porm.Optional(float, nullable=True)\n",
    "    sd_so        = porm.Optional(float, nullable=True)\n",
    "    qn_8_y       = porm.Optional(float, nullable=True)\n",
    "    v_vv         = porm.Optional(float, nullable=True)\n",
    "    v_vv_i       = porm.Optional(str, nullable=True)\n",
    "    d            = porm.Optional(float, nullable=True)\n",
    "    f            = porm.Optional(float, nullable=True)\n",
    "    qn_3         = porm.Optional(float, nullable=True)\n",
    "\n",
    "    porm.PrimaryKey(mess_datum, stations_id)\n",
    "    #db.generate_mapping(create_tables=True)\n",
    "@porm.db_session\n",
    "def _insert_without_pandas(df, table_name):\n",
    "    table_obj = db.entities[table_name]\n",
    "    pk = table_obj._pk_columns_\n",
    "\n",
    "    if df.index.name is None:\n",
    "        df_q = df.set_index(pk)\n",
    "    else:\n",
    "        df_q = df.copy()\n",
    "\n",
    "    for i in df_q.index:\n",
    "        try:\n",
    "            table_obj[i]\n",
    "        except ObjectNotFound:\n",
    "            try:\n",
    "                table_obj(**{**dict(zip(pk, i)),\n",
    "                             **df_q.loc[i].to_dict()})\n",
    "            except TypeError:\n",
    "                table_obj(**{**{pk : i},\n",
    "                             **df_q.loc[i].to_dict()})\n",
    "\n",
    "\n",
    "@porm.db_session\n",
    "def _insert_with_pandas(df, table_name, auto_id=False, overwrite=False):\n",
    "    indices_to_keep = []\n",
    "    rows_to_delete = []\n",
    "    table_obj = db.entities[table_name]\n",
    "\n",
    "    if df.index.name is None and not auto_id:\n",
    "        df_q = df.set_index(table_obj._pk_columns_)\n",
    "    else:\n",
    "        df_q = df.copy()\n",
    "\n",
    "    try:\n",
    "        df_q.to_sql(table_name.lower(), conn_url, if_exists='append', index=not auto_id)\n",
    "    except:\n",
    "        for i in df_q.index:\n",
    "            try:\n",
    "                row = table_obj[i]\n",
    "\n",
    "                if overwrite:\n",
    "                    rows_to_delete.append(row)\n",
    "                    indices_to_keep.append(i)\n",
    "\n",
    "            except ObjectNotFound:\n",
    "                indices_to_keep.append(i)\n",
    "\n",
    "            except:\n",
    "                print(i)\n",
    "\n",
    "        table_obj.select(lambda x: x in rows_to_delete).delete(bulk = True)\n",
    "        porm.commit()\n",
    "        print('starting insert')\n",
    "        df_to_insert = df_q.loc[indices_to_keep]\n",
    "        df_to_insert.to_sql(table_name.lower(), conn_url, if_exists='append', index=not auto_id)\n",
    "\n",
    "\n",
    "@porm.db_session\n",
    "def insert_into_table(df, table_name, use_pandas=True, auto_id=False, overwrite=False):\n",
    "    if use_pandas:\n",
    "        _insert_with_pandas(df, table_name, auto_id, overwrite)\n",
    "    else:\n",
    "        _insert_without_pandas(df, table_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n"
     ]
    }
   ],
   "source": [
    "dwd_path = '/local/data_dwd/'\n",
    "station_number = 3\n",
    "x = pr.process_data(dwd_path, station_number, 'hourly', 'all')\n",
    "x['mess_datum']=x['mess_datum'].apply(int).apply(str).apply(lambda s: datetime.datetime.strptime(s, \"%Y%m%d%H\")).apply(lambda t: datetime.datetime(t.year, t.month, t.day, t.hour))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "db.bind(provider='postgres', user='bence', \\\n",
    "        password='12345678', host='127.0.0.1', database='dwd_hourly')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "db.generate_mapping(create_tables=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#x_ = x.head(10000)\n",
    "x_ = x[np.isfinite(x['stations_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_ = x_[200000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#x_['stations_id'] =10000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 46)"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:10:24.648258\n"
     ]
    }
   ],
   "source": [
    "table_name = 'HourlyMeasurement'\n",
    "conn_url = 'postgresql://{}:{}@{}:5432/{}'.format('bence', \\\n",
    "                                    '12345678', '127.0.0.1', 'dwd_hourly')\n",
    "auto_id = False\n",
    "n1 = datetime.datetime.now()\n",
    "x_.to_sql(table_name.lower(), conn_url, index=False, \\\n",
    "          if_exists='append')#, index=not auto_id, if_exists='append'\n",
    "n2 = datetime.datetime.now()\n",
    "print(n2-n1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR:root:Internal Python error in the inspect module.\n",
      "Below is the traceback from this internal error.\n",
      "\n",
      "\n",
      "KeyboardInterrupt\n",
      "\n"
     ]
    }
   ],
   "source": [
    "try:\n",
    "    insert_into_table(x, 'HourlyMeasurement', \\\n",
    "                      overwrite=False, auto_id=False)\n",
    "except Exception as e:\n",
    "    print('duplicated insert')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#get all station ids\n",
    "stations = porm.select(s for s in Station)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1176"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "porm.count(station for station in stations)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(513928, 46)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:50000].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(563928, 46)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#x_.ix[5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "13.125424200000001"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(17.1 * x.shape[0] * 1176) /(60*60*24*10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16.071948000000003"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape[0]*17.1/(60*10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "563928"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "stations_pd = pd.read_sql_query(stations.get_sql(), conn_url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "stations_arr = stations_pd['stations_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1176,)"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stations_arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "station number:1\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:44\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n",
      "station number:52\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n",
      "error by parsing process_data\n",
      "station number:61\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:70\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:71\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:72\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n",
      "error by parsing process_data\n",
      "station number:73\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:78\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:91\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:93\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:98\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "error by parsing process_data\n",
      "station number:102\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no solar\n",
      "station number:106\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:116\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "error by parsing process_data\n",
      "station number:125\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "no solar\n",
      "station number:129\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "error by parsing process_data\n",
      "station number:131\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:132\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:142\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:150\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:151\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:154\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no solar\n",
      "station number:161\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "no solar\n",
      "0:00:51.114609\n",
      "station number:164\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "hist exists\n",
      "rec exists\n",
      "no solar\n"
     ]
    }
   ],
   "source": [
    "for station_number in stations_arr:\n",
    "    if station_number !=3:\n",
    "        print('station number:'+str(station_number))\n",
    "        try:\n",
    "            x = pr.process_data(dwd_path, station_number, 'hourly', 'all')\n",
    "        \n",
    "            if x.shape[1] == 46:   #there are entries with shape 43....\n",
    "                x_ = x[np.isfinite(x['stations_id'])]\n",
    "                if isinstance(x_[\"mess_datum\"][0], datetime.datetime) == False:\n",
    "                    x_['mess_datum']=x_['mess_datum'].apply(int).apply(str).apply(lambda s: datetime.datetime.strptime(s, \"%Y%m%d%H\")).apply(lambda t: datetime.datetime(t.year, t.month, t.day, t.hour))\n",
    "                    print('mess_datum has shape float and not datetime')\n",
    "\n",
    "                conn_url = 'postgresql://{}:{}@{}:5432/{}'.format('bence', \\\n",
    "                                                    '12345678', '127.0.0.1', 'dwd_hourly')\n",
    "\n",
    "                n1 = datetime.datetime.now()\n",
    "                x_.to_sql(table_name.lower(), conn_url, index=False, \\\n",
    "                          if_exists='append')#, index=not auto_id, if_exists='append'\n",
    "                n2 = datetime.datetime.now()\n",
    "                print(n2-n1)\n",
    "            else:\n",
    "                print('second x shape is not 46')\n",
    "        except Exception as e:\n",
    "            print('error by parsing process_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n"
     ]
    }
   ],
   "source": [
    "x = pr.process_data(dwd_path, 3, 'hourly', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "43"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x_ = x[np.isfinite(x['stations_id'])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(534716, 46)"
      ]
     },
     "execution_count": 195,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "int() argument must be a string, a bytes-like object or a number, not 'Timestamp'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-153-55cd4681eb0b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mess_datum'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mx_\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'mess_datum'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrptime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ms\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"%Y%m%d%H\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mdatetime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdatetime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0myear\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmonth\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mday\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhour\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/series.py\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, func, convert_dtype, args, **kwds)\u001b[0m\n\u001b[1;32m   2292\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2293\u001b[0m                 \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masobject\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2294\u001b[0;31m                 \u001b[0mmapped\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlib\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_infer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconvert_dtype\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2295\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2296\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmapped\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mSeries\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/src/inference.pyx\u001b[0m in \u001b[0;36mpandas.lib.map_infer (pandas/lib.c:66124)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: int() argument must be a string, a bytes-like object or a number, not 'Timestamp'"
     ]
    }
   ],
   "source": [
    "x_['mess_datum']=x_['mess_datum'].apply(int).apply(str).apply(lambda s: datetime.datetime.strptime(s, \"%Y%m%d%H\")).apply(lambda t: datetime.datetime(t.year, t.month, t.day, t.hour))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['stations_id', 'mess_datum', 'qn_9', 'tt_tu', 'rf_tu', 'v_n', 'qn_8_x',\n",
       "       'v_n_i', 'v_s1_cs', 'v_s1_csa', 'v_s1_hhs', 'v_s1_ns', 'v_s2_cs',\n",
       "       'v_s2_csa', 'v_s2_hhs', 'v_s2_ns', 'v_s3_cs', 'v_s3_csa', 'v_s3_hhs',\n",
       "       'v_s3_ns', 'v_s4_cs', 'v_s4_csa', 'v_s4_hhs', 'v_s4_ns', 'qn_8_y', 'r1',\n",
       "       'rs_ind', 'wrtr', 'qn_2', 'v_te002', 'v_te005', 'v_te010', 'v_te020',\n",
       "       'v_te050', 'v_te100', 'qn_7', 'sd_so', 'qn_8', 'v_vv', 'v_vv_i', 'd',\n",
       "       'f', 'qn_3'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no hist\n",
      "no rec\n",
      "hist exists\n",
      "no rec\n",
      "no solar\n"
     ]
    },
    {
     "ename": "KeyError",
     "evalue": "'stations_id'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2133\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2134\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2135\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4433)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4279)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/src/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13742)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/src/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13696)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'stations_id'",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-204-238b574332db>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdwd_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m52\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'hourly'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'all'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/home/timon/Documents/server_hourly/dwd_2018/process_data.py\u001b[0m in \u001b[0;36mprocess_data\u001b[0;34m(userpath, stationnumber, time_interval, folder)\u001b[0m\n\u001b[1;32m     15\u001b[0m         \u001b[0mclean_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m         \u001b[0mclean_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclean_merged\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmerged\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtime_interval\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mclean_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/timon/Documents/server_hourly/dwd_2018/process_data.py\u001b[0m in \u001b[0;36mclean_merged\u001b[0;34m(merged, time_interval)\u001b[0m\n\u001b[1;32m    127\u001b[0m     merged_clean = merged_clean.drop_duplicates(subset = ['stations_id',\n\u001b[1;32m    128\u001b[0m                                                           'mess_datum'],\n\u001b[0;32m--> 129\u001b[0;31m                                                 keep='first')\n\u001b[0m\u001b[1;32m    130\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mtime_interval\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;34m'hourly'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/util/decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_deprecate_kwarg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mdrop_duplicates\u001b[0;34m(self, subset, keep, inplace)\u001b[0m\n\u001b[1;32m   3136\u001b[0m         \u001b[0mdeduplicated\u001b[0m \u001b[0;34m:\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3137\u001b[0m         \"\"\"\n\u001b[0;32m-> 3138\u001b[0;31m         \u001b[0mduplicated\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mduplicated\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msubset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkeep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3139\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3140\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0minplace\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/util/decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     89\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m                     \u001b[0mkwargs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mnew_arg_name\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_arg_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 91\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     92\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     93\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_deprecate_kwarg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36mduplicated\u001b[0;34m(self, subset, keep)\u001b[0m\n\u001b[1;32m   3186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3187\u001b[0m         \u001b[0mvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msubset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3188\u001b[0;31m         \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3190\u001b[0m         \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_group_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msort\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mxnull\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m   3185\u001b[0m             \u001b[0msubset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msubset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3186\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3187\u001b[0;31m         \u001b[0mvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mcol\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcol\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msubset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3188\u001b[0m         \u001b[0mlabels\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mmap\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2057\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2058\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2059\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2060\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2061\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m_getitem_column\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   2064\u001b[0m         \u001b[0;31m# get column\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2065\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2066\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2067\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2068\u001b[0m         \u001b[0;31m# duplicate columns & possible reduce dimensionality\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/generic.py\u001b[0m in \u001b[0;36m_get_item_cache\u001b[0;34m(self, item)\u001b[0m\n\u001b[1;32m   1384\u001b[0m         \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1385\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1386\u001b[0;31m             \u001b[0mvalues\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1387\u001b[0m             \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1388\u001b[0m             \u001b[0mcache\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mres\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/core/internals.py\u001b[0m in \u001b[0;36mget\u001b[0;34m(self, item, fastpath)\u001b[0m\n\u001b[1;32m   3541\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3542\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3543\u001b[0;31m                 \u001b[0mloc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3544\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3545\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3/dist-packages/pandas/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   2134\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2135\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2136\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_maybe_cast_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2137\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2138\u001b[0m         \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmethod\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmethod\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtolerance\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtolerance\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4433)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/index.pyx\u001b[0m in \u001b[0;36mpandas.index.IndexEngine.get_loc (pandas/index.c:4279)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/src/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13742)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/src/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas.hashtable.PyObjectHashTable.get_item (pandas/hashtable.c:13696)\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'stations_id'"
     ]
    }
   ],
   "source": [
    "x = pr.process_data(dwd_path, 52, 'hourly', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 184,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "isinstance(x_[\"mess_datum\"][0], datetime.datetime)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}