storm_drains.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gp\n",
    "from geopandas import GeoSeries\n",
    "import numpy as np\n",
    "from pyproj import Proj, transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "county_df = pd.read_csv('county_storm_drains.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID</th>\n",
       "      <th>MUNI_ID</th>\n",
       "      <th>MMSD_OWNER</th>\n",
       "      <th>MUNI_FEATURE_TYPE</th>\n",
       "      <th>RIM_ELEV</th>\n",
       "      <th>SIZE</th>\n",
       "      <th>STATUS</th>\n",
       "      <th>MUNI_OWNER</th>\n",
       "      <th>MMSD_FEATURE</th>\n",
       "      <th>YEAR_RECEIVED</th>\n",
       "      <th>NOTES</th>\n",
       "      <th>X_COORD</th>\n",
       "      <th>Y_COORD</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0.0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511625.574</td>\n",
       "      <td>434654.0883</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   OBJECTID MUNI_ID MMSD_OWNER MUNI_FEATURE_TYPE  RIM_ELEV SIZE  STATUS  \\\n",
       "0         1                 BD                CB       0.0       Active   \n",
       "\n",
       "  MUNI_OWNER    MMSD_FEATURE  YEAR_RECEIVED NOTES      X_COORD      Y_COORD  \n",
       "0             StormStructure           2015        2511625.574  434654.0883  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "county_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-88.08203123832631, 43.17641751919515\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "county_gpdf = gp.read_file('county_storm_drains.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2511591.298'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "county_gpdf.X_COORD.iloc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from shapely.geometry import Point\n",
    "def create_point(row):\n",
    "    #print(row)\n",
    "    return Point(float(row['X_COORD']),float(row['Y_COORD']))\n",
    "\n",
    "county_gpdf.geometry = GeoSeries(county_gpdf.apply(lambda x: create_point(x), axis=1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OBJECTID</th>\n",
       "      <th>MUNI_ID</th>\n",
       "      <th>MMSD_OWNER</th>\n",
       "      <th>MUNI_FEATURE_TYPE</th>\n",
       "      <th>RIM_ELEV</th>\n",
       "      <th>SIZE</th>\n",
       "      <th>STATUS</th>\n",
       "      <th>MUNI_OWNER</th>\n",
       "      <th>MMSD_FEATURE</th>\n",
       "      <th>YEAR_RECEIVED</th>\n",
       "      <th>NOTES</th>\n",
       "      <th>X_COORD</th>\n",
       "      <th>Y_COORD</th>\n",
       "      <th>geometry</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511625.574</td>\n",
       "      <td>434654.0883</td>\n",
       "      <td>POINT (2511625.574 434654.0883)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511591.298</td>\n",
       "      <td>434641.8147</td>\n",
       "      <td>POINT (2511591.298 434641.8147)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511577.575</td>\n",
       "      <td>434653.7241</td>\n",
       "      <td>POINT (2511577.575 434653.7241)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511563.075</td>\n",
       "      <td>434677.4672</td>\n",
       "      <td>POINT (2511563.075 434677.4672)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td></td>\n",
       "      <td>BD</td>\n",
       "      <td>CB</td>\n",
       "      <td>0</td>\n",
       "      <td></td>\n",
       "      <td>Active</td>\n",
       "      <td></td>\n",
       "      <td>StormStructure</td>\n",
       "      <td>2015</td>\n",
       "      <td></td>\n",
       "      <td>2511814.72</td>\n",
       "      <td>434655.6529</td>\n",
       "      <td>POINT (2511814.72 434655.6529)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  OBJECTID MUNI_ID MMSD_OWNER MUNI_FEATURE_TYPE RIM_ELEV SIZE  STATUS  \\\n",
       "0        1                 BD                CB        0       Active   \n",
       "1        2                 BD                CB        0       Active   \n",
       "2        3                 BD                CB        0       Active   \n",
       "3        4                 BD                CB        0       Active   \n",
       "4        5                 BD                CB        0       Active   \n",
       "\n",
       "  MUNI_OWNER    MMSD_FEATURE YEAR_RECEIVED NOTES      X_COORD      Y_COORD  \\\n",
       "0             StormStructure          2015        2511625.574  434654.0883   \n",
       "1             StormStructure          2015        2511591.298  434641.8147   \n",
       "2             StormStructure          2015        2511577.575  434653.7241   \n",
       "3             StormStructure          2015        2511563.075  434677.4672   \n",
       "4             StormStructure          2015         2511814.72  434655.6529   \n",
       "\n",
       "                          geometry  \n",
       "0  POINT (2511625.574 434654.0883)  \n",
       "1  POINT (2511591.298 434641.8147)  \n",
       "2  POINT (2511577.575 434653.7241)  \n",
       "3  POINT (2511563.075 434677.4672)  \n",
       "4   POINT (2511814.72 434655.6529)  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "county_gpdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#https://gis.stackexchange.com/questions/224526/convert-shapefile-from-projected-coordinates-using-python\n",
    "#WINNNER\n",
    "#gp.GeoDataFrame(pts,crs={'init' :'epsg:32054'}).rename(columns={0:'geometry'}).to_crs(epsg=4326)#.to_crs(crs={'init' :'epsg:4326'})\n",
    "\n",
    "county_gpdf.geometry = GeoSeries(county_gpdf.apply(lambda x: create_point(x), axis=1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "county_gpdf.crs = {'init' :'epsg:32054'}\n",
    "\n",
    "#mp?\n",
    "county_gpdf.lat_lon = county_gpdf.to_crs(epsg=4326)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "county_gpdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#gp.GeoDataFrame(pts).rename(columns={0:'geometry'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def paralellize_func(df, func, num_processes, num_splits):\n",
    "    \n",
    "    df_split = np.array_split(df, num_splits)\n",
    "    pool = Pool(num_processes)\n",
    "    \n",
    "    #mess with chunksize dont really understand\n",
    "    out = pool.map_async(func, df_split)\n",
    "    pool.close()\n",
    "    pool.join()\n",
    "   \n",
    "    print('test1')\n",
    "    \n",
    "    return out.get()\n",
    "\n",
    "\n",
    "\n",
    "def mp_apply_get_lat_lon(df):\n",
    "    print('test')\n",
    "    return df.apply(get_lat_lon, axis=1)\n",
    "\n",
    "def get_lat_lon(x):\n",
    "    inProj = Proj(init='epsg:32054', preserve_units = True)\n",
    "    outProj = Proj(init='epsg:4326')\n",
    "    x1,y1 = x['X_COORD'],x['Y_COORD']\n",
    "    x2,y2 = transform(inProj,outProj,x1,y1)\n",
    "    return {'lat_lon': Point(x2,y2), 'X_COORD':x1, 'Y_COORD':y1}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test\n",
      "test1\n",
      "1352.4573140144348\n"
     ]
    }
   ],
   "source": [
    "#can I make this work some way??\n",
    "import warnings\n",
    "import time\n",
    "#warnings.filterwarnings('ignore')\n",
    "from multiprocessing import Pool\n",
    "#links_dict = {}\n",
    "#out2= []\n",
    "start = time.time()\n",
    "#df_split = np.array_split(meta_df, 20)\n",
    "#with Pool(processes=20) as pool:\n",
    "#         for i, row in enumerate(meta_df.iterrows()):\n",
    "#             #print(row)\n",
    "#             rowDriver = visit_site(row[1]['link'], True)\n",
    "#             cv = check_passed(rowDriver)\n",
    "#             if cv == 'Passed':\n",
    "#                 rv = get_passed_dets_link(rowDriver)\n",
    "#                 if rv:\n",
    "#                     links_dict[i] = rv\n",
    "out = paralellize_func(county_gpdf, mp_apply_get_lat_lon, 8,16)\n",
    "\n",
    "end=time.time()\n",
    "\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "unified = []\n",
    "for i in out:\n",
    "    unified.extend(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "geolocated_sds = pd.DataFrame(unified)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_ready_df = pd.merge(county_gpdf, geolocated_sds, how='inner', on = ['X_COORD','Y_COORD'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "map_ready_df.to_csv('mp_ready.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}