From 6bc3c9bee347ea14d63508b8076d92536907d4cb Mon Sep 17 00:00:00 2001 From: Ryan Lam <53351930+ryan-lam@users.noreply.github.com> Date: Sun, 5 Nov 2023 00:33:54 -0400 Subject: [PATCH] Migrate sql usage of location service to db gateway (#29) * db gateway now create location service * add etl for location_service * remove sqlalchemy and use raw sql querries * bug fixes, remote fkey rule for etl purposes * Remove print stmt --- db_gateway/src/database.py | 2 +- db_gateway/src/models.py | 10 +- db_gateway/uw_sample_gpx.csv | 140 ------------------ etl/etl.py | 52 +++++-- .../etl_location_service.py | 51 +++++++ location_service/.env.sample | 6 + location_service/Dockerfile | 9 +- location_service/requirements.txt | 20 +-- location_service/src/main.py | 131 ++++++---------- location_service/uw_sample_gpx.csv | 140 ------------------ 10 files changed, 163 insertions(+), 398 deletions(-) delete mode 100644 db_gateway/uw_sample_gpx.csv create mode 100644 etl/etl_location_service/etl_location_service.py delete mode 100644 location_service/uw_sample_gpx.csv diff --git a/db_gateway/src/database.py b/db_gateway/src/database.py index 30ca170..174cf38 100644 --- a/db_gateway/src/database.py +++ b/db_gateway/src/database.py @@ -25,7 +25,7 @@ def check_null(value): def reset_tables(): - from models import RouteModel, Weather + from models import RouteModel, Weather, LocationService Base.metadata.drop_all(bind=engine) Base.metadata.create_all(bind=engine) diff --git a/db_gateway/src/models.py b/db_gateway/src/models.py index da77203..e3710de 100644 --- a/db_gateway/src/models.py +++ b/db_gateway/src/models.py @@ -20,7 +20,15 @@ class RouteModel(Base): gpx_elapsed_dist_m = Column(Float) geopy_elapsed_dist_m = Column(Float) geopy_dist_from_last_m = Column(Float) - weather_id = Column(Integer, ForeignKey("weather.id")) + weather_id = Column(Integer) + + +class LocationService(Base): + __tablename__ = "location_service" + id = Column(Integer, primary_key=True) + lat = Column(Float) + lon = Column(Float) + geo = Column(Geometry(geometry_type="POINT", srid=4326)) class Weather(Base): diff --git a/db_gateway/uw_sample_gpx.csv b/db_gateway/uw_sample_gpx.csv deleted file mode 100644 index 9b8ec77..0000000 --- a/db_gateway/uw_sample_gpx.csv +++ /dev/null @@ -1,140 +0,0 @@ -,lon,lat,type,step,next_turn,dir,gpx_dist_to_next_waypoint_m,gpx_elapsed_dist_m,geopy_elapsed_dist_m,geopy_dist_from_last_m -0,-80.53641449999999,43.472216,waypoint,,,Head northwest on Phillip St,605.0,0.0,0.0,0.0 -1,-80.53648,43.4723,,,,,,,10.732178609621512,10.732178609621512 -2,-80.53657,43.47242,,,,,,,25.92399208354522,15.191813473923707 -3,-80.53667,43.47256,,,,,,,43.45846414006442,17.534472056519206 -4,-80.53682,43.47276,,,,,,,68.77812453900842,25.31966039894399 -5,-80.53699,43.47297,,,,,,,95.86044502713501,27.082320488126594 -6,-80.5372,43.47326,,,,,,,132.2884486668262,36.42800363969119 -7,-80.53725,43.47332,,,,,,,140.08526506594694,7.796816399120742 -8,-80.53755,43.47374,,,,,,,192.68848951585713,52.60322444991019 -9,-80.53769,43.47391,,,,,,,214.7101232192087,22.021633703351572 -10,-80.53786,43.47413,,,,,,,242.75671981563548,28.04659659642677 -11,-80.53794,43.47424,,,,,,,256.58715101114944,13.830431195513956 -12,-80.53809,43.47443,,,,,,,280.93578812568967,24.34863711454024 -13,-80.53821,43.47461,,,,,,,303.1701380189277,22.234349893237983 -14,-80.53845,43.47493,,,,,,,343.6812714119526,40.511133393024906 -15,-80.53855,43.47508,,,,,,,362.20986811935666,18.528596707404052 -16,-80.53877,43.47537,,,,,,,399.01987986847035,36.81001174911368 -17,-80.5389,43.47556,,,,,,,422.60777913062174,23.58789926215136 -18,-80.53922,43.47599,,,,,,,476.9482643222167,54.34048519159496 -19,-80.53929,43.47608,,,,,,,488.4397636795335,11.491499357316789 -20,-80.5393,43.47609,,,,,,,489.81363498183816,1.3738713023047011 -21,-80.53946,43.47631,,,,,,,517.4742964365688,27.660661454730626 -22,-80.53949,43.47635,,,,,,,522.5381582370097,5.06386180044095 -23,-80.53972,43.47667,,,,,,,562.6695628988008,40.131404661791116 -24,-80.53977,43.47673,,,,,,,570.4662614876102,7.7966985888093525 -25,-80.53983,43.4768,,,,,,,579.6327181578249,9.166456670214728 -26,-80.53984,43.47682,,,,,,,581.99847611722,2.365757959395088 -27,-80.53997,43.477,,,,,,,604.5957247688925,22.597248651672555 -28,-80.5399735,43.477,waypoint,turn-left,turn-left,Turn left onto Columbia St W,1629.0,605.0,604.8781355101895,0.2824107412970305 -29,-80.54021,43.47692,,,,,,,625.9325713194646,21.054435809275134 -30,-80.54146,43.47645,,,,,,,739.5297866105944,113.59721529112979 -31,-80.54204,43.47625,,,,,,,791.3449740397789,51.815187429184434 -32,-80.54263,43.47605,,,,,,,843.8902425489968,52.5452685092179 -33,-80.54275,43.47605,,,,,,,853.5730487597679,9.68280621077111 -34,-80.54294,43.47598,,,,,,,870.7668993931212,17.193850633353403 -35,-80.54322,43.47589,,,,,,,895.4773524830368,24.710453089915628 -36,-80.54412,43.47559,,,,,,,975.3939447245533,79.91659224151643 -37,-80.54553,43.47509,,,,,,,1102.0260110494726,126.6320663249192 -38,-80.54631,43.47482,,,,,,,1171.7592637550895,69.73325270561683 -39,-80.54653,43.47476,,,,,,,1190.7237462710775,18.96448251598797 -40,-80.54666,43.47472,,,,,,,1202.1176776685165,11.393931397438966 -41,-80.54728,43.4745,,,,,,,1257.8073324039792,55.689654735462625 -42,-80.54811,43.47417,,,,,,,1334.1753901669583,76.36805776297905 -43,-80.54905,43.47383,,,,,,,1418.9263566189397,84.75096645198134 -44,-80.54907,43.47382,,,,,,,1420.8861985085673,1.9598418896274936 -45,-80.55024,43.47339,,,,,,,1526.714543807155,105.82834529858775 -46,-80.5504,43.47334,,,,,,,1540.7717242036838,14.057180396528729 -47,-80.55109,43.47309,,,,,,,1603.004309024841,62.23258482115715 -48,-80.55123,43.47304,,,,,,,1615.595449144822,12.591140119980926 -49,-80.55345,43.47225,,,,,,,1815.1159942079355,199.5205450631135 -50,-80.55501,43.47169,,,,,,,1955.5598874763627,140.4438932684273 -51,-80.55631,43.47122,,,,,,,2072.76207310612,117.20218562975761 -52,-80.55697,43.47098,,,,,,,2132.333813750923,59.57174064480281 -53,-80.5579,43.47064,,,,,,,2216.3668968964143,84.033083145491 -54,-80.55792,43.47063,,,,,,,2218.3268089351704,1.9599120387559648 -55,-80.5581,43.47057,,,,,,,2234.3112499894646,15.98444105429416 -56,-80.558103,43.4705729,waypoint,turn-left,turn-left,Turn left onto Westmount Rd N,1624.0,2234.0,2234.7144778916545,0.4032279021897861 -57,-80.5583,43.47052,,,,,,,2251.6652139361754,16.950736044521133 -58,-80.55812,43.47044,,,,,,,2268.698222016757,17.033008080581652 -59,-80.55734,43.46999,,,,,,,2349.108217023129,80.40999500637174 -60,-80.5569,43.46976,,,,,,,2392.8670879422793,43.758870919150354 -61,-80.55651,43.46958,,,,,,,2430.164801942688,37.29771400040877 -62,-80.55631,43.46949,,,,,,,2449.1553981197003,18.990596177012527 -63,-80.55606,43.46939,,,,,,,2472.191516931287,23.03611881158661 -64,-80.55573,43.46927,,,,,,,2501.9780844981415,29.786567566854494 -65,-80.55539,43.46917,,,,,,,2531.5833517736223,29.60526727548078 -66,-80.5551,43.46908,,,,,,,2557.0360940863325,25.45274231271019 -67,-80.55479,43.46901,,,,,,,2583.235833770164,26.199739683831666 -68,-80.55443,43.46895,,,,,,,2613.0438817144,29.808047944235803 -69,-80.55414,43.46892,,,,,,,2636.683302998931,23.639421284530968 -70,-80.55385,43.4689,,,,,,,2660.1916104349584,23.508307436027355 -71,-80.55347,43.4689,,,,,,,2690.857457916249,30.66584748129091 -72,-80.55309,43.46893,,,,,,,2721.7042024300263,30.84674451377699 -73,-80.55221,43.46901,,,,,,,2793.274742651888,71.57054022186168 -74,-80.55184,43.46906,,,,,,,2823.646734629452,30.37199197756408 -75,-80.55146,43.46912,,,,,,,2855.0298501131583,31.383115483706042 -76,-80.55113,43.46914,,,,,,,2881.7533126093317,26.723462496173315 -77,-80.5509,43.46915,,,,,,,2900.3474228587575,18.594110249425675 -78,-80.55067,43.46914,,,,,,,2918.941533108208,18.594110249450697 -79,-80.55047,43.46912,,,,,,,2935.233886066828,16.292352958619563 -80,-80.55024,43.46909,,,,,,,2954.0921168752743,18.858230808446425 -81,-80.55002,43.46905,,,,,,,2972.394645723721,18.302528848446713 -82,-80.54992,43.46902,,,,,,,2981.1268775846356,8.732231860914522 -83,-80.54975,43.46897,,,,,,,2995.929558552943,14.802680968307259 -84,-80.54959,43.46892,,,,,,,3009.9876062132657,14.058047660322691 -85,-80.54946,43.46886,,,,,,,3022.4202917865796,12.432685573313965 -86,-80.54923,43.46872,,,,,,,3046.6452735677494,24.22498178116988 -87,-80.54909,43.46866,,,,,,,3059.766099348887,13.120825781137528 -88,-80.54881,43.46851,,,,,,,3087.8512779754024,28.085178626515415 -89,-80.54858,43.46834,,,,,,,3114.343584914226,26.492306938823504 -90,-80.54797,43.46774,,,,,,,3197.2562327698147,82.91264785558845 -91,-80.54792,43.4677,,,,,,,3203.261618134266,6.005385364451377 -92,-80.54754,43.4673,,,,,,,3257.2869198772805,54.02530174301446 -93,-80.54728,43.46703,,,,,,,3293.9151342650243,36.628214387743824 -94,-80.54725,43.467,,,,,,,3298.0369615605605,4.12182729553596 -95,-80.5471,43.46686,,,,,,,3317.7570100718567,19.720048511296113 -96,-80.54695,43.46672,,,,,,,3337.4770757935066,19.720065721649945 -97,-80.54694,43.46672,,,,,,,3338.284100885447,0.8070250919402361 -98,-80.5468,43.46658,,,,,,,3357.5193350050854,19.235234119638363 -99,-80.54678,43.46656,,,,,,,3360.267226848473,2.7478918433877535 -100,-80.5466,43.46643,,,,,,,3380.7605630056237,20.493336157150825 -101,-80.54635,43.46625,,,,,,,3409.180026157979,28.41946315235491 -102,-80.54624,43.46619,,,,,,,3420.2849341282954,11.104907970316829 -103,-80.54604,43.46607,,,,,,,3441.2269368196917,20.942002691396283 -104,-80.54587,43.46599,,,,,,,3457.5780462866132,16.351109466921717 -105,-80.54567,43.46589,,,,,,,3477.1781990724467,19.600152785833533 -106,-80.54542,43.46578,,,,,,,3500.7721909017746,23.593991829328154 -107,-80.54484,43.46551,,,,,,,3556.3813328834426,55.609141981668074 -108,-80.54375,43.465,,,,,,,3661.044182886329,104.66285000288642 -109,-80.54367,43.46496,,,,,,,3668.8843284366103,7.840145550281192 -110,-80.54303,43.46468,,,,,,,3729.1936499922417,60.309321555631534 -111,-80.54282,43.46459,,,,,,,3748.875867618599,19.682217626357293 -112,-80.54193,43.46431,,,,,,,3827.161347505395,78.28547988679594 -113,-80.54156,43.46421,,,,,,,3859.025612151667,31.86426464627159 -114,-80.5415593,43.4642077,waypoint,turn-left,turn-left,Turn left onto University Ave W,942.0,3858.0,3859.2875262159805,0.26191406431389364 -115,-80.54145,43.46415,,,,,,,3870.195196557855,10.907670341874628 -116,-80.54139,43.46426,,,,,,,3883.3503083931805,13.155111835325455 -117,-80.54124,43.46442,,,,,,,3904.8695776824525,21.519269289272053 -118,-80.54065,43.46556,,,,,,,4040.27996655707,135.41038887461787 -119,-80.54002,43.46679,,,,,,,4186.194443807475,145.91447725040484 -120,-80.53998,43.46686,,,,,,,4194.62094333784,8.426499530365062 -121,-80.53943,43.46791,,,,,,,4319.528107106862,124.90716376902242 -122,-80.53904,43.46866,,,,,,,4408.665669451085,89.13756234422262 -123,-80.53899,43.46876,,,,,,,4420.4946437033705,11.82897425228524 -124,-80.53856,43.46961,,,,,,,4521.17918332208,100.68453961870951 -125,-80.53819,43.47031,,,,,,,4604.546140791025,83.36695746894563 -126,-80.53813,43.47043,,,,,,,4618.740864532841,14.194723741816377 -127,-80.53797,43.47073,,,,,,,4654.510963998629,35.770099465787595 -128,-80.53786,43.4709,,,,,,,4675.394569289684,20.883605291054693 -129,-80.53781,43.47098,,,,,,,4685.162464793208,9.767895503523915 -130,-80.53774,43.47107,,,,,,,4696.654194323604,11.491729530397103 -131,-80.53764,43.4713,,,,,,,4723.471973234399,26.817778910795237 -132,-80.53748,43.47143,,,,,,,4742.853973791091,19.382000556692528 -133,-80.53741,43.47149,,,,,,,4751.595817326036,8.741843534944964 -134,-80.53733,43.47155,,,,,,,4760.879542654073,9.283725328036818 -135,-80.53715,43.47167,,,,,,,4780.603392376451,19.723849722377654 -136,-80.53701,43.47174,,,,,,,4794.322611945383,13.719219568932191 -137,-80.53694,43.47178,,,,,,,4801.512244642542,7.189632697158927 -138,-80.5369395,43.4717778,waypoint,,,,,,4801.760178886321,0.24793424377920087 diff --git a/etl/etl.py b/etl/etl.py index 6291995..7a025bc 100644 --- a/etl/etl.py +++ b/etl/etl.py @@ -7,6 +7,7 @@ from etl_routemodel.etl_routemodel import main as run_routemodel_etl from etl_weather.etl_weather import main as run_weather_etl from etl_speed_limit.etl_speed_limit import main as run_speed_limit_etl +from etl_location_service.etl_location_service import main as run_location_service_etl from termcolor import colored @@ -60,23 +61,27 @@ def cmd_routemodel(db_user, db_password, db_host, db_name): print(colored("routemodel ETL success", "green")) -def cmd_streetname_speedlimit(db_user, db_password, db_host, db_name): +def cmd_location_service(db_user, db_password, db_host, db_name): answers = questionary.form( - bingmaps_api_key=questionary.password("Bing Maps API key", default=""), + csv_filepath=questionary.path( + "Path to data (CSV file)", + default="", + validate=lambda p: validate_path(p, ".csv"), + ), confirm=questionary.confirm( - "Confirm street_name/speed_limit ETL operation", + "Confirm routemodel ETL operation", default=False, auto_enter=False, ), ).ask() - bingmaps_api_key = answers["bingmaps_api_key"] + csv_filepath = answers["csv_filepath"] confirm = answers["confirm"] if not confirm: - print(colored("streetname/speedlimit ETL cancelled", "red")) + print(colored("location_service ETL cancelled", "red")) else: - run_speed_limit_etl(db_user, db_password, db_host, db_name, bingmaps_api_key) - print(colored("streetname/speedlimit ETL success", "green")) + run_location_service_etl(csv_filepath, db_user, db_password, db_host, db_name) + print(colored("location_service ETL success", "green")) def cmd_weather(db_user, db_password, db_host, db_name): @@ -111,6 +116,25 @@ def cmd_weather(db_user, db_password, db_host, db_name): print(colored("weather ETL success", "green")) +def cmd_streetname_speedlimit(db_user, db_password, db_host, db_name): + answers = questionary.form( + bingmaps_api_key=questionary.password("Bing Maps API key", default=""), + confirm=questionary.confirm( + "Confirm street_name/speed_limit ETL operation", + default=False, + auto_enter=False, + ), + ).ask() + bingmaps_api_key = answers["bingmaps_api_key"] + confirm = answers["confirm"] + + if not confirm: + print(colored("streetname/speedlimit ETL cancelled", "red")) + else: + run_speed_limit_etl(db_user, db_password, db_host, db_name, bingmaps_api_key) + print(colored("streetname/speedlimit ETL success", "green")) + + def cmd_drop_tables(db_user, db_password, db_host, db_name): answers = questionary.form( table_names=questionary.checkbox( @@ -146,7 +170,13 @@ def cmd_drop_tables(db_user, db_password, db_host, db_name): ) etl_name = questionary.select( "Select ETL operation", - choices=["routemodel", "weather", "speed_limit/street_names", "drop_tables"], + choices=[ + "routemodel", + "location_service", + "weather", + "speed_limit/street_names", + "drop_tables", + ], ).ask() auth, db_user, db_password, db_host, db_name = validate_db_creds() @@ -156,9 +186,11 @@ def cmd_drop_tables(db_user, db_password, db_host, db_name): if etl_name == "routemodel": cmd_routemodel(db_user, db_password, db_host, db_name) + elif etl_name == "location_service": + cmd_location_service(db_user, db_password, db_host, db_name) elif etl_name == "weather": cmd_weather(db_user, db_password, db_host, db_name) - elif etl_name == "drop_tables": - cmd_drop_tables(db_user, db_password, db_host, db_name) elif etl_name == "speed_limit/street_names": cmd_streetname_speedlimit(db_user, db_password, db_host, db_name) + elif etl_name == "drop_tables": + cmd_drop_tables(db_user, db_password, db_host, db_name) diff --git a/etl/etl_location_service/etl_location_service.py b/etl/etl_location_service/etl_location_service.py new file mode 100644 index 0000000..cdf071c --- /dev/null +++ b/etl/etl_location_service/etl_location_service.py @@ -0,0 +1,51 @@ +from pathlib import Path + +import geopandas as gpd +import numpy as np +import pandas as pd +from geoalchemy2 import Geometry +from sqlalchemy import create_engine + + +def seed_from_csv(csv_filepath, db_user, db_password, db_host, db_name): + file = Path(csv_filepath) + if not file.is_file(): + raise FileNotFoundError("No file exists at the location specified") + gdf = gpd.GeoDataFrame(pd.read_csv(file)) + gdf.fillna(np.nan).replace([np.nan], [None]) + gdf = gdf[["lat", "lon", "geo"]] + gdf.index.name = "id" + gdf.index += 1 + # gdf = gdf.head(10) # For testing purposes + + engine = create_engine( + f"postgresql+psycopg2://{db_user}:{db_password}@{db_host}/{db_name}" + ) + + response = gdf.to_sql( + name="location_service", + con=engine, + schema="public", + if_exists="replace", + index=True, + method="multi", + dtype={"geo": Geometry("POINT", srid=4326)}, + ) + if gdf.shape[0] != response: + raise SystemError("dataframe insertion failed") + else: + print("location_service dataframe insertion success") + + +def main(csv_filepath, db_user, db_password, db_host, db_name): + print("1) Parsing and seeding routemodel into location_service...") + seed_from_csv(csv_filepath, db_user, db_password, db_host, db_name) + + +if __name__ == "__main__": + csv_filepath = "" + db_user = "" + db_password = "" + db_host = "" + db_name = "" + main(csv_filepath, db_user, db_password, db_host, db_name) diff --git a/location_service/.env.sample b/location_service/.env.sample index f377940..9a802ad 100644 --- a/location_service/.env.sample +++ b/location_service/.env.sample @@ -1,2 +1,8 @@ DATABASE_URI="postgresql+psycopg2://user:password@host:port/database" +DB_HOST="" +DB_NAME="" +DB_USER="" +DB_PASSWORD="" +DB_PORT="" +REQUIRE_AUTH="" AUTH_KEY="" diff --git a/location_service/Dockerfile b/location_service/Dockerfile index 6726233..da84922 100644 --- a/location_service/Dockerfile +++ b/location_service/Dockerfile @@ -1,7 +1,12 @@ FROM python:3.11.4-buster RUN pip install --upgrade cython pip -ENV DATABASE_URI="" +ENV DB_HOST="" +ENV DB_NAME="" +ENV DB_USER="" +ENV DB_PASSWORD="" +ENV DB_PORT="" +ENV REQUIRE_AUTH="" ENV AUTH_KEY="" WORKDIR /usr/location_service @@ -13,4 +18,4 @@ COPY . . EXPOSE 5000 -CMD [ "python", "src/main.py", "--create-table", "true", "--seed-filename", "./uw_sample_gpx.csv" ] +CMD [ "python", "src/main.py" ] diff --git a/location_service/requirements.txt b/location_service/requirements.txt index 97e0e73..bcd3089 100644 --- a/location_service/requirements.txt +++ b/location_service/requirements.txt @@ -1,22 +1,10 @@ -blinker==1.6.2 -click==8.1.5 +blinker==1.7.0 +click==8.1.7 colorama==0.4.6 -Flask==2.3.2 -Flask-SQLAlchemy==3.0.5 -GeoAlchemy2==0.14.0 -greenlet==2.0.2 +Flask==3.0.0 itsdangerous==2.1.2 Jinja2==3.1.2 MarkupSafe==2.1.3 -numpy==1.25.1 -packaging==23.1 -pandas==2.0.3 -psycopg2==2.9.6 -python-dateutil==2.8.2 +psycopg2==2.9.9 python-dotenv==1.0.0 -pytz==2023.3 -six==1.16.0 -SQLAlchemy==2.0.18 -typing_extensions==4.7.1 -tzdata==2023.3 Werkzeug==3.0.1 diff --git a/location_service/src/main.py b/location_service/src/main.py index dd7b5f4..e203062 100644 --- a/location_service/src/main.py +++ b/location_service/src/main.py @@ -1,81 +1,35 @@ import json import os -from argparse import ArgumentParser -import pandas as pd from dotenv import load_dotenv from flask import Flask, request -from flask_sqlalchemy import SQLAlchemy -from geoalchemy2 import Geometry -from geoalchemy2.comparator import Comparator -from sqlalchemy import asc, func +from psycopg2 import pool as psycopg2_pool load_dotenv() -DATABASE_URI = os.environ.get("DATABASE_URI") +REQUIRE_AUTH = os.getenv("REQUIRE_AUTH", "False").lower() in ("true", "1", "t") AUTH_KEY = os.environ.get("AUTH_KEY") - - -parser = ArgumentParser() -parser.add_argument( - "--create-table", - type=bool, - help="If you want a table to be created in the database", - required=False, -) -parser.add_argument( - "--seed-filename", - type=str, - help="String of the filepath of the csv file if you want to seed the database with the csv file contents", - required=False, -) -args = parser.parse_args() -args = vars(args) +DB_HOST = os.environ.get("DB_HOST") +DB_PORT = os.environ.get("DB_PORT") +DB_NAME = os.environ.get("DB_NAME") +DB_USER = os.environ.get("DB_USER") +DB_PASSWORD = os.environ.get("DB_PASSWORD") app = Flask(__name__) -app.config["SQLALCHEMY_DATABASE_URI"] = DATABASE_URI -app.config["SQLALCHEMY_ECHO"] = False -app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False -db = SQLAlchemy(app) -app.app_context().push() - - -class Location(db.Model): - __tablename__ = "location_test" # Change table name - - id = db.Column(db.Integer, primary_key=True) - lat = db.Column(db.Float) - lon = db.Column(db.Float) - geo = db.Column(Geometry(geometry_type="POINT", srid=4326)) - - def to_dict(self): - return { - "id": self.id, - "lat": self.lat, - "lon": self.lon, - } - - @staticmethod - def create(lat, lon): - geo = "POINT({} {})".format(lon, lat) # Note that postgis is (lon, lat) - location = Location(lat=lat, lon=lon, geo=geo) - db.session.add(location) - db.session.commit() - - @staticmethod - def seed_from_csv(filename): - df = pd.read_csv(filename) - db.session.query(Location).delete() - for row in df.itertuples(): - geo = "POINT({} {})".format( - row.lon, row.lat - ) # Note that postgis is (lon, lat) - location = Location(id=row.Index + 1, lat=row.lat, lon=row.lon, geo=geo) - db.session.add(location) - db.session.commit() +postgres_pool = psycopg2_pool.SimpleConnectionPool( + minconn=1, + maxconn=10, + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + user=DB_USER, + password=DB_PASSWORD, +) def authorized(auth_key): + if not REQUIRE_AUTH: + return True return auth_key == AUTH_KEY @@ -84,14 +38,21 @@ def index(): return "location service" -@app.route("/all", methods=["GET"]) +@app.route("/all", methods=["GET", "POST"]) def all(): body = json.loads(request.data) auth_key = body.get("auth_key", None) if not authorized(auth_key): return "Not authorized", 401 - locations = db.session.query(Location).order_by(asc(Location.id)) - return [l.to_dict() for l in locations] + + query = "SELECT id, lat, lon FROM location_service;" + conn = postgres_pool.getconn() + cur = conn.cursor() + cur.execute(query) + locations = [dict(zip(("id", "lat", "lon"), values)) for values in cur.fetchall()] + cur.close() + postgres_pool.putconn(conn) + return locations @app.route("/location", methods=["POST"]) @@ -104,27 +65,21 @@ def closest_location(): return "Not authorized", 401 if lat is None or lon is None: return "Invalid request body", 400 - location = ( - db.session.query(Location) - .order_by( - Comparator.distance_centroid( - Location.geo, - func.Geometry( - func.ST_GeographyFromText( - "POINT({} {})".format(lon, lat) - ) # Note that postgis is (lon, lat) - ), - ) - ) - .limit(1) - .first() - ) - return location.to_dict() + + query = f""" + SELECT * + FROM location_service + ORDER BY location_service.geo <-> ST_SetSRID(ST_MakePoint({lon}, {lat}), 4326) + LIMIT 1; + """ + conn = postgres_pool.getconn() + cur = conn.cursor() + cur.execute(query) + location = dict(zip(("id", "lat", "lon"), cur.fetchone())) + cur.close() + postgres_pool.putconn(conn) + return location if __name__ == "__main__": - if args["create_table"]: - db.create_all() - if args["seed_filename"]: - Location.seed_from_csv(args["seed_filename"]) - app.run(debug=False, host="0.0.0.0", port=5000) + app.run(debug=True, host="0.0.0.0", port=5000) diff --git a/location_service/uw_sample_gpx.csv b/location_service/uw_sample_gpx.csv deleted file mode 100644 index 0e156be..0000000 --- a/location_service/uw_sample_gpx.csv +++ /dev/null @@ -1,140 +0,0 @@ -,lon,lat,type,step,next_turn,dir,dist_to_next_m,elapsed_dist_m,time_to_next_s -0,-80.53641449999999,43.472216,waypoint,,,Head northwest on Phillip St,605.0,0.0,115.0 -1,-80.53648,43.4723,,,,,,, -2,-80.53657,43.47242,,,,,,, -3,-80.53667,43.47256,,,,,,, -4,-80.53682,43.47276,,,,,,, -5,-80.53699,43.47297,,,,,,, -6,-80.5372,43.47326,,,,,,, -7,-80.53725,43.47332,,,,,,, -8,-80.53755,43.47374,,,,,,, -9,-80.53769,43.47391,,,,,,, -10,-80.53786,43.47413,,,,,,, -11,-80.53794,43.47424,,,,,,, -12,-80.53809,43.47443,,,,,,, -13,-80.53821,43.47461,,,,,,, -14,-80.53845,43.47493,,,,,,, -15,-80.53855,43.47508,,,,,,, -16,-80.53877,43.47537,,,,,,, -17,-80.5389,43.47556,,,,,,, -18,-80.53922,43.47599,,,,,,, -19,-80.53929,43.47608,,,,,,, -20,-80.5393,43.47609,,,,,,, -21,-80.53946,43.47631,,,,,,, -22,-80.53949,43.47635,,,,,,, -23,-80.53972,43.47667,,,,,,, -24,-80.53977,43.47673,,,,,,, -25,-80.53983,43.4768,,,,,,, -26,-80.53984,43.47682,,,,,,, -27,-80.53997,43.477,,,,,,, -28,-80.5399735,43.477,waypoint,turn-left,turn-left,Turn left onto Columbia St W,1629.0,605.0,150.0 -29,-80.54021,43.47692,,,,,,, -30,-80.54146,43.47645,,,,,,, -31,-80.54204,43.47625,,,,,,, -32,-80.54263,43.47605,,,,,,, -33,-80.54275,43.47605,,,,,,, -34,-80.54294,43.47598,,,,,,, -35,-80.54322,43.47589,,,,,,, -36,-80.54412,43.47559,,,,,,, -37,-80.54553,43.47509,,,,,,, -38,-80.54631,43.47482,,,,,,, -39,-80.54653,43.47476,,,,,,, -40,-80.54666,43.47472,,,,,,, -41,-80.54728,43.4745,,,,,,, -42,-80.54811,43.47417,,,,,,, -43,-80.54905,43.47383,,,,,,, -44,-80.54907,43.47382,,,,,,, -45,-80.55024,43.47339,,,,,,, -46,-80.5504,43.47334,,,,,,, -47,-80.55109,43.47309,,,,,,, -48,-80.55123,43.47304,,,,,,, -49,-80.55345,43.47225,,,,,,, -50,-80.55501,43.47169,,,,,,, -51,-80.55631,43.47122,,,,,,, -52,-80.55697,43.47098,,,,,,, -53,-80.5579,43.47064,,,,,,, -54,-80.55792,43.47063,,,,,,, -55,-80.5581,43.47057,,,,,,, -56,-80.558103,43.4705729,waypoint,turn-left,turn-left,Turn left onto Westmount Rd N,1624.0,2234.0,128.0 -57,-80.5583,43.47052,,,,,,, -58,-80.55812,43.47044,,,,,,, -59,-80.55734,43.46999,,,,,,, -60,-80.5569,43.46976,,,,,,, -61,-80.55651,43.46958,,,,,,, -62,-80.55631,43.46949,,,,,,, -63,-80.55606,43.46939,,,,,,, -64,-80.55573,43.46927,,,,,,, -65,-80.55539,43.46917,,,,,,, -66,-80.5551,43.46908,,,,,,, -67,-80.55479,43.46901,,,,,,, -68,-80.55443,43.46895,,,,,,, -69,-80.55414,43.46892,,,,,,, -70,-80.55385,43.4689,,,,,,, -71,-80.55347,43.4689,,,,,,, -72,-80.55309,43.46893,,,,,,, -73,-80.55221,43.46901,,,,,,, -74,-80.55184,43.46906,,,,,,, -75,-80.55146,43.46912,,,,,,, -76,-80.55113,43.46914,,,,,,, -77,-80.5509,43.46915,,,,,,, -78,-80.55067,43.46914,,,,,,, -79,-80.55047,43.46912,,,,,,, -80,-80.55024,43.46909,,,,,,, -81,-80.55002,43.46905,,,,,,, -82,-80.54992,43.46902,,,,,,, -83,-80.54975,43.46897,,,,,,, -84,-80.54959,43.46892,,,,,,, -85,-80.54946,43.46886,,,,,,, -86,-80.54923,43.46872,,,,,,, -87,-80.54909,43.46866,,,,,,, -88,-80.54881,43.46851,,,,,,, -89,-80.54858,43.46834,,,,,,, -90,-80.54797,43.46774,,,,,,, -91,-80.54792,43.4677,,,,,,, -92,-80.54754,43.4673,,,,,,, -93,-80.54728,43.46703,,,,,,, -94,-80.54725,43.467,,,,,,, -95,-80.5471,43.46686,,,,,,, -96,-80.54695,43.46672,,,,,,, -97,-80.54694,43.46672,,,,,,, -98,-80.5468,43.46658,,,,,,, -99,-80.54678,43.46656,,,,,,, -100,-80.5466,43.46643,,,,,,, -101,-80.54635,43.46625,,,,,,, -102,-80.54624,43.46619,,,,,,, -103,-80.54604,43.46607,,,,,,, -104,-80.54587,43.46599,,,,,,, -105,-80.54567,43.46589,,,,,,, -106,-80.54542,43.46578,,,,,,, -107,-80.54484,43.46551,,,,,,, -108,-80.54375,43.465,,,,,,, -109,-80.54367,43.46496,,,,,,, -110,-80.54303,43.46468,,,,,,, -111,-80.54282,43.46459,,,,,,, -112,-80.54193,43.46431,,,,,,, -113,-80.54156,43.46421,,,,,,, -114,-80.5415593,43.4642077,waypoint,turn-left,turn-left,Turn left onto University Ave W,942.0,3858.0,122.0 -115,-80.54145,43.46415,,,,,,, -116,-80.54139,43.46426,,,,,,, -117,-80.54124,43.46442,,,,,,, -118,-80.54065,43.46556,,,,,,, -119,-80.54002,43.46679,,,,,,, -120,-80.53998,43.46686,,,,,,, -121,-80.53943,43.46791,,,,,,, -122,-80.53904,43.46866,,,,,,, -123,-80.53899,43.46876,,,,,,, -124,-80.53856,43.46961,,,,,,, -125,-80.53819,43.47031,,,,,,, -126,-80.53813,43.47043,,,,,,, -127,-80.53797,43.47073,,,,,,, -128,-80.53786,43.4709,,,,,,, -129,-80.53781,43.47098,,,,,,, -130,-80.53774,43.47107,,,,,,, -131,-80.53764,43.4713,,,,,,, -132,-80.53748,43.47143,,,,,,, -133,-80.53741,43.47149,,,,,,, -134,-80.53733,43.47155,,,,,,, -135,-80.53715,43.47167,,,,,,, -136,-80.53701,43.47174,,,,,,, -137,-80.53694,43.47178,,,,,,, -138,-80.5369395,43.4717778,waypoint,,,,,,