Merge pull request #84 from MikeHoffert/dev

Merging Milestone 3 files into master

.gitignore

@@ -26,3 +26,4 @@ nohup.out
 
 # Sublime-workspace files are user specific
 *.sublime-workspace
+*.classpath

app/controllers/MapController.scala

@@ -7,10 +7,19 @@ import scala.util.{Try, Success, Failure}
 import scala.util.matching._
 
 object MapController extends Controller {
+
+  /**
+   * A function that generates a map. Supplied with the location
+   * to be displayed, it will clean the address of any superfluous
+   * elements and then direct the browser accordingly.
+   * @param address The address of the location within the city
+   * @param city The city the location is located within
+   */
   def showMap(address: String, city: String) = Action {
     // Remove bad parts of addresses. Currently we just know about "c/o", so we'll remove that here.
     val coMatcher = """[Cc][\\/][Oo]""".r // Case insensitive "c/o" and "c\o"
     val cleanAddress = coMatcher replaceAllIn(address, m => "")
+
     Ok(views.html.locations.displayMap(cleanAddress, city))
   }
-}
+}

app/coordinatesGetter/PopulateCoordinates.scala

@@ -0,0 +1,213 @@
+package coordinatesGetter
+
+import scala.concurrent.Future
+import scala.util.Failure
+import scala.util.Success
+import scala.util.Try
+import anorm.SQL
+import globals.ActiveDatabase
+import play.api.Play.current
+import play.api.db.DB
+import play.api.libs.functional.syntax.functionalCanBuildApplicative
+import play.api.libs.functional.syntax.toFunctionalBuilderOps
+import play.api.libs.json.JsPath
+import play.api.libs.json.Reads
+import play.api.libs.ws.WS
+import play.api.libs.ws.WSRequestHolder
+import play.api.mvc.Controller
+import play.api.libs.json.JsResult
+import scala.concurrent.Await
+import scala.concurrent.duration.Duration
+
+/**
+ * Represent a coordinate, which contains a latitude and longitude
+ * @param lat The latitude of this coordinate
+ * @param long The longitude of this coordinate
+ */
+case class Coordinate(lat: Double, long: Double)
+
+/**
+ * Represent a http response, which contains a result that includes a coordinate, and a status code
+ * @param coordinate The coordinate
+ * @param status The status code, include: OK, ZERO_RESULTS, OVER_QUERY_LIMIT, REQUEST_DENIED, INVALID_REQUEST, UNKNOWN_ERROR
+ */
+case class Response(coordinate: Coordinate, status: String)
+
+/**
+ * Represent a location, that doesn't have coordinate in the database, and need to get the coordinate from Google Map API.
+ * Use both address and city to find the exact coordinate
+ * @param id The unique of this location
+ * @param address The address of this location
+ * @param city The city this location is in
+ */
+case class NoCoordinateLocation(id: Int, address: String, city: String)
+
+
+
+object PopulateCoordinates{
+
+  /**
+   * The Google Map API URL that accept HTTP get request and return a JSON object that contains 
+   * the information and coordinate of each search result.
+   * Should only use the first result.
+   * Use "address + city" as parameter
+   * eg. https://maps.googleapis.com/maps/api/geocode/json?address=101+Cumberland+Ave,+Saskatoon
+   */
+  var GEOCODING_URL = "https://maps.googleapis.com/maps/api/geocode/json"
+
+
+  /**
+   * Read a coordinate object from JSON
+   */
+  implicit val coordinateReads: Reads[Coordinate] = (
+    (JsPath \ "geometry" \\ "location" \ "lat").read[Double] and
+    (JsPath \ "geometry" \\ "location" \ "lng").read[Double]
+  )(Coordinate.apply _)
+
+  /**
+   * Read a response from JSON
+   */
+  implicit val responseReads: Reads[Response] = (
+    (JsPath \ "results")(0).read[Coordinate].orElse(Reads.pure(Coordinate(0,0))) and
+    (JsPath \ "status").read[String]
+  )(Response.apply _)
+
+
+  /**
+   * Update the coordinates of those locations that already have a corresponding coordinate stored in the 
+   * "coordinate" backup table. This will be run first after each time the database is re-created.
+   * @param connection The database connection 
+   */
+  def updateFromBackup()(implicit connection: java.sql.Connection): Boolean = {
+
+    SQL(
+        """
+          UPDATE location
+          SET latitude = coordinate.latitude,
+          longitude = coordinate.longitude
+          FROM coordinate
+          WHERE location.address = coordinate.address
+          AND location.city = coordinate.city;
+        """
+    ).execute()
+  }
+
+
+   /**
+   * Update the coordinate in the location table
+   * @param id The id of the location that the coordinate need to be updated
+   * @param coordinate The coordinate for this location
+   * @param connection The database connection 
+   */
+  def updateCoordinate(id: Int, coordinate: Coordinate)(implicit connection: java.sql.Connection): Boolean = {
+
+    SQL(
+       """
+         UPDATE location SET latitude = """ + coordinate.lat + """, 
+         longitude = """ + coordinate.long + """
+         WHERE id = """ + id + """;
+       """
+    ).execute()
+  }
+
+   /**
+   * Backup the coordinate, address, and city to "coordinate" table, whenever a new coordinate 
+   * is got from Google Map API. 
+   * @param connection The database connection 
+   */
+  def backupCoordinate(id: Int)(implicit connection: java.sql.Connection): Boolean = {
+
+    SQL(
+       """
+         INSERT INTO coordinate(address, city, latitude, longitude)
+         SELECT address, city, latitude, longitude
+         FROM location
+         WHERE location.id = """ + id + """;
+       """
+    ).execute()
+  }
+
+  /**
+   * Get a list of locations that don't have a coordinate (the latitude and longitude is null)
+   * @param connection The database connection
+   */
+  def getNoCoordinateLocations()(implicit connection: java.sql.Connection): Try[Seq[NoCoordinateLocation]] = {
+
+    Try {
+      val query = SQL(
+         """
+           SELECT id, address, city
+           FROM location
+           WHERE location.latitude IS NULL
+           AND location.longitude IS NULL
+           ORDER BY id;
+         """
+      )
+
+      query().map{
+        row => NoCoordinateLocation(row[Int]("id"), row[String]("address"), row[String]("city"))
+      }.toList
+    }
+  }
+
+  def main()(implicit db: ActiveDatabase): Unit = {
+    DB.withConnection { implicit connection =>  
+
+      //First, update the coordinates from the backup "coordinate" table
+      updateFromBackup()
+
+      //Then get a list of locations that don't have coordinates in backup table
+      getNoCoordinateLocations() match {
+        case Success(locations) => 
+          locations.map { location =>
+
+            //Call the web service 
+            val holder : WSRequestHolder = WS.url(GEOCODING_URL)
+
+            //An execution context, required for Future.map:
+            implicit val context = scala.concurrent.ExecutionContext.Implicits.global
+
+            //The parameter for the HTTP get request
+            val parameterString = location.address + ", " + location.city;
+
+            //Map the response JSON to a coordinate object
+            val futureResult : Future[Response] = holder.withQueryString("address" -> parameterString).get().map{
+              response => response.json.as[Response]
+            }
+
+            futureResult.onComplete{
+              case Success(coordinateResult) => 
+                DB.withConnection { implicit connection =>  
+                  val status = coordinateResult.status
+                  val coordinate: Coordinate = coordinateResult.coordinate
+                  //for debugging
+                  println("status" + status)
+                  println("id" + location.id + ": " + parameterString + " lat: "+ coordinate.lat + " long:" + coordinate.long)
+                  //update this location
+                  updateCoordinate(location.id, coordinate)
+
+                  //backup this new coordinate
+                  backupCoordinate(location.id)
+                }
+
+              case Failure(ex) => 
+                 DB.withConnection { implicit connection =>  
+                  val coordinate = Coordinate(0,0) //if fail to get coordinate, just use (0,0) for now
+                  //for debugging
+                  println("id" + location.id + ": " + parameterString + " Failed to get coordinate: " + ex)
+                  //update this location
+                  updateCoordinate(location.id, coordinate)
+
+                  //backup this new coordinate, or should backup (0,0)?
+                  backupCoordinate(location.id)
+                }
+
+            }
+            Await.ready(futureResult,Duration(100, "second"))
+          }
+        case Failure(ex) =>
+          println("Failed to get no coordinates locations" + ex)
+      }
+    }
+  }  
+}

app/databaseParser/CSVLoader.scala

@@ -39,7 +39,7 @@ class CSVLoader(writer: Writer) {
    * duplicate information) for each violation in the inspection.
    *
    * Note that there's a few places where fields are left blank in the source CSVs. The output
-   * will use "DATA MISSING" as a placeholder.
+   * will use "Unknown" as a placeholder.
    * @param csvFile Path to the CSV file to load.
    * @param locationId The ID to use for the location.
    * @param inspectionIdIn The first inspection ID to use.
@@ -89,10 +89,10 @@ class CSVLoader(writer: Writer) {
 
     // Fix capitalization and escape single quotes
     val locationName = WordUtils.capitalizeFully(dataMatrix(0)(1)).replaceAll("'","''")
-    val locationAddress = WordUtils.capitalizeFully(testNull(dataMatrix(0)(3)).replaceAll("'","''"))
+    val locationAddress = WordUtils.capitalizeFully(getIfExists(dataMatrix(0)(3)).replaceAll("'","''"))
     val locationPostcode = getPostcode(dataMatrix(0)(5))
     val locationCity = getCity(dataMatrix(0)(5)).replaceAll("'","''")
-    val locationRha = testNull(dataMatrix(0)(7))
+    val locationRha = getIfExists(dataMatrix(0)(7))
 
     // Insert location
     writer.write("INSERT INTO location(id, name, address, postcode, city, rha)\n" +
@@ -106,9 +106,9 @@ class CSVLoader(writer: Writer) {
     for(i <- 0 until dataMatrix.size) {
       // Test if this is a new inspection; if yes, insert; if no, just insert violations
       if(i == 0 || !(dataMatrix(i)(2) == (dataMatrix(i - 1)(2)))) {
-        val inspectionDate = testNull(dataMatrix(i)(2));
-        val inspectionType = testNull(dataMatrix(i)(4));
-        val reinspectionPriority = testNull(dataMatrix(i)(6));
+        val inspectionDate = getIfExists(dataMatrix(i)(2));
+        val inspectionType = getIfExists(dataMatrix(i)(4));
+        val reinspectionPriority = getIfExists(dataMatrix(i)(6));
 
         // Insert inspection
         writer.write("INSERT INTO inspection(id, location_id, inspection_date, inspection_type, reinspection_priority)\n" +
@@ -166,38 +166,45 @@ class CSVLoader(writer: Writer) {
    * @return City name, if it exists, or a placeholder if it does not.
    */
   def getCity(string: String): String = {
-    if(string == "" || string.length < 7) {
-      "DATA MISSING";
+    if(string == "") {
+      "Unknown";
     }
     else if(string.matches("^.+, .+$")) {
       WordUtils.capitalizeFully(string.substring(0, string.lastIndexOf(',')))
     }
+    else if(string.matches(", .+$")) {
+      //some files have this column as ", SASKATCHEWAN" 
+      //(see 'Saskatoon_Other Locations_Leaning Maple Meats - Catering [MCKILLOP].csv')
+      //need to return Unknown for this case 
+      "Unknown"
+    }
     else {
-      "DATA MISSING";
+      //if not match the above regexs, then it only contains the city name
+      string;
     }
   }
 
   /**
    * Gets the postcode of a location, which is the last 7 characters of the string.
    * @param string Full 
-   * @return string of postcode, or "DATA MISSING" if the pattern is not matched
+   * @return string of postcode, or "Unknown" if the pattern is not matched
    */
   def getPostcode(string: String): String = {
     if(string == "" || string.length < 7) {
-      "DATA MISSING"
+      "Unknown"
     }
     else if(string.substring(string.length - 7).matches("^[ABCEGHJKLMNPRSTVXY]{1}\\d{1}[A-Z]{1} \\d{1}[A-Z]{1}\\d{1}$")) {
       string.substring(string.length - 7)
     }
     else {
-      "DATA MISSING"
+      "Unknown"
     }
   }
 
   /**
    * Tests if the string is empty, returning a placeholder if it is or the original data otherwise.
    * @param string The full column from the CSV file.
-   * @return "DATA MISSING" if contains nothing or the string itself otherwise.
+   * @return "Unknown" if contains nothing or the string itself otherwise.
    */
-  def testNull(string: String): String = if(string == "") "DATA MISSING" else string
+  def getIfExists(string: String): String = if(string == "") "Unknown" else string
 }

app/downloader/Download.java

@@ -59,7 +59,7 @@ public void setUp() throws Exception {
     baseUrl = "http://orii.health.gov.sk.ca/"; //the URL for the "Saskatchewan Online Restaurant Inspection Information"
     driver.manage().timeouts().implicitlyWait(30, TimeUnit.SECONDS);
 
-    RHAList = new ArrayList<String>(); //Ten RHA for Saskatchewan
+    RHAList = new ArrayList<String>(); //13 RHAs for Saskatchewan
     RHAList.add("Cypress");
     RHAList.add("Five Hills");
     RHAList.add("Heartland");
@@ -70,6 +70,10 @@ public void setUp() throws Exception {
     RHAList.add("Saskatoon");
     RHAList.add("Sun Country");
     RHAList.add("Sunrise");
+    RHAList.add("Athabasca Health Authority");
+    RHAList.add("Keewatin Yatthé");
+    RHAList.add("Mamawetan Churchill River");
+    //Northern Health includes: Athabasca Health Authority, Keewatin Yatthé, Mamawetan Churchill River
   }
 
   @Test

app/models/Inspection.scala

@@ -21,12 +21,14 @@ object Inspection {
    * Gets a list of inspections belonging to a given location.
    *
    * @param locationId The ID of the location we want the inspections for.
-   * @param db this is a implicit parameter that is used to specify what database is to be accessed
+   * @param connection this is a implicit parameter that is used to share the database connection to improve performance
    * @return List of inspection objects representing the inspections for that location.
    */
-  def getInspections(locationId: Int)(implicit db: ActiveDatabase): Try[Seq[Inspection]] = {
+  def getInspections(locationId: Int)(implicit connection: java.sql.Connection): Try[Seq[Inspection]] = {
     Try {
-      DB.withConnection(db.name) { implicit connection =>
+      require(locationId > 0, "Location IDs should be greater than 0.")
+
+
         val query = SQL(
            """
              SELECT id, inspection_date, inspection_type, reinspection_priority
@@ -49,7 +51,6 @@ object Inspection {
         // And convert that into a Seq[Inspection] (which the outside Try will wrap into a
         // Try[Seq[Inspection]])
         tryInspections.map(_.get)
-      }
     }
   }
-}
+}