Skip to content

Commit

Permalink
Updated whittlesea_vic_gov_au to use OpenCities API (#2326)
Browse files Browse the repository at this point in the history 
* Updated whittlesea_vic_gov_au to use OpenCities API

* Update whittlesea_vic_gov_au doc to reflect API changes

* reformatting

---------

Co-authored-by: 5ila5 <5ila5@users.noreply.github.com>
  • Loading branch information
@EthanBezz @5ila5
EthanBezz and 5ila5 authored Jul 26, 2024
1 parent 7c2ade2 commit 83eed62
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 138 deletions.
170 changes: 53 additions & 117 deletions ...nents/waste_collection_schedule/waste_collection_schedule/source/whittlesea_vic_gov_au.py
View file
Original file line number Diff line number Diff line change
@@ -1,146 +1,82 @@
import json
import logging
import re
from datetime import datetime

import requests
from dateutil.parser import parse
from bs4 import BeautifulSoup
from waste_collection_schedule import Collection # type: ignore[attr-defined]
from datetime import timedelta

TITLE = "Whittlesea City Council"
DESCRIPTION = "Source for Whittlesea Council (VIC) rubbish collection."
URL = "https://whittlesea.vic.gov.au/community-support/my-neighbourhood/"

URL = "https://www.whittlesea.vic.gov.au/My-Neighbourhood"
TEST_CASES = {
"Random address": {
"street_number": "5",
"street_name": "Hawkstowe Parade",
"suburb": "South Morang",
"postcode": 3752,
},
"Whittlesea Council Office": {
"street_number": 25,
"street_name": "Ferres Boulevard",
"suburb": "South Morang",
"postcode": "3752",
},
"street_address": "25 Ferres Boulevard, South Morang 3752"
}
}

_LOGGER = logging.getLogger(__name__)

ICON_MAP = {
"rubbish": "mdi:trash-can",
"recycle": "mdi:recycle",
"glass": "mdi:glass-fragile",
"green": "mdi:leaf",
"General Waste": "mdi:trash-can",
"Recycling": "mdi:recycle",
"Green Waste": "mdi:leaf",
"Glass": "mdi:glass-fragile",
}

# Only a year's worth of dates is available
WEEKS = 53


class Source:
def __init__(self, suburb, street_name, street_number, postcode):
self.suburb = suburb
self.street_name = street_name
self.street_number = str(street_number)
self.postcode = str(postcode)
def __init__(self, street_address):
self._street_address = street_address

def fetch(self):
# Retrieve geolocation for our address
# (TODO: cache the LAT/LON results)
address = (
self.street_number
+ " "
+ self.street_name
+ " "
+ self.suburb
+ " "
+ self.postcode
)
PARAMS = {"address": address}
url = "https://www.whittlesea.vic.gov.au/umbraco/api/vicmap/GetAddressResultsUsingArcGis/"
r = requests.get(
url,
params=PARAMS,
)
r.raise_for_status()

# TODO: better error handling of parsing issues
json_string = (
r.text.encode("raw_unicode_escape")
.decode("unicode_escape")
.lstrip('"')
.rstrip('"')
)
data = json.loads(json_string)

if not isinstance(data, dict):
raise Exception("malformed response from web query")

features = data.get("features")
session = requests.Session()

# Find the coordinates for our address
# TODO: check that there is only one geometry
geometry = features[0].get("geometry")
geo_x = str(geometry.get("x"))
geo_y = str(geometry.get("y"))
response = session.get("https://www.whittlesea.vic.gov.au/My-Neighbourhood")
response.raise_for_status()

# Armed with the LAT and LON coordinates, we construct
# a request to fetch the waste pick-up schedules
url = "https://www.whittlesea.vic.gov.au/umbraco/api/cartomap/GetQueryResultsArcGisWasteCollection"

firstQuery = (
"geometry%3D"
+ geo_x
+ ","
+ geo_y
+ "%26geometryType%3DesriGeometryPoint%26inSR%3D4326%26spatialRel%3DesriSpatialRelIntersects%26outFields%3DName%26returnGeometry%3Dfalse%26f%3Djson"
response = session.get(
"https://www.whittlesea.vic.gov.au/api/v1/myarea/search",
params={"keywords": self._street_address},
)

secondQuery = (
"where%3Dzonename%253D%2527%7B0%7D%2527%2Band%2Bdate%3ECURRENT_TIMESTAMP-1%26time%3D%26topFilter%3D%257B%250D%250A%2B%2B%2522groupByFields%2522%253A%2B%2522zonename%2522%252C%250D%250A%2B%2B%2522topCount%2522%253A%2B"
+ str(WEEKS)
+ "%252C%250D%250A%2B%2B%2522orderByFields%2522%253A%2B%2522date%2522%250D%250A%257D%26outFields%3D*%26orderByFields%3Ddate%26resultRecordCount%3D"
+ str(WEEKS)
+ "%26f%3Djson"
response.raise_for_status()
addressSearchApiResults = response.json()
if (
addressSearchApiResults["Items"] is None
or len(addressSearchApiResults["Items"]) < 1
):
raise Exception(
f"Address search for '{self._street_address}' returned no results. Check your address on https://www.whittlesea.vic.gov.au/My-Neighbourhood"
)

addressSearchTopHit = addressSearchApiResults["Items"][0]
_LOGGER.debug("Address search top hit: %s", addressSearchTopHit)

geolocationid = addressSearchTopHit["Id"]
_LOGGER.debug("Geolocationid: %s", geolocationid)

response = session.get(
"https://www.whittlesea.vic.gov.au/ocapi/Public/myarea/wasteservices?ocsvclang=en-AU",
params={"geolocationid": geolocationid},
)
response.raise_for_status()

url += "?firstQuery=" + firstQuery + "&secondQuery=" + secondQuery
wasteApiResult = response.json()
_LOGGER.debug("Waste API result: %s", wasteApiResult)

r = requests.get(
url,
)
r.raise_for_status()

json_string = (
r.text.encode("raw_unicode_escape")
.decode("unicode_escape")
.lstrip('"')
.rstrip('"')
)
data = json.loads(json_string)
soup = BeautifulSoup(wasteApiResult["responseContent"], "html.parser")

entries = []

for item in data["rows"]:
if "cartodb_id" in item:
# adding 1 day to the date to fix timezone issue (covers AEST and AEDST)
# https://github.com/mampfes/hacs_waste_collection_schedule/issues/912
collection_date = (parse(item["date"]) + timedelta(days=1)).date()
for article in soup.find_all("article"):
waste_type = article.h3.string
icon = ICON_MAP.get(waste_type)
next_pickup = article.find(class_="next-service").string.strip()
if re.match(r"[^\s]* \d{1,2}\/\d{1,2}\/\d{4}", next_pickup):
next_pickup_date = datetime.strptime(
next_pickup.split(sep=" ")[1], "%d/%m/%Y"
).date()
entries.append(
Collection(
date=collection_date,
t="rubbish",
icon=ICON_MAP.get("rubbish"),
)
Collection(date=next_pickup_date, t=waste_type, icon=icon)
)

# test extra waste types
for waste_type in ["recycling", "green", "glass"]:
if item[waste_type] == 1:
entries.append(
Collection(
date=collection_date,
t=waste_type,
icon=ICON_MAP.get(waste_type),
)
)

return entries
27 changes: 6 additions & 21 deletions doc/source/whittlesea_vic_gov_au.md
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# Whittlesea Council (VIC)
# City of Whittlesea Council

Support for schedules provided by [Whittlesea Council (VIC)](https://whittlesea.vic.gov.au/community-support/my-neighbourhood/).
Support for schedules provided by [City of Whittlesea Council](https://www.whittlesea.vic.gov.au/).

## Configuration via configuration.yaml

Expand All @@ -9,24 +9,12 @@ waste_collection_schedule:
sources:
- name: whittlesea_vic_gov_au
args:
street_number: STREET_NUMBER
suburb: SUBURB
street_name: STREET_NAME
postcode: POSTCODE
street_address: STREET_ADDRESS
```
### Configuration Variables
**street_number**<br>
*(string) (required)*
**street_name**<br>
*(string) (required)*
**suburb**<br>
*(string) (required)*
**postcode**<br>
**street_address**
*(string) (required)*
## Example
Expand All @@ -36,12 +24,9 @@ waste_collection_schedule:
sources:
- name: whittlesea_vic_gov_au
args:
street_number: '25'
street_name: Ferres Bouleavard
suburb: South Morang
postcode: '3752'
street_address: 25 Ferres Boulevard, South Morang 3752
```
## How to get the source arguments
Visit the [Whittlesea Council (VIC)](https://whittlesea.vic.gov.au/community-support/my-neighbourhood/) page and search for your address. The arguments should exactly match the results shown.
Visit the [City of Whittlesea Council My Neighbourhood](https://www.whittlesea.vic.gov.au/My-Neighbourhood) page and search for your address. The arguments should exactly match the street address shown in the autocomplete result.

0 comments on commit 83eed62

Please sign in to comment.