diff --git a/custom_components/waste_collection_schedule/waste_collection_schedule/source/whittlesea_vic_gov_au.py b/custom_components/waste_collection_schedule/waste_collection_schedule/source/whittlesea_vic_gov_au.py index 7ed484a1c..62f948ffd 100644 --- a/custom_components/waste_collection_schedule/waste_collection_schedule/source/whittlesea_vic_gov_au.py +++ b/custom_components/waste_collection_schedule/waste_collection_schedule/source/whittlesea_vic_gov_au.py @@ -1,146 +1,82 @@ -import json +import logging +import re +from datetime import datetime import requests -from dateutil.parser import parse +from bs4 import BeautifulSoup from waste_collection_schedule import Collection # type: ignore[attr-defined] -from datetime import timedelta TITLE = "Whittlesea City Council" DESCRIPTION = "Source for Whittlesea Council (VIC) rubbish collection." -URL = "https://whittlesea.vic.gov.au/community-support/my-neighbourhood/" - +URL = "https://www.whittlesea.vic.gov.au/My-Neighbourhood" TEST_CASES = { - "Random address": { - "street_number": "5", - "street_name": "Hawkstowe Parade", - "suburb": "South Morang", - "postcode": 3752, - }, "Whittlesea Council Office": { - "street_number": 25, - "street_name": "Ferres Boulevard", - "suburb": "South Morang", - "postcode": "3752", - }, + "street_address": "25 Ferres Boulevard, South Morang 3752" + } } +_LOGGER = logging.getLogger(__name__) + ICON_MAP = { - "rubbish": "mdi:trash-can", - "recycle": "mdi:recycle", - "glass": "mdi:glass-fragile", - "green": "mdi:leaf", + "General Waste": "mdi:trash-can", + "Recycling": "mdi:recycle", + "Green Waste": "mdi:leaf", + "Glass": "mdi:glass-fragile", } -# Only a year's worth of dates is available -WEEKS = 53 - class Source: - def __init__(self, suburb, street_name, street_number, postcode): - self.suburb = suburb - self.street_name = street_name - self.street_number = str(street_number) - self.postcode = str(postcode) + def __init__(self, street_address): + self._street_address = street_address def fetch(self): - # Retrieve geolocation for our address - # (TODO: cache the LAT/LON results) - address = ( - self.street_number - + " " - + self.street_name - + " " - + self.suburb - + " " - + self.postcode - ) - PARAMS = {"address": address} - url = "https://www.whittlesea.vic.gov.au/umbraco/api/vicmap/GetAddressResultsUsingArcGis/" - r = requests.get( - url, - params=PARAMS, - ) - r.raise_for_status() - - # TODO: better error handling of parsing issues - json_string = ( - r.text.encode("raw_unicode_escape") - .decode("unicode_escape") - .lstrip('"') - .rstrip('"') - ) - data = json.loads(json_string) - - if not isinstance(data, dict): - raise Exception("malformed response from web query") - - features = data.get("features") + session = requests.Session() - # Find the coordinates for our address - # TODO: check that there is only one geometry - geometry = features[0].get("geometry") - geo_x = str(geometry.get("x")) - geo_y = str(geometry.get("y")) + response = session.get("https://www.whittlesea.vic.gov.au/My-Neighbourhood") + response.raise_for_status() - # Armed with the LAT and LON coordinates, we construct - # a request to fetch the waste pick-up schedules - url = "https://www.whittlesea.vic.gov.au/umbraco/api/cartomap/GetQueryResultsArcGisWasteCollection" - - firstQuery = ( - "geometry%3D" - + geo_x - + "," - + geo_y - + "%26geometryType%3DesriGeometryPoint%26inSR%3D4326%26spatialRel%3DesriSpatialRelIntersects%26outFields%3DName%26returnGeometry%3Dfalse%26f%3Djson" + response = session.get( + "https://www.whittlesea.vic.gov.au/api/v1/myarea/search", + params={"keywords": self._street_address}, ) - - secondQuery = ( - "where%3Dzonename%253D%2527%7B0%7D%2527%2Band%2Bdate%3ECURRENT_TIMESTAMP-1%26time%3D%26topFilter%3D%257B%250D%250A%2B%2B%2522groupByFields%2522%253A%2B%2522zonename%2522%252C%250D%250A%2B%2B%2522topCount%2522%253A%2B" - + str(WEEKS) - + "%252C%250D%250A%2B%2B%2522orderByFields%2522%253A%2B%2522date%2522%250D%250A%257D%26outFields%3D*%26orderByFields%3Ddate%26resultRecordCount%3D" - + str(WEEKS) - + "%26f%3Djson" + response.raise_for_status() + addressSearchApiResults = response.json() + if ( + addressSearchApiResults["Items"] is None + or len(addressSearchApiResults["Items"]) < 1 + ): + raise Exception( + f"Address search for '{self._street_address}' returned no results. Check your address on https://www.whittlesea.vic.gov.au/My-Neighbourhood" + ) + + addressSearchTopHit = addressSearchApiResults["Items"][0] + _LOGGER.debug("Address search top hit: %s", addressSearchTopHit) + + geolocationid = addressSearchTopHit["Id"] + _LOGGER.debug("Geolocationid: %s", geolocationid) + + response = session.get( + "https://www.whittlesea.vic.gov.au/ocapi/Public/myarea/wasteservices?ocsvclang=en-AU", + params={"geolocationid": geolocationid}, ) + response.raise_for_status() - url += "?firstQuery=" + firstQuery + "&secondQuery=" + secondQuery + wasteApiResult = response.json() + _LOGGER.debug("Waste API result: %s", wasteApiResult) - r = requests.get( - url, - ) - r.raise_for_status() - - json_string = ( - r.text.encode("raw_unicode_escape") - .decode("unicode_escape") - .lstrip('"') - .rstrip('"') - ) - data = json.loads(json_string) + soup = BeautifulSoup(wasteApiResult["responseContent"], "html.parser") entries = [] - - for item in data["rows"]: - if "cartodb_id" in item: - # adding 1 day to the date to fix timezone issue (covers AEST and AEDST) - # https://github.com/mampfes/hacs_waste_collection_schedule/issues/912 - collection_date = (parse(item["date"]) + timedelta(days=1)).date() + for article in soup.find_all("article"): + waste_type = article.h3.string + icon = ICON_MAP.get(waste_type) + next_pickup = article.find(class_="next-service").string.strip() + if re.match(r"[^\s]* \d{1,2}\/\d{1,2}\/\d{4}", next_pickup): + next_pickup_date = datetime.strptime( + next_pickup.split(sep=" ")[1], "%d/%m/%Y" + ).date() entries.append( - Collection( - date=collection_date, - t="rubbish", - icon=ICON_MAP.get("rubbish"), - ) + Collection(date=next_pickup_date, t=waste_type, icon=icon) ) - # test extra waste types - for waste_type in ["recycling", "green", "glass"]: - if item[waste_type] == 1: - entries.append( - Collection( - date=collection_date, - t=waste_type, - icon=ICON_MAP.get(waste_type), - ) - ) - return entries diff --git a/doc/source/whittlesea_vic_gov_au.md b/doc/source/whittlesea_vic_gov_au.md index 546f8771d..716f119e5 100644 --- a/doc/source/whittlesea_vic_gov_au.md +++ b/doc/source/whittlesea_vic_gov_au.md @@ -1,6 +1,6 @@ -# Whittlesea Council (VIC) +# City of Whittlesea Council -Support for schedules provided by [Whittlesea Council (VIC)](https://whittlesea.vic.gov.au/community-support/my-neighbourhood/). +Support for schedules provided by [City of Whittlesea Council](https://www.whittlesea.vic.gov.au/). ## Configuration via configuration.yaml @@ -9,24 +9,12 @@ waste_collection_schedule: sources: - name: whittlesea_vic_gov_au args: - street_number: STREET_NUMBER - suburb: SUBURB - street_name: STREET_NAME - postcode: POSTCODE + street_address: STREET_ADDRESS ``` ### Configuration Variables -**street_number**
-*(string) (required)* - -**street_name**
-*(string) (required)* - -**suburb**
-*(string) (required)* - -**postcode**
+**street_address** *(string) (required)* ## Example @@ -36,12 +24,9 @@ waste_collection_schedule: sources: - name: whittlesea_vic_gov_au args: - street_number: '25' - street_name: Ferres Bouleavard - suburb: South Morang - postcode: '3752' + street_address: 25 Ferres Boulevard, South Morang 3752 ``` ## How to get the source arguments -Visit the [Whittlesea Council (VIC)](https://whittlesea.vic.gov.au/community-support/my-neighbourhood/) page and search for your address. The arguments should exactly match the results shown. +Visit the [City of Whittlesea Council My Neighbourhood](https://www.whittlesea.vic.gov.au/My-Neighbourhood) page and search for your address. The arguments should exactly match the street address shown in the autocomplete result.