From f777b041cc4a1ab3541a6198ba83a4906f63f005 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Tue, 7 Jan 2025 23:04:55 +0000 Subject: [PATCH 01/19] docs: Update Councils.md from input.json --- wiki/Councils.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/wiki/Councils.md b/wiki/Councils.md index 494a9a08d7..f2aa4d3c40 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -69,8 +69,8 @@ This document is still a work in progress, don't worry if your council isn't lis - [Conwy County Borough Council](#conwy-county-borough-council) - [Copeland Borough Council](#copeland-borough-council) - [Cornwall Council](#cornwall-council) -- [Coventry City Council](#coventry-city-council) - [Cotswold District Council](#cotswold-district-council) +- [Coventry City Council](#coventry-city-council) - [Crawley Borough Council](#crawley-borough-council) - [Croydon Council](#croydon-council) - [Cumberland Borough Council](#cumberland-borough-council) @@ -1019,15 +1019,6 @@ Note: Use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find your U --- -### Coventry City Council -```commandline -python collect_data.py CoventryCityCouncil https://www.coventry.gov.uk/directory_record/XXXXXX/XXXXXX -``` - -Note: Follow the instructions [here](https://www.coventry.gov.uk/bin-collection-calendar) until you get the page that shows the weekly collections for your address then copy the URL and replace the URL in the command. - ---- - ### Cotswold District Council ```commandline python collect_data.py CotswoldDistrictCouncil https://community.cotswold.gov.uk/s/waste-collection-enquiry -s -p "XXXX XXX" -n XX -w http://HOST:PORT/ @@ -1042,6 +1033,15 @@ Note: Pass the full address in the house number and postcode in --- +### Coventry City Council +```commandline +python collect_data.py CoventryCityCouncil https://www.coventry.gov.uk/directory_record/XXXXXX/XXXXXX +``` + +Note: Follow the instructions [here](https://www.coventry.gov.uk/bin-collection-calendar) until you get the page that shows the weekly collections for your address then copy the URL and replace the URL in the command. + +--- + ### Crawley Borough Council ```commandline python collect_data.py CrawleyBoroughCouncil https://my.crawley.gov.uk/ -s -u XXXXXXXX -n XX From 58036a7c8a8c6d2e6c3c481f7f43d10db201cc84 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Tue, 7 Jan 2025 23:28:14 +0000 Subject: [PATCH 02/19] fix: West Oxfordshire District Council --- .../councils/WestOxfordshireDistrictCouncil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py index 99b39edbd7..7a18c76448 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestOxfordshireDistrictCouncil.py @@ -42,14 +42,14 @@ def parse_data(self, page: str, **kwargs) -> dict: wait = WebDriverWait(driver, 60) address_entry_field = wait.until( EC.presence_of_element_located( - (By.XPATH, '//*[@id="combobox-input-19"]') + (By.XPATH, '//*[@id="combobox-input-20"]') ) ) address_entry_field.send_keys(str(full_address)) address_entry_field = wait.until( - EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-19"]')) + EC.element_to_be_clickable((By.XPATH, '//*[@id="combobox-input-20"]')) ) address_entry_field.click() address_entry_field.send_keys(Keys.BACKSPACE) @@ -57,7 +57,7 @@ def parse_data(self, page: str, **kwargs) -> dict: first_found_address = wait.until( EC.element_to_be_clickable( - (By.XPATH, '//*[@id="dropdown-element-19"]/ul') + (By.XPATH, '//*[@id="dropdown-element-20"]/ul') ) ) From 778b542451ee927c8b6072b77eb60650f9e3f321 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:00:06 +0000 Subject: [PATCH 03/19] fix: Vale of Glamorgan Council --- .../councils/ValeofGlamorganCouncil.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py index 876e0b4a76..c5bdd48e5c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ValeofGlamorganCouncil.py @@ -1,4 +1,5 @@ from bs4 import BeautifulSoup + from uk_bin_collection.uk_bin_collection.common import * from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass @@ -31,12 +32,12 @@ def parse_data(self, page: str, **kwargs) -> dict: params = { "RequestType": "LocalInfo", "ms": "ValeOfGlamorgan/AllMaps", - "group": "Community and Living|Refuse HIDE2", - "type": "json", + "group": "Waste|new_refuse", + "type": "jsonp", "callback": "AddressInfoCallback", "uid": user_uprn, - "import": "jQuery35108514154283927682_1673022974838", - "_": "1673022974840", + "import": "jQuery35107288886041176057_1736292844067", + "_": "1736292844068", } # Get a response from the council @@ -46,13 +47,15 @@ def parse_data(self, page: str, **kwargs) -> dict: headers=headers, ).text + response = response.replace("AddressInfoCallback(", "").rstrip(");") + # Load the JSON and seek out the bin week text, then add it to the calendar URL. Also take the weekly # collection type and generate dates for it. Then make a GET request for the calendar bin_week = str( - json.loads(response)["Results"]["Refuse_HIDE2"]["Your_Refuse_round_is"] + json.loads(response)["Results"]["waste"]["roundday_residual"] ).replace(" ", "-") weekly_collection = str( - json.loads(response)["Results"]["Refuse_HIDE2"]["Recycling__type"] + json.loads(response)["Results"]["waste"]["recycling_code"] ).capitalize() weekly_dates = get_weekday_dates_in_period( datetime.now(), days_of_week.get(bin_week.split("-")[0].strip()), amount=48 From 83dfb8a4450c5eb14c7dddabc7600cf7df89f782 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:05:32 +0000 Subject: [PATCH 04/19] fix: Stroud District Council --- .../councils/StroudDistrictCouncil.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py index 480f1bc5fa..6d580311a2 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StroudDistrictCouncil.py @@ -1,8 +1,10 @@ +from datetime import datetime, timedelta +from typing import Any, Dict + from bs4 import BeautifulSoup + from uk_bin_collection.uk_bin_collection.common import * from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass -from datetime import datetime, timedelta -from typing import Dict, Any class CouncilClass(AbstractGetBinDataClass): @@ -78,7 +80,10 @@ def parse_data(self, page: Any, **kwargs: Any) -> Dict[str, Any]: formatted_date = self.get_next_weekday(day_name) else: # Convert date format from "Tuesday 28 May 2024" to "28/05/2024" - date_obj = datetime.strptime(value, "%A %d %B %Y") + try: + date_obj = datetime.strptime(value, "%A %d %B %Y") + except: + continue formatted_date = date_obj.strftime("%d/%m/%Y") bin_entry = { From cacf3de5eb11bc199f2a44f75764b4803525f148 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:08:57 +0000 Subject: [PATCH 05/19] fix: Staffordshire Moorlands District Council --- .../StaffordshireMoorlandsDistrictCouncil.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py index 6b36d8c280..89c0d4c11e 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py @@ -30,12 +30,12 @@ def parse_data(self, page: str, **kwargs) -> dict: driver.get("https://www.staffsmoorlands.gov.uk/findyourbinday") # Close cookies banner - cookieAccept = WebDriverWait(driver, 10).until( - EC.presence_of_element_located( - (By.CSS_SELECTOR, ".cookiemessage__link--close") - ) - ) - cookieAccept.click() + # cookieAccept = WebDriverWait(driver, 10).until( + # EC.presence_of_element_located( + # (By.CSS_SELECTOR, ".cookiemessage__link--close") + # ) + # ) + # cookieAccept.click() # Wait for the postcode field to appear then populate it inputElement_postcode = WebDriverWait(driver, 30).until( From 4e48d40419747700f9e5273634369e3ff27ea2df Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:16:46 +0000 Subject: [PATCH 06/19] fix: Powys Council --- .../uk_bin_collection/councils/PowysCouncil.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/PowysCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PowysCouncil.py index b5980279cd..d5c68dc97b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PowysCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PowysCouncil.py @@ -127,12 +127,15 @@ def parse_data(self, page: str, **kwargs) -> dict: li.text for li in garden_waste_section.find_next("ul").find_all("li") ] for date in garden_waste_dates: - dict_data = { - "type": "Garden Waste", - "collectionDate": datetime.strptime( - remove_ordinal_indicator_from_date_string(date), "%d %B %Y" - ).strftime(date_format), - } - data["bins"].append(dict_data) + try: + dict_data = { + "type": "Garden Waste", + "collectionDate": datetime.strptime( + remove_ordinal_indicator_from_date_string(date), "%d %B %Y" + ).strftime(date_format), + } + data["bins"].append(dict_data) + except: + continue return data From 40493f0e52639883c0de6db21e6a0ca017c3f313 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:47:02 +0000 Subject: [PATCH 07/19] fix: Rushcliffe Borough Council --- uk_bin_collection/tests/input.json | 8 ---- .../councils/RushcliffeBoroughCouncil.py | 37 +++++++++++++------ 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 321ad7a1d6..ecceac116b 100755 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1418,14 +1418,6 @@ "wiki_name": "Powys Council", "wiki_note": "Pass the house name/number and postcode in their respective parameters. This parser requires a Selenium webdriver." }, - "PowysCouncil": { - "house_number": "LANE COTTAGE", - "postcode": "HR3 5JS", - "skip_get_url": true, - "url": "https://www.powys.gov.uk", - "web_driver": "http://selenium:4444", - "wiki_name": "Powys Council" - }, "PrestonCityCouncil": { "house_number": "Town Hall", "postcode": "PR1 2RL", diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index 579cb59675..d524eb2a94 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -37,37 +37,52 @@ def parse_data(self, page: str, **kwargs) -> dict: # Populate postcode field inputElement_postcode = driver.find_element( By.ID, - "ctl00_ContentPlaceHolder1_FF3518TB", + "FF3518-text", ) inputElement_postcode.send_keys(user_postcode) # Click search button driver.find_element( By.ID, - "ctl00_ContentPlaceHolder1_FF3518BTN", + "FF3518-find", ).click() # Wait for the 'Select address' dropdown to appear and select option matching UPRN dropdown = WebDriverWait(driver, 10).until( - EC.presence_of_element_located( - (By.ID, "ctl00_ContentPlaceHolder1_FF3518DDL") - ) + EC.presence_of_element_located((By.ID, "FF3518-list")) ) - # Create a 'Select' for it, then select the matching URPN option + + WebDriverWait(driver, 10).until( + EC.element_to_be_clickable( + ( + By.XPATH, + f"//select[@id='FF3518-list']/option[starts-with(@value, 'U{user_uprn}')]", + ) + ) + ).click() + + """# Create a 'Select' for it, then select the matching URPN option dropdownSelect = Select(dropdown) - dropdownSelect.select_by_value("U" + user_uprn) + target_prefix = "U" + user_uprn + for option in dropdownSelect.options: + option_value = option.get_attribute("value") + if option_value.startswith(target_prefix): # Search by visible text + dropdownSelect.select_by_visible_text(option.text) + # dropdownSelect.select_by_value("U" + user_uprn)""" # Wait for the submit button to appear, then click it to get the collection dates submit = WebDriverWait(driver, 10).until( - EC.presence_of_element_located( - (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit") - ) + EC.presence_of_element_located((By.ID, "submit-button")) ) submit.click() + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.CLASS_NAME, "ss_confPanel")) + ) + soup = BeautifulSoup(driver.page_source, features="html.parser") - bins_text = soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation") + bins_text = soup.find("div", id="body-content") if bins_text: results = re.findall( From 6de409fd5bdafa18945d3eab2f96a47fa5068950 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Wed, 8 Jan 2025 00:47:35 +0000 Subject: [PATCH 08/19] docs: Update Councils.md from input.json --- wiki/Councils.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wiki/Councils.md b/wiki/Councils.md index f2aa4d3c40..753ba7f668 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -2533,6 +2533,8 @@ Additional parameters: - `-n` - house number - `-w` - remote Selenium web driver URL (required for Home Assistant) +Note: Pass the house name/number and postcode in their respective parameters. This parser requires a Selenium webdriver. + --- ### Preston City Council From bfa06595128822c795ba9dd7899cbb23513d8367 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 00:59:54 +0000 Subject: [PATCH 09/19] fix: NewarkAndSherwoodDC --- uk_bin_collection/tests/input.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index ecceac116b..74e5718e21 100755 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1208,8 +1208,8 @@ "wiki_note": "Pass the postcode and UPRN. This parser requires a Selenium webdriver." }, "NewarkAndSherwoodDC": { - "url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529&nc=1", - "wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX&nc=1", + "url": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=200004258529", + "wiki_command_url_override": "http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX", "wiki_name": "Newark and Sherwood District Council", "wiki_note": "Replace XXXXXXXX with your UPRN." }, From 2eeb430cdfdb09d19da9fa04c4302128d454caa2 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Wed, 8 Jan 2025 01:00:47 +0000 Subject: [PATCH 10/19] docs: Update Councils.md from input.json --- wiki/Councils.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wiki/Councils.md b/wiki/Councils.md index 753ba7f668..b9ae4dc7ec 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -2199,7 +2199,7 @@ Note: Pass the postcode and UPRN. This parser requires a Selenium webdriver. ### Newark and Sherwood District Council ```commandline -python collect_data.py NewarkAndSherwoodDC http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX&nc=1 +python collect_data.py NewarkAndSherwoodDC http://app.newark-sherwooddc.gov.uk/bincollection/calendar?pid=XXXXXXXX ``` Note: Replace XXXXXXXX with your UPRN. From 17b566b1183b348783f041b3897dab49bfc045f9 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 8 Jan 2025 01:07:36 +0000 Subject: [PATCH 11/19] fix: Merton Council --- uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py b/uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py index 0912ce3aee..e84066b009 100644 --- a/uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/MertonCouncil.py @@ -47,10 +47,10 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_type = cells[0].get_text().strip() # Date is on the second cell, second paragraph, wrapped in p collectionDate = None - for date_format in possible_formats: + for format in possible_formats: try: collectionDate = datetime.strptime( - cells[1].select("p > b")[2].get_text(strip=True), date_format + cells[1].select("p > b")[2].get_text(strip=True), format ) break # Exit the loop if parsing is successful except ValueError: From c004c2e88cb5ebca7e00c37a73a4f73897b2f588 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:16:08 +0000 Subject: [PATCH 12/19] Revert "fix: Rushcliffe Borough Council" This reverts commit 40493f0e52639883c0de6db21e6a0ca017c3f313. --- uk_bin_collection/tests/input.json | 8 ++++ .../councils/RushcliffeBoroughCouncil.py | 37 ++++++------------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 74e5718e21..5726adbbb5 100755 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1418,6 +1418,14 @@ "wiki_name": "Powys Council", "wiki_note": "Pass the house name/number and postcode in their respective parameters. This parser requires a Selenium webdriver." }, + "PowysCouncil": { + "house_number": "LANE COTTAGE", + "postcode": "HR3 5JS", + "skip_get_url": true, + "url": "https://www.powys.gov.uk", + "web_driver": "http://selenium:4444", + "wiki_name": "Powys Council" + }, "PrestonCityCouncil": { "house_number": "Town Hall", "postcode": "PR1 2RL", diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index d524eb2a94..579cb59675 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -37,52 +37,37 @@ def parse_data(self, page: str, **kwargs) -> dict: # Populate postcode field inputElement_postcode = driver.find_element( By.ID, - "FF3518-text", + "ctl00_ContentPlaceHolder1_FF3518TB", ) inputElement_postcode.send_keys(user_postcode) # Click search button driver.find_element( By.ID, - "FF3518-find", + "ctl00_ContentPlaceHolder1_FF3518BTN", ).click() # Wait for the 'Select address' dropdown to appear and select option matching UPRN dropdown = WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.ID, "FF3518-list")) - ) - - WebDriverWait(driver, 10).until( - EC.element_to_be_clickable( - ( - By.XPATH, - f"//select[@id='FF3518-list']/option[starts-with(@value, 'U{user_uprn}')]", - ) + EC.presence_of_element_located( + (By.ID, "ctl00_ContentPlaceHolder1_FF3518DDL") ) - ).click() - - """# Create a 'Select' for it, then select the matching URPN option + ) + # Create a 'Select' for it, then select the matching URPN option dropdownSelect = Select(dropdown) - target_prefix = "U" + user_uprn - for option in dropdownSelect.options: - option_value = option.get_attribute("value") - if option_value.startswith(target_prefix): # Search by visible text - dropdownSelect.select_by_visible_text(option.text) - # dropdownSelect.select_by_value("U" + user_uprn)""" + dropdownSelect.select_by_value("U" + user_uprn) # Wait for the submit button to appear, then click it to get the collection dates submit = WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.ID, "submit-button")) + EC.presence_of_element_located( + (By.ID, "ctl00_ContentPlaceHolder1_btnSubmit") + ) ) submit.click() - WebDriverWait(driver, 10).until( - EC.presence_of_element_located((By.CLASS_NAME, "ss_confPanel")) - ) - soup = BeautifulSoup(driver.page_source, features="html.parser") - bins_text = soup.find("div", id="body-content") + bins_text = soup.find("div", id="ctl00_ContentPlaceHolder1_pnlConfirmation") if bins_text: results = re.findall( From 376e27be2969da5a6fd784cbd801acf3c6f0a833 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Wed, 15 Jan 2025 17:16:55 +0000 Subject: [PATCH 13/19] docs: Update Councils.md from input.json --- wiki/Councils.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/wiki/Councils.md b/wiki/Councils.md index b9ae4dc7ec..fb7a7f09e0 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -2533,8 +2533,6 @@ Additional parameters: - `-n` - house number - `-w` - remote Selenium web driver URL (required for Home Assistant) -Note: Pass the house name/number and postcode in their respective parameters. This parser requires a Selenium webdriver. - --- ### Preston City Council From 5ce842a0e580fc0365a080e99b2f7f88671a5f26 Mon Sep 17 00:00:00 2001 From: Wiki GitHub Action Date: Wed, 15 Jan 2025 17:18:40 +0000 Subject: [PATCH 14/19] docs: Update Councils.md from input.json --- wiki/Councils.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/wiki/Councils.md b/wiki/Councils.md index fb7a7f09e0..af3a0a17fe 100644 --- a/wiki/Councils.md +++ b/wiki/Councils.md @@ -242,6 +242,7 @@ This document is still a work in progress, don't worry if your council isn't lis - [Stratford Upon Avon Council](#stratford-upon-avon-council) - [Stroud District Council](#stroud-district-council) - [Sunderland City Council](#sunderland-city-council) +- [Surrey Heath Borough Council / Joint Waste Solutions](#surrey-heath-borough-council-/-joint-waste-solutions) - [Swale Borough Council](#swale-borough-council) - [Swansea Council](#swansea-council) - [Swindon Borough Council](#swindon-borough-council) @@ -3074,6 +3075,19 @@ Note: Provide your house number (without quotes) and postcode (wrapped in double --- +### Surrey Heath Borough Council / Joint Waste Solutions +```commandline +python collect_data.py SurreyHeathBoroughCouncil https://asjwsw-wrpsurreyheathmunicipal-live.whitespacews.com/ -s -p "XXXX XXX" -n XX +``` +Additional parameters: +- `-s` - skip get URL +- `-p` - postcode +- `-n` - house number + +Note: Provide your house number in the `house_number` parameter and postcode in the `postcode` parameter. + +--- + ### Swale Borough Council ```commandline python collect_data.py SwaleBoroughCouncil https://swale.gov.uk/bins-littering-and-the-environment/bins/collection-days -s -p "XXXX XXX" -n XX -w http://HOST:PORT/ From f58c81215bf6bd8a0f8b3335b8b264aa8ef1bcfc Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:32:40 +0000 Subject: [PATCH 15/19] fix: Glasgow City Council fix: #1166 --- .../councils/GlasgowCityCouncil.py | 25 +++++++++++++------ 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py index 4c0eaaa78d..559b9bdcc6 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GlasgowCityCouncil.py @@ -1,3 +1,5 @@ +import datetime + from bs4 import BeautifulSoup from uk_bin_collection.uk_bin_collection.common import * @@ -32,15 +34,24 @@ def parse_data(self, page: str, **kwargs) -> dict: for p in ps: collection = p.text.strip().replace("Your next ", "").split(".")[0] bin_type = collection.split(" day is")[0] - collection_date = datetime.strptime( - remove_ordinal_indicator_from_date_string(collection).split("day is ")[ - 1 - ], - "%A %d %B %Y", - ) + collection_date = remove_ordinal_indicator_from_date_string( + collection + ).split("day is ")[1] + if collection_date == "Today": + collection_date = datetime.today().strftime(date_format) + elif collection_date == "Tomorrow": + collection_date = (datetime.today() + timedelta(days=1)).strftime( + date_format + ) + print(collection_date) + else: + collection_date = datetime.strptime( + collection_date, + "%A %d %B %Y", + ).strftime(date_format) dict_data = { "type": bin_type, - "collectionDate": collection_date.strftime(date_format), + "collectionDate": collection_date, } data["bins"].append(dict_data) From 46abca88d07d80f09825aa950877296ebe09e323 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Wed, 15 Jan 2025 17:41:21 +0000 Subject: [PATCH 16/19] fix: Carmarthenshire County Council fix: #1167 --- .../councils/CarmarthenshireCountyCouncil.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/CarmarthenshireCountyCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CarmarthenshireCountyCouncil.py index 9bedb1ca6b..22814d82ad 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CarmarthenshireCountyCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CarmarthenshireCountyCouncil.py @@ -34,10 +34,10 @@ def parse_data(self, page: str, **kwargs) -> dict: # Find the next collection date date_tag = container.find(class_="font11 text-center") - if date_tag.text.strip() == "": - continue - else: + if date_tag: collection_date = date_tag.text.strip() + else: + continue dict_data = { "type": bin_type, From 25072c60bdf58efa5872f46142651ca11dab4aa5 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Thu, 16 Jan 2025 00:53:57 +0000 Subject: [PATCH 17/19] fix: Surrey Heath Borough Council fix: #1164 --- uk_bin_collection/tests/input.json | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 655d6e4bc0..07b2c1bd19 100755 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -1409,15 +1409,6 @@ "wiki_name": "Portsmouth City Council", "wiki_note": "Pass the postcode and UPRN. This parser requires a Selenium webdriver." }, - "PowysCouncil": { - "house_number": "LANE COTTAGE", - "postcode": "HR3 5JS", - "skip_get_url": true, - "url": "https://www.powys.gov.uk", - "web_driver": "http://selenium:4444", - "wiki_name": "Powys Council", - "wiki_note": "Pass the house name/number and postcode in their respective parameters. This parser requires a Selenium webdriver." - }, "PowysCouncil": { "house_number": "LANE COTTAGE", "postcode": "HR3 5JS", @@ -1768,7 +1759,7 @@ "wiki_name": "Sunderland City Council", "wiki_note": "Provide your house number (without quotes) and postcode (wrapped in double quotes with a space)." }, - "SurreyHeathBoroughCouncil": { + "SurreyHeathBoroughCouncil": { "house_number": "36", "postcode": "GU20 6PN", "skip_get_url": true, From 6a75f29253b7e20d4102410c3b81828bc1733f78 Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Thu, 16 Jan 2025 01:08:28 +0000 Subject: [PATCH 18/19] fix: South Oxfordshire Council fix: #1158 --- .../councils/SouthOxfordshireCouncil.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py index 3d368616be..683246c479 100644 --- a/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py @@ -1,5 +1,6 @@ import requests from bs4 import BeautifulSoup + from uk_bin_collection.uk_bin_collection.common import * from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass @@ -66,7 +67,7 @@ def parse_data(self, page: str, **kwargs) -> dict: "%A %d %B - %Y", ) ).strftime(date_format) - bin_type = str.capitalize(' '.join(bin_info[1:])) + bin_type = str.capitalize(" ".join(bin_info[1:])) # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string else: bin_date = get_next_occurrence_from_day_month( @@ -75,9 +76,9 @@ def parse_data(self, page: str, **kwargs) -> dict: "%A %d %B - %Y", ) ).strftime(date_format) - str.capitalize(' '.join(bin_info[2:])) - except Exception as ex: - raise ValueError(f"Error parsing bin data: {ex}") + str.capitalize(" ".join(bin_info[2:])) + except: + continue # Build data dict for each entry dict_data = { From c43d47bf78ae4d4555934526705aea5a6fd1565c Mon Sep 17 00:00:00 2001 From: m26dvd <31007572+m26dvd@users.noreply.github.com> Date: Thu, 16 Jan 2025 01:21:13 +0000 Subject: [PATCH 19/19] fix: Vale of White Horse fix: #1156 --- .../councils/SouthOxfordshireCouncil.py | 22 ++++++++++------ .../councils/ValeofWhiteHorseCouncil.py | 25 +++++++++++++------ 2 files changed, 33 insertions(+), 14 deletions(-) diff --git a/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py index 683246c479..268b89aa33 100644 --- a/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/SouthOxfordshireCouncil.py @@ -55,6 +55,9 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} + current_year = datetime.now().year + next_year = current_year + 1 + # Page has slider info side by side, which are two instances of this class for bin in soup.find_all("div", {"class": "binextra"}): bin_info = list(bin.stripped_strings) @@ -63,27 +66,32 @@ def parse_data(self, page: str, **kwargs) -> dict: if contains_date(bin_info[0]): bin_date = get_next_occurrence_from_day_month( datetime.strptime( - bin_info[0] + " " + datetime.today().strftime("%Y"), - "%A %d %B - %Y", + bin_info[0], + "%A %d %B -", ) - ).strftime(date_format) + ) bin_type = str.capitalize(" ".join(bin_info[1:])) # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string else: bin_date = get_next_occurrence_from_day_month( datetime.strptime( - bin_info[1] + " " + datetime.today().strftime("%Y"), - "%A %d %B - %Y", + bin_info[1], + "%A %d %B -", ) - ).strftime(date_format) + ) str.capitalize(" ".join(bin_info[2:])) except: continue + if (datetime.now().month == 12) and (bin_date.month == 1): + bin_date = bin_date.replace(year=next_year) + else: + bin_date = bin_date.replace(year=current_year) + # Build data dict for each entry dict_data = { "type": bin_type, - "collectionDate": bin_date, + "collectionDate": bin_date.strftime(date_format), } data["bins"].append(dict_data) diff --git a/uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py index 3f9f73a35c..86df3218f7 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ValeofWhiteHorseCouncil.py @@ -53,6 +53,9 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} + current_year = datetime.now().year + next_year = current_year + 1 + # Page has slider info side by side, which are two instances of this class for bin in soup.find_all("div", {"class": "bintxt"}): try: @@ -74,23 +77,31 @@ def parse_data(self, page: str, **kwargs) -> dict: if contains_date(bin_date_info[0]): bin_date = get_next_occurrence_from_day_month( datetime.strptime( - bin_date_info[0] + " " + datetime.today().strftime("%Y"), - "%A %d %B - %Y", + bin_date_info[0], + "%A %d %B -", ) - ).strftime(date_format) + ) # On exceptional collection schedule (e.g. around English Bank Holidays), date will be contained in the second stripped string else: bin_date = get_next_occurrence_from_day_month( datetime.strptime( - bin_date_info[1] + " " + datetime.today().strftime("%Y"), - "%A %d %B - %Y", + bin_date_info[1], + "%A %d %B -", ) - ).strftime(date_format) + ) except Exception as ex: raise ValueError(f"Error parsing bin data: {ex}") + if (datetime.now().month == 12) and (bin_date.month == 1): + bin_date = bin_date.replace(year=next_year) + else: + bin_date = bin_date.replace(year=current_year) + # Build data dict for each entry - dict_data = {"type": bin_type, "collectionDate": bin_date} + dict_data = { + "type": bin_type, + "collectionDate": bin_date.strftime(date_format), + } data["bins"].append(dict_data) data["bins"].sort(