From 2e483a376c4000c3ed36027abc30d39bbb9f02e7 Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 6 Jan 2024 11:36:30 +0000 Subject: [PATCH 1/5] feat: adding headless control --- uk_bin_collection/tests/input.json | 1 + .../step_defs/step_helpers/file_handler.py | 34 ++++---- .../tests/step_defs/test_validate_council.py | 78 ++++++++++--------- .../uk_bin_collection/collect_data.py | 51 ++++++------ uk_bin_collection/uk_bin_collection/common.py | 5 +- .../councils/BexleyCouncil.py | 3 +- .../councils/BlackburnCouncil.py | 3 +- .../councils/BoltonCouncil.py | 3 +- .../councils/BrightonandHoveCityCouncil.py | 3 +- .../councils/BromleyBoroughCouncil.py | 3 +- .../councils/BroxtoweBoroughCouncil.py | 3 +- .../councils/BuckinghamshireCouncil.py | 3 +- .../councils/CalderdaleCouncil.py | 3 +- .../councils/CastlepointDistrictCouncil.py | 3 +- .../councils/ChelmsfordCityCouncil.py | 3 +- .../DerbyshireDalesDistrictCouncil.py | 3 +- .../councils/EastLindseyDistrictCouncil.py | 3 +- .../councils/EastRidingCouncil.py | 3 +- .../councils/EastSuffolkCouncil.py | 3 +- .../councils/ForestOfDeanDistrictCouncil.py | 3 +- .../councils/GatesheadCouncil.py | 3 +- .../councils/GuildfordCouncil.py | 3 +- .../councils/HaltonBoroughCouncil.py | 3 +- .../councils/HighPeakCouncil.py | 3 +- .../councils/LeedsCityCouncil.py | 3 +- .../councils/LondonBoroughRedbridge.py | 3 +- .../MidAndEastAntrimBoroughCouncil.py | 3 +- .../councils/NeathPortTalbotCouncil.py | 3 +- .../NorthEastDerbyshireDistrictCouncil.py | 3 +- .../councils/NorthNorfolkDistrictCouncil.py | 3 +- .../councils/NorthWestLeicestershire.py | 3 +- .../councils/NorthumberlandCouncil.py | 3 +- .../councils/PortsmouthCityCouncil.py | 3 +- .../councils/PrestonCityCouncil.py | 3 +- .../ReigateAndBansteadBoroughCouncil.py | 3 +- .../councils/RushcliffeBoroughCouncil.py | 3 +- .../councils/SevenoaksDistrictCouncil.py | 3 +- .../StaffordshireMoorlandsDistrictCouncil.py | 3 +- .../councils/WestLothianCouncil.py | 3 +- .../councils/WestSuffolkCouncil.py | 3 +- .../uk_bin_collection/get_bin_data.py | 3 + 41 files changed, 161 insertions(+), 116 deletions(-) diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 40fd5b203a..0744577696 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -46,6 +46,7 @@ "web_driver": "http://selenium:4444", "postcode": "DA5 3AH", "uprn": "100020196143", + "headless": true, "house_number": "1 Dorchester Avenue, Bexley", "wiki_name": "Bexley Council", "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app) (available for [iOS](https://apps.apple.com/gb/app/home-collection-reminder/id1050703690) and [Android](https://play.google.com/store/apps/details?id=com.contender.athome.android)).\nComplete the setup by entering your email and setting your address with postcode and address line.\nOnce you can see the calendar, you _should_ be good to run the parser.\nJust pass the email you used in quotes in the UPRN parameter.\n" diff --git a/uk_bin_collection/tests/step_defs/step_helpers/file_handler.py b/uk_bin_collection/tests/step_defs/step_helpers/file_handler.py index 5bbe878ad9..9f26958f88 100644 --- a/uk_bin_collection/tests/step_defs/step_helpers/file_handler.py +++ b/uk_bin_collection/tests/step_defs/step_helpers/file_handler.py @@ -1,42 +1,38 @@ import json import logging -import os from jsonschema import validate, ValidationError +from pathlib import Path -logging.basicConfig(level=logging.INFO, format='%(levelname)s - %(message)s') +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") -def load_inputs_file(file_name): - cwd = os.getcwd() - with open(os.path.join(cwd, "uk_bin_collection", "tests", file_name)) as f: - data = json.load(f) - logging.info(f"{file_name} Input file loaded") - return data +# Dynamically compute the base path relative to this file's location +current_file_path = Path(__file__).resolve() +BASE_PATH = current_file_path.parent.parent.parent.parent / "tests" -def load_schema_file(file_name): - cwd = os.getcwd() - with open(os.path.join(cwd, "uk_bin_collection", "tests", file_name)) as f: +def load_json_file(file_name): + file_path = BASE_PATH / file_name + with open(file_path) as f: data = json.load(f) - logging.info(f"{file_name} Schema file loaded") + logging.info(f"{file_name} file loaded") return data def validate_json(json_str): try: - json.loads(json_str) + return json.loads(json_str) except ValueError as err: - logging.info(f"The following error occured {err}") - return False - return True + logging.error(f"JSON validation error: {err}") + raise def validate_json_schema(json_str, schema): - json_data = json.loads(json_str) + json_data = validate_json(json_str) try: validate(instance=json_data, schema=schema) except ValidationError as err: - logging.info(f"The following error occured {err}") + logging.error(f"Schema validation error: {err}") logging.info(f"Data: {json_str}") logging.info(f"Schema: {schema}") - return False + raise return True diff --git a/uk_bin_collection/tests/step_defs/test_validate_council.py b/uk_bin_collection/tests/step_defs/test_validate_council.py index d1c5946b2c..ddcc92ad66 100644 --- a/uk_bin_collection/tests/step_defs/test_validate_council.py +++ b/uk_bin_collection/tests/step_defs/test_validate_council.py @@ -3,17 +3,32 @@ import traceback from pytest_bdd import scenario, given, when, then, parsers from hamcrest import assert_that, equal_to +from functools import wraps from step_helpers import file_handler from uk_bin_collection.uk_bin_collection import collect_data -logging.basicConfig(level=logging.INFO, format='%(levelname)s - %(message)s') +logging.basicConfig(level=logging.INFO, format="%(levelname)s - %(message)s") + @scenario("../features/validate_council_outputs.feature", "Validate Council Output") def test_scenario_outline(): pass +def handle_test_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as e: + logging.error(f"Error in test '{func.__name__}': {e}") + logging.error(traceback.format_exc()) + raise e + + return wrapper + + @pytest.fixture def context(): class Context(object): @@ -22,18 +37,20 @@ class Context(object): return Context() +@handle_test_errors @given(parsers.parse("the council: {council_name}")) def get_council_step(context, council_name): - try: - council_input_data = file_handler.load_inputs_file("input.json") - context.metadata = council_input_data[council_name] - except Exception as err: - logging.error(traceback.format_exc()) - logging.info(f"Validate Output: {err}") - raise (err) + council_input_data = file_handler.load_json_file("input.json") + context.metadata = council_input_data[council_name] + # When we scrape the data from using and the is set. -@when(parsers.parse("we scrape the data from {council} using {selenium_mode} and the {selenium_url} is set")) +@handle_test_errors +@when( + parsers.parse( + "we scrape the data from {council} using {selenium_mode} and the {selenium_url} is set" + ) +) def scrape_step(context, council, selenium_mode, selenium_url): context.council = council context.selenium_mode = selenium_mode @@ -53,50 +70,35 @@ def scrape_step(context, council, selenium_mode, selenium_url): if "usrn" in context.metadata: usrn = context.metadata["usrn"] args.append(f"-us={usrn}") + if "headless" in context.metadata: + args.append(f"--headless") # TODO we should somehow run this test with and without this argument passed # TODO I do think this would make the testing of the councils a lot longer and cause a double hit from us # At the moment the feature file is set to local execution of the selenium so no url will be set # And it the behave test will execute locally - if selenium_mode != 'None' and selenium_url != 'None': - if selenium_mode != 'local': + if selenium_mode != "None" and selenium_url != "None": + if selenium_mode != "local": web_driver = context.metadata["web_driver"] args.append(f"-w={web_driver}") if "skip_get_url" in context.metadata: args.append(f"-s") - try: - CollectData = collect_data.UKBinCollectionApp() - CollectData.set_args(args) - context.parse_result = CollectData.run() - except Exception as err: - logging.error(traceback.format_exc()) - logging.info(f"Schema: {err}") - raise (err) + CollectData = collect_data.UKBinCollectionApp() + CollectData.set_args(args) + context.parse_result = CollectData.run() +@handle_test_errors @then("the result is valid json") def validate_json_step(context): - try: - valid_json = file_handler.validate_json(context.parse_result) - assert_that(valid_json, True) - except Exception as err: - logging.error(traceback.format_exc()) - logging.info(f"Validate Output: {err}") - logging.info(f"JSON Output: {context.parse_result}") - raise (err) + assert file_handler.validate_json(context.parse_result), "Invalid JSON output" +@handle_test_errors @then("the output should validate against the schema") def validate_output_step(context): - try: - council_schema = file_handler.load_schema_file(f"output.schema") - schema_result = file_handler.validate_json_schema( - context.parse_result, council_schema - ) - assert_that(schema_result, True) - except Exception as err: - logging.error(traceback.format_exc()) - logging.info(f"Validate Output: {err}") - logging.info(f"JSON Output: {context.parse_result}") - raise (err) + council_schema = file_handler.load_json_file(f"output.schema") + assert file_handler.validate_json_schema( + context.parse_result, council_schema + ), "Schema validation failed" diff --git a/uk_bin_collection/uk_bin_collection/collect_data.py b/uk_bin_collection/uk_bin_collection/collect_data.py index b89af8a4bf..298c6b16e2 100644 --- a/uk_bin_collection/uk_bin_collection/collect_data.py +++ b/uk_bin_collection/uk_bin_collection/collect_data.py @@ -10,15 +10,21 @@ _LOGGER = logging.getLogger(__name__) -# We use this method to dynamically import the council processor -SRC_PATH = os.path.join("councils") -module_path = os.path.realpath(os.path.join(os.path.dirname(__file__), SRC_PATH)) -sys.path.append(module_path) +# Dynamically importing the council processor +def import_council_module(module_name, src_path="councils"): + module_path = os.path.realpath(os.path.join(os.path.dirname(__file__), src_path)) + if module_path not in sys.path: + sys.path.append(module_path) + return importlib.import_module(module_name) class UKBinCollectionApp: def __init__(self): - self.parser = argparse.ArgumentParser(description="") + self.setup_arg_parser() + self.parsed_args = None + + def setup_arg_parser(self): + self.parser = argparse.ArgumentParser(description="UK Bin Collection Data Parser") self.parser.add_argument( "module", type=str, help="Name of council module to use" ) @@ -52,6 +58,13 @@ def __init__(self): help="URL for remote Selenium web driver - should be wrapped in double quotes", required=False, ) + self.parser.add_argument( + "--headless", + action="store_true", + help="Should Selenium be headless. Defaults to true. Can be set to false to debug council", + required=False, + ) + self.parser.add_argument( "-d", "--dev_mode", @@ -64,26 +77,16 @@ def __init__(self): def set_args(self, args): self.parsed_args = self.parser.parse_args(args) - def get_council_module(self, council_module_str): - return importlib.import_module(council_module_str) - - def client_code(self, get_bin_data_class, address_url, **kwargs) -> None: - """ - The client code calls the template method to execute the algorithm. Client - code does not have to know the concrete class of an object it works with, - as long as it works with objects through the interface of their base class. - """ - return get_bin_data_class.template_method(address_url, **kwargs) - def run(self): council_module_str = self.parsed_args.module + council_module = import_council_module(council_module_str) address_url = self.parsed_args.URL - council_module = self.get_council_module(council_module_str) postcode = self.parsed_args.postcode paon = self.parsed_args.number uprn = self.parsed_args.uprn skip_get_url = self.parsed_args.skip_get_url web_driver = self.parsed_args.web_driver + headless = self.parsed_args.headless dev_mode = self.parsed_args.dev_mode return self.client_code( @@ -94,17 +97,21 @@ def run(self): uprn=uprn, skip_get_url=skip_get_url, web_driver=web_driver, + headless=headless, dev_mode=dev_mode, council_module_str=council_module_str, ) + def client_code(self, get_bin_data_class, address_url, **kwargs) -> None: + """ + The client code calls the template method to execute the algorithm. Client + code does not have to know the concrete class of an object it works with, + as long as it works with objects through the interface of their base class. + """ + return get_bin_data_class.template_method(address_url, **kwargs) if __name__ == "__main__": - import sys - _LOGGER = setup_logging(LOGGING_CONFIG, None) - app = UKBinCollectionApp() app.set_args(sys.argv[1:]) - data = app.run() - print(data) + print(app.run()) \ No newline at end of file diff --git a/uk_bin_collection/uk_bin_collection/common.py b/uk_bin_collection/uk_bin_collection/common.py index 63a451b87b..93f2b2db65 100644 --- a/uk_bin_collection/uk_bin_collection/common.py +++ b/uk_bin_collection/uk_bin_collection/common.py @@ -261,14 +261,15 @@ def contains_date(string, fuzzy=False) -> bool: return False -def create_webdriver(web_driver) -> webdriver.Chrome: +def create_webdriver(web_driver: str, headless: bool) -> webdriver.Chrome: """ Create and return a headless Selenium webdriver :rtype: webdriver.Chrome """ # Set up Selenium to run 'headless' options = webdriver.ChromeOptions() - options.add_argument("--headless") + if headless is None: + options.add_argument("--headless") options.add_argument("--no-sandbox") options.add_argument("--disable-gpu") options.add_argument("--start-maximized") diff --git a/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py index 3f9bec836b..3a71a69f8c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py @@ -30,9 +30,10 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py index a650b53dce..97b8c9b638 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py @@ -39,13 +39,14 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} uprn = kwargs.get("uprn") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") current_month = datetime.today().strftime("%m") current_year = datetime.today().strftime("%Y") url = ( f"https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn={uprn}&month={current_month}" f"&year={current_year}" ) - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(url) soup = BeautifulSoup(driver.page_source, "html.parser") diff --git a/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py index 568705c9f0..5c3888a918 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py @@ -25,13 +25,14 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") check_postcode(user_postcode) web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") data = {"bins": []} # Get our initial session running page = "https://carehomes.bolton.gov.uk/bins.aspx" - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py index b923ecd6b5..558e767b2e 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py @@ -32,7 +32,8 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - driver = create_webdriver(web_driver) + headless= kwargs.get("headless") + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py index f601e3a7d9..e487cc2dfa 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py @@ -29,11 +29,12 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_data_dict = {"bins": []} collections = [] web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") data = {"bins": []} # Get our initial session running - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py index 0fee519c89..f0a6d0b009 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py index 0adaea3547..63f559f4de 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py @@ -37,9 +37,10 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Enter postcode in text box and wait diff --git a/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py index e138ab9bc0..b73e064f1b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py @@ -33,11 +33,12 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_data_dict = {"bins": []} collections = [] web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") data = {"bins": []} # Get our initial session running - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py index 7dbdb0578e..fdaa06aa70 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py @@ -31,7 +31,8 @@ def parse_data(self, page: str, **kwargs) -> dict: uprn = kwargs.get("uprn") check_uprn(uprn) web_driver = kwargs.get("web_driver") - driver = create_webdriver(web_driver) + headless= kwargs.get("headless") + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py index 29d2c2bf06..4e94a6e285 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py @@ -32,7 +32,8 @@ def parse_data(self, page: str, **kwargs) -> dict: postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") - driver = create_webdriver(web_driver) + headless= kwargs.get("headless") + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 120) diff --git a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py index 83b4d8a08d..e35e11c824 100644 --- a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py index 50b91fe35b..796364c9da 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get( "https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py index 0b3095fcdd..95bccda22d 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py @@ -25,11 +25,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver page = f"https://www.eastriding.gov.uk/environment/bins-rubbish-recycling/bins-and-collections/bin-collection-dates/" - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index 3b1040b0c7..857247b900 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get("https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder") # Wait for iframe to load and switch to it diff --git a/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py index 2ea07311b9..309ef13925 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py @@ -30,9 +30,10 @@ def parse_data(self, page: str, **kwargs) -> dict: postcode = kwargs.get("postcode") full_address = f"{house_number}, {postcode}" web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py index 59ed7a7669..c29da26896 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get( "https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py index 18f237cdc7..efe0fb0fc0 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py @@ -32,8 +32,9 @@ def parse_data(self, page: str, **kwargs) -> dict: url = "https://my.guildford.gov.uk/customers/s/view-bin-collections" web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 120) diff --git a/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py index da8e1d7551..c4935e17d5 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py @@ -25,10 +25,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver page = f"https://webapp.halton.gov.uk/PublicWebForms/WasteServiceSearchv1.aspx" - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py index 5e2039cc24..a1895f4603 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py @@ -61,9 +61,10 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Hide Cookies diff --git a/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py index 88e231cffc..29663230c3 100644 --- a/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py @@ -29,6 +29,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver @@ -36,7 +37,7 @@ def parse_data(self, page: str, **kwargs) -> dict: f"https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day" ) - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py b/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py index bda2cd68d6..3bc52d06e8 100644 --- a/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py +++ b/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py @@ -31,7 +31,8 @@ def parse_data(self, page: str, **kwargs) -> dict: uprn = kwargs.get("uprn") postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - driver = create_webdriver(web_driver) + headless= kwargs.get("headless") + driver = create_webdriver(web_driver,headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py index fca94f0f8a..2f3c1f418b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") # not used: user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") # Create Selenium webdriver options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-logging"]) - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) diff --git a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py index e5c094a9d3..b94efc35ac 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py @@ -22,11 +22,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get("https://www.npt.gov.uk/2195") # Accept cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py index 971a550d3a..ac87e014d7 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py @@ -27,10 +27,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py index 7d3cc863a9..5e7c664e1c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py b/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py index 0ec0858436..3103c1b1cb 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py @@ -25,6 +25,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver @@ -32,7 +33,7 @@ def parse_data(self, page: str, **kwargs) -> dict: f"https://my.nwleics.gov.uk/my-property-finder?address={user_postcode}&go=1" ) - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py index 9fc7d4e6fd..46e73c4a8a 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py @@ -32,11 +32,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) time.sleep(1) diff --git a/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py index 058dd4e839..a540a06468 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py @@ -28,10 +28,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py index b12464e572..a858f8140f 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py @@ -28,11 +28,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py index 66bc83474a..5f09957045 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py @@ -18,12 +18,13 @@ class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) # Pad UPRN with 0's at the start for any that aren't 12 chars user_uprn = user_uprn.zfill(12) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get( f"https://my.reigate-banstead.gov.uk/en/service/Bins_and_recycling___collections_calendar?uprn={user_uprn}" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index ccd8b623c0..bc3c77f092 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -24,11 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py index 6933d4c554..5fc3051bff 100644 --- a/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py @@ -22,6 +22,7 @@ def wait_for_element_conditions(self, driver, conditions, timeout: int = 5): def parse_data(self, page: str, **kwargs) -> dict: web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") page = "https://sevenoaks-dc-host01.oncreate.app/w/webpage/waste-collection-day" # Assign user info @@ -29,7 +30,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Enter postcode diff --git a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py index b69306b526..25f728c7d2 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py @@ -21,11 +21,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get("https://www.staffsmoorlands.gov.uk/findyourbinday") # Close cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py index c68d5fb402..4a339ccd3c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py @@ -20,11 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get( "https://www.westlothian.gov.uk/article/31528/Bin-Collection-Calendar-Dates" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py index bb9521b96c..5fbd4f8a24 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py @@ -23,6 +23,7 @@ def wait_for_element_conditions(self, driver, conditions, timeout: int = 5): def parse_data(self, page: str, **kwargs) -> dict: web_driver = kwargs.get("web_driver") + headless= kwargs.get("headless") page = "https://westsuffolk-self.achieveservice.com/service/WSS_EX_Inf_Bin_Collection_Postcode_Lookup" # Assign user info @@ -30,7 +31,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") # Create Selenium webdriver - driver = create_webdriver(web_driver) + driver = create_webdriver(web_driver,headless) driver.get(page) # Click the cookie button diff --git a/uk_bin_collection/uk_bin_collection/get_bin_data.py b/uk_bin_collection/uk_bin_collection/get_bin_data.py index 282db47f6a..735f65c77c 100644 --- a/uk_bin_collection/uk_bin_collection/get_bin_data.py +++ b/uk_bin_collection/uk_bin_collection/get_bin_data.py @@ -55,6 +55,7 @@ def template_method(self, address_url: str, **kwargs) -> None: # pragma: no cov this_uprn = kwargs.get("uprn", None) this_usrn = kwargs.get("usrn", None) this_web_driver = kwargs.get("web_driver", None) + this_headless = kwargs.get("headless", None) skip_get_url = kwargs.get("skip_get_url", None) dev_mode = kwargs.get("dev_mode", False) council_module_str = kwargs.get("council_module_str", None) @@ -69,6 +70,7 @@ def template_method(self, address_url: str, **kwargs) -> None: # pragma: no cov uprn=this_uprn, usrn=this_usrn, web_driver=this_web_driver, + headless=this_headless, url=this_url, ) json_output = self.output_json(bin_data_dict) @@ -80,6 +82,7 @@ def template_method(self, address_url: str, **kwargs) -> None: # pragma: no cov uprn=this_uprn, usrn=this_usrn, web_driver=this_web_driver, + headless=this_headless, url=this_url, ) json_output = self.output_json(bin_data_dict) From c810759d65dfc5144d3916d5ba9c2e6de093536b Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 6 Jan 2024 12:01:31 +0000 Subject: [PATCH 2/5] feat: adding headless control --- .../uk_bin_collection/config_flow.py | 13 ++-- custom_components/uk_bin_collection/const.py | 2 +- custom_components/uk_bin_collection/sensor.py | 33 ++++++--- uk_bin_collection/tests/input.json | 1 - .../tests/test_common_functions.py | 72 +++++++++++++------ .../uk_bin_collection/collect_data.py | 10 ++- uk_bin_collection/uk_bin_collection/common.py | 7 +- .../councils/BexleyCouncil.py | 2 +- .../councils/BlackburnCouncil.py | 4 +- .../councils/BoltonCouncil.py | 4 +- .../councils/BrightonandHoveCityCouncil.py | 4 +- .../councils/BromleyBoroughCouncil.py | 4 +- .../councils/BroxtoweBoroughCouncil.py | 4 +- .../councils/BuckinghamshireCouncil.py | 4 +- .../councils/CalderdaleCouncil.py | 4 +- .../councils/CastlepointDistrictCouncil.py | 4 +- .../councils/ChelmsfordCityCouncil.py | 8 +-- .../councils/CheshireEastCouncil.py | 14 ++-- .../councils/ConwyCountyBorough.py | 13 ++-- .../DerbyshireDalesDistrictCouncil.py | 4 +- .../councils/EastLindseyDistrictCouncil.py | 4 +- .../councils/EastRidingCouncil.py | 4 +- .../councils/EastSuffolkCouncil.py | 4 +- .../councils/ForestOfDeanDistrictCouncil.py | 4 +- .../councils/GatesheadCouncil.py | 4 +- .../councils/GuildfordCouncil.py | 4 +- .../councils/HaltonBoroughCouncil.py | 4 +- .../councils/HighPeakCouncil.py | 4 +- .../councils/LeedsCityCouncil.py | 4 +- .../councils/LondonBoroughRedbridge.py | 4 +- .../MidAndEastAntrimBoroughCouncil.py | 4 +- .../councils/NeathPortTalbotCouncil.py | 4 +- .../NorthEastDerbyshireDistrictCouncil.py | 4 +- .../councils/NorthNorfolkDistrictCouncil.py | 4 +- .../councils/NorthWestLeicestershire.py | 4 +- .../councils/NorthumberlandCouncil.py | 4 +- .../councils/NottinghamCityCouncil.py | 4 +- .../councils/PortsmouthCityCouncil.py | 4 +- .../councils/PrestonCityCouncil.py | 4 +- .../ReigateAndBansteadBoroughCouncil.py | 4 +- .../councils/RushcliffeBoroughCouncil.py | 4 +- .../councils/SevenoaksDistrictCouncil.py | 4 +- .../StaffordshireMoorlandsDistrictCouncil.py | 4 +- .../councils/WestLothianCouncil.py | 4 +- .../councils/WestSuffolkCouncil.py | 4 +- .../councilclasstemplate.py | 7 +- wiki/generate_wiki.py | 35 +++++---- 47 files changed, 210 insertions(+), 143 deletions(-) diff --git a/custom_components/uk_bin_collection/config_flow.py b/custom_components/uk_bin_collection/config_flow.py index c70e68390b..2008ff6317 100644 --- a/custom_components/uk_bin_collection/config_flow.py +++ b/custom_components/uk_bin_collection/config_flow.py @@ -29,8 +29,10 @@ async def get_council_schema(self, council=str) -> vol.Schema: if self.councils_data is None: self.councils_data = await self.get_councils_json() council_schema = vol.Schema({}) - if ("skip_get_url" not in self.councils_data[council] or - "custom_component_show_url_field" in self.councils_data[council]): + if ( + "skip_get_url" not in self.councils_data[council] + or "custom_component_show_url_field" in self.councils_data[council] + ): council_schema = council_schema.extend( {vol.Required("url", default=""): cv.string} ) @@ -54,6 +56,9 @@ async def get_council_schema(self, council=str) -> vol.Schema: council_schema = council_schema.extend( {vol.Required("web_driver", default=""): cv.string} ) + council_schema = council_schema.extend( + {vol.Required("headless", default=True): cv.boolean} + ) return council_schema async def async_step_user(self, user_input=None): @@ -116,9 +121,7 @@ async def async_step_council(self, user_input=None): # Create the config entry _LOGGER.info(LOG_PREFIX + "Creating config entry with data: %s", user_input) - return self.async_create_entry( - title=user_input["name"], data=user_input - ) + return self.async_create_entry(title=user_input["name"], data=user_input) # Show the configuration form to the user with the specific councils necessary fields council_schema = await self.get_council_schema(self.data["council"]) diff --git a/custom_components/uk_bin_collection/const.py b/custom_components/uk_bin_collection/const.py index 6a42c4cc7a..b7c491f3bf 100644 --- a/custom_components/uk_bin_collection/const.py +++ b/custom_components/uk_bin_collection/const.py @@ -15,4 +15,4 @@ STATE_ATTR_NEXT_COLLECTION = "next_collection" STATE_ATTR_DAYS = "days" -DEVICE_CLASS = "bin_collection_schedule" \ No newline at end of file +DEVICE_CLASS = "bin_collection_schedule" diff --git a/custom_components/uk_bin_collection/sensor.py b/custom_components/uk_bin_collection/sensor.py index de82d89193..f4eb0033d9 100644 --- a/custom_components/uk_bin_collection/sensor.py +++ b/custom_components/uk_bin_collection/sensor.py @@ -73,6 +73,7 @@ async def async_setup_entry( for bin_type in coordinator.data.keys() ) + def get_latest_collection_info(data) -> dict: # Get the current date current_date = datetime.now() @@ -92,14 +93,17 @@ def get_latest_collection_info(data) -> dict: if collection_date.date() >= current_date.date(): # If the bin type is in the dict, update its collection date if needed; otherwise, add it if bin_type in next_collection_dates: - if collection_date < datetime.strptime(next_collection_dates[bin_type], "%d/%m/%Y"): + if collection_date < datetime.strptime( + next_collection_dates[bin_type], "%d/%m/%Y" + ): next_collection_dates[bin_type] = collection_date_str else: next_collection_dates[bin_type] = collection_date_str - + _LOGGER.info(f"{LOG_PREFIX} Next Collection Dates: {next_collection_dates}") return next_collection_dates + class HouseholdBinCoordinator(DataUpdateCoordinator): """Household Bin Coordinator""" @@ -124,8 +128,10 @@ async def _async_update_data(self): _LOGGER.info(f"{LOG_PREFIX} UKBinCollectionApp: {data}") if cm.expired: - _LOGGER.warning(f"{LOG_PREFIX} UKBinCollectionApp timeout expired during run") - + _LOGGER.warning( + f"{LOG_PREFIX} UKBinCollectionApp timeout expired during run" + ) + return get_latest_collection_info(json.loads(data)) @@ -162,7 +168,9 @@ def apply_values(self): name = "{} {}".format(self.coordinator.name, self._bin_type) self._id = name self._name = name - self._next_collection = parser.parse(self.coordinator.data[self._bin_type], dayfirst=True).date() + self._next_collection = parser.parse( + self.coordinator.data[self._bin_type], dayfirst=True + ).date() self._hidden = False self._icon = "mdi:trash-can" self._colour = "red" @@ -179,16 +187,22 @@ def apply_values(self): next_week_start = this_week_end + timedelta(days=1) next_week_end = next_week_start + timedelta(days=6) - self._days = (self._next_collection- now.date()).days + self._days = (self._next_collection - now.date()).days _LOGGER.info(f"{LOG_PREFIX} _days: {self._days}") if self._next_collection == now.date(): self._state = "Today" elif self._next_collection == (now + timedelta(days=1)).date(): self._state = "Tomorrow" - elif self._next_collection >= this_week_start and self._next_collection <= this_week_end: + elif ( + self._next_collection >= this_week_start + and self._next_collection <= this_week_end + ): self._state = f"This Week: {self._next_collection.strftime('%A')}" - elif self._next_collection >= next_week_start and self._next_collection <= next_week_end: + elif ( + self._next_collection >= next_week_start + and self._next_collection <= next_week_end + ): self._state = f"Next Week: {self._next_collection.strftime('%A')}" elif self._next_collection > next_week_end: self._state = f"Future: {self._next_collection}" @@ -199,7 +213,6 @@ def apply_values(self): _LOGGER.info(f"{LOG_PREFIX} State of the sensor: {self._state}") - @property def name(self): """Return the name of the bin.""" @@ -239,7 +252,7 @@ def colour(self): def unique_id(self): """Return a unique ID to use for this sensor.""" return self._id - + @property def bin_type(self): """Return the bin type.""" diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index ad2fd0e988..997daf7531 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -54,7 +54,6 @@ "web_driver": "http://selenium:4444", "postcode": "DA5 3AH", "uprn": "100020196143", - "headless": true, "house_number": "1 Dorchester Avenue, Bexley", "wiki_name": "Bexley Council", "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app) (available for [iOS](https://apps.apple.com/gb/app/home-collection-reminder/id1050703690) and [Android](https://play.google.com/store/apps/details?id=com.contender.athome.android)).\nComplete the setup by entering your email and setting your address with postcode and address line.\nOnce you can see the calendar, you _should_ be good to run the parser.\nJust pass the email you used in quotes in the UPRN parameter.\n" diff --git a/uk_bin_collection/tests/test_common_functions.py b/uk_bin_collection/tests/test_common_functions.py index fc2b991624..41d3a18822 100644 --- a/uk_bin_collection/tests/test_common_functions.py +++ b/uk_bin_collection/tests/test_common_functions.py @@ -75,7 +75,7 @@ def test_get_date_with_ordinal_exception(): result = get_date_with_ordinal(date_number) assert exc_info.type == TypeError assert ( - exc_info.value.args[0] == "not all arguments converted during string formatting" + exc_info.value.args[0] == "not all arguments converted during string formatting" ) @@ -91,11 +91,13 @@ def test_is_holiday(): result = is_holiday("2022, 12, 25") assert result is True + def test_is_holiday(): date = "20" result = is_holiday("2022, 12, 25") assert result is True + def test_is_holiday_region(): date = "20" result = is_holiday("2022, 12, 25", Region.WLS) @@ -153,52 +155,82 @@ def test_get_weekday_dates_in_period_bad(): assert len(result) != 8 assert result[6] != "08/04/20232" + def test_get_next_occurrence_from_day_month_false(): - result = get_next_occurrence_from_day_month(datetime(2023,12,1)) + result = get_next_occurrence_from_day_month(datetime(2023, 12, 1)) assert result == datetime(2023, 12, 1, 0, 0) + def test_get_next_occurrence_from_day_month_true(): - result = get_next_occurrence_from_day_month(datetime(2023,9,1)) - assert result == pd.Timestamp('2024-09-01 00:00:00') + result = get_next_occurrence_from_day_month(datetime(2023, 9, 1)) + assert result == pd.Timestamp("2024-09-01 00:00:00") + def test_update_input_json(): council = "test_council" url = "TEST_URL" - postcode="TEST_POSTCODE" - uprn="TEST_UPRN" - web_driver="TEST_WEBDRIVER" + postcode = "TEST_POSTCODE" + uprn = "TEST_UPRN" + web_driver = "TEST_WEBDRIVER" skip_get_url = True - update_input_json(council, url, postcode=postcode, uprn=uprn, web_driver=web_driver, skip_get_url=skip_get_url) + update_input_json( + council, + url, + postcode=postcode, + uprn=uprn, + web_driver=web_driver, + skip_get_url=skip_get_url, + ) cwd = os.getcwd() input_file_path = os.path.join(cwd, "uk_bin_collection", "tests", "input.json") - result1 = os.path.exists(input_file_path) - with open(input_file_path, 'r') as f: - data = json.load(f) + result1 = os.path.exists(input_file_path) + with open(input_file_path, "r") as f: + data = json.load(f) assert result1 == True - assert data[council] == {"postcode": postcode, "skip_get_url": skip_get_url, "uprn": uprn, "url": url, "web_driver": web_driver, "wiki_name": council} + assert data[council] == { + "postcode": postcode, + "skip_get_url": skip_get_url, + "uprn": uprn, + "url": url, + "web_driver": web_driver, + "wiki_name": council, + } + def test_update_input_json_fail(capsys, monkeypatch): def mock_os_path_exists(path): return False # Simulate the path not existing - monkeypatch.setattr(os.path, 'exists', mock_os_path_exists) + monkeypatch.setattr(os.path, "exists", mock_os_path_exists) council = "test_council" url = "TEST_URL" - postcode="TEST_POSTCODE" - uprn="TEST_UPRN" - web_driver="TEST_WEBDRIVER" + postcode = "TEST_POSTCODE" + uprn = "TEST_UPRN" + web_driver = "TEST_WEBDRIVER" skip_get_url = True - update_input_json(council, url, postcode=postcode, uprn=uprn, web_driver=web_driver, skip_get_url=skip_get_url) + update_input_json( + council, + url, + postcode=postcode, + uprn=uprn, + web_driver=web_driver, + skip_get_url=skip_get_url, + ) captured = capsys.readouterr() - assert "Exception encountered: Unable to update input.json file for the council." in captured.out + assert ( + "Exception encountered: Unable to update input.json file for the council." + in captured.out + ) assert "Please check you're running developer mode" in captured.out + def test_create_webdriver_local(): result = create_webdriver(None) - assert result.name == 'chrome' + assert result.name == "chrome" + def test_create_webdriver_remote(): result = create_webdriver("http://selenium:4444") - assert result.name == 'chrome' + assert result.name == "chrome" diff --git a/uk_bin_collection/uk_bin_collection/collect_data.py b/uk_bin_collection/uk_bin_collection/collect_data.py index 298c6b16e2..1b439f467f 100755 --- a/uk_bin_collection/uk_bin_collection/collect_data.py +++ b/uk_bin_collection/uk_bin_collection/collect_data.py @@ -10,6 +10,7 @@ _LOGGER = logging.getLogger(__name__) + # Dynamically importing the council processor def import_council_module(module_name, src_path="councils"): module_path = os.path.realpath(os.path.join(os.path.dirname(__file__), src_path)) @@ -24,7 +25,9 @@ def __init__(self): self.parsed_args = None def setup_arg_parser(self): - self.parser = argparse.ArgumentParser(description="UK Bin Collection Data Parser") + self.parser = argparse.ArgumentParser( + description="UK Bin Collection Data Parser" + ) self.parser.add_argument( "module", type=str, help="Name of council module to use" ) @@ -60,7 +63,7 @@ def setup_arg_parser(self): ) self.parser.add_argument( "--headless", - action="store_true", + action="store_false", help="Should Selenium be headless. Defaults to true. Can be set to false to debug council", required=False, ) @@ -110,8 +113,9 @@ def client_code(self, get_bin_data_class, address_url, **kwargs) -> None: """ return get_bin_data_class.template_method(address_url, **kwargs) + if __name__ == "__main__": _LOGGER = setup_logging(LOGGING_CONFIG, None) app = UKBinCollectionApp() app.set_args(sys.argv[1:]) - print(app.run()) \ No newline at end of file + print(app.run()) diff --git a/uk_bin_collection/uk_bin_collection/common.py b/uk_bin_collection/uk_bin_collection/common.py index 93f2b2db65..2981500511 100644 --- a/uk_bin_collection/uk_bin_collection/common.py +++ b/uk_bin_collection/uk_bin_collection/common.py @@ -246,6 +246,7 @@ def validate_dates(bin_dates: dict) -> dict: raise NotImplementedError() # If a date is in December and the next is in January, increase the year + def contains_date(string, fuzzy=False) -> bool: """ Return whether the string can be interpreted as a date. @@ -253,7 +254,7 @@ def contains_date(string, fuzzy=False) -> bool: :param string: str, string to check for date :param fuzzy: bool, ignore unknown tokens in string if True """ - try: + try: parse(string, fuzzy=fuzzy) return True @@ -268,8 +269,8 @@ def create_webdriver(web_driver: str, headless: bool) -> webdriver.Chrome: """ # Set up Selenium to run 'headless' options = webdriver.ChromeOptions() - if headless is None: - options.add_argument("--headless") + if headless is True: + options.add_argument("--headless=new") options.add_argument("--no-sandbox") options.add_argument("--disable-gpu") options.add_argument("--start-maximized") diff --git a/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py index 3a71a69f8c..09bda96107 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BexleyCouncil.py @@ -30,7 +30,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver driver = create_webdriver(web_driver, headless) diff --git a/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py index 97b8c9b638..42e266295d 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BlackburnCouncil.py @@ -39,14 +39,14 @@ def parse_data(self, page: str, **kwargs) -> dict: data = {"bins": []} uprn = kwargs.get("uprn") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") current_month = datetime.today().strftime("%m") current_year = datetime.today().strftime("%Y") url = ( f"https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn={uprn}&month={current_month}" f"&year={current_year}" ) - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(url) soup = BeautifulSoup(driver.page_source, "html.parser") diff --git a/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py index 5c3888a918..95b4d7cffa 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BoltonCouncil.py @@ -25,14 +25,14 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") check_postcode(user_postcode) web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") data = {"bins": []} # Get our initial session running page = "https://carehomes.bolton.gov.uk/bins.aspx" - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py index 558e767b2e..e10e64d187 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BrightonandHoveCityCouncil.py @@ -32,8 +32,8 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") - driver = create_webdriver(web_driver,headless) + headless = kwargs.get("headless") + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py index e487cc2dfa..48d83696af 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BromleyBoroughCouncil.py @@ -29,12 +29,12 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_data_dict = {"bins": []} collections = [] web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") data = {"bins": []} # Get our initial session running - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py index f0a6d0b009..8d573778fd 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BroxtoweBoroughCouncil.py @@ -24,12 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py index 63f559f4de..b767a37e7b 100644 --- a/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/BuckinghamshireCouncil.py @@ -37,10 +37,10 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Enter postcode in text box and wait diff --git a/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py index b73e064f1b..78337fe616 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CalderdaleCouncil.py @@ -33,12 +33,12 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_data_dict = {"bins": []} collections = [] web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") data = {"bins": []} # Get our initial session running - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py index fdaa06aa70..3a83f201c8 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CastlepointDistrictCouncil.py @@ -31,8 +31,8 @@ def parse_data(self, page: str, **kwargs) -> dict: uprn = kwargs.get("uprn") check_uprn(uprn) web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") - driver = create_webdriver(web_driver,headless) + headless = kwargs.get("headless") + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 30) diff --git a/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py index 3cba1b3c26..158b9ca719 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ChelmsfordCityCouncil.py @@ -32,12 +32,12 @@ def parse_data(self, page: str, **kwargs) -> dict: postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") - driver = create_webdriver(web_driver,headless) - driver.get(kwargs.get("url")) + headless = kwargs.get("headless") + driver = create_webdriver(web_driver, headless) + url = kwargs.get("url") driver.execute_script(f"window.location.href='{url}'") - + wait = WebDriverWait(driver, 120) post_code_search = wait.until( EC.presence_of_element_located((By.XPATH, '//input[@name="keyword"]')) diff --git a/uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py b/uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py index 3eb8b08348..711655b662 100644 --- a/uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/CheshireEastCouncil.py @@ -13,16 +13,20 @@ def parse_data(self, page: str, **kwargs) -> dict: rows = table.find_all("tr", {"class": "data-row"}) for row in rows: - cells = row.find_all("td", {"class": lambda L: L and L.startswith("visible-cell")}) + cells = row.find_all( + "td", {"class": lambda L: L and L.startswith("visible-cell")} + ) labels = cells[0].find_all("label") if cells else [] if len(labels) >= 3: bin_type = labels[2].get_text(strip=True) collection_date = labels[1].get_text(strip=True) - bin_data_dict["bins"].append({ - "type": bin_type, - "collectionDate": collection_date, - }) + bin_data_dict["bins"].append( + { + "type": bin_type, + "collectionDate": collection_date, + } + ) return bin_data_dict diff --git a/uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py b/uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py index dcf6b0df49..ee1fc645f3 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py +++ b/uk_bin_collection/uk_bin_collection/councils/ConwyCountyBorough.py @@ -3,6 +3,7 @@ from uk_bin_collection.uk_bin_collection.common import * from datetime import datetime + class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: soup = BeautifulSoup(page.text, features="html.parser") @@ -14,11 +15,13 @@ def parse_data(self, page: str, **kwargs) -> dict: bin_types = bin_section.find(id="main1").findAll("li") for bin_type in bin_types: - bin_type_name = bin_type.text.split('(')[0].strip() + bin_type_name = bin_type.text.split("(")[0].strip() - data["bins"].append({ - "type": bin_type_name, - "collectionDate": collection_date.strftime(date_format), - }) + data["bins"].append( + { + "type": bin_type_name, + "collectionDate": collection_date.strftime(date_format), + } + ) return data diff --git a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py index e35e11c824..61c251462d 100644 --- a/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/DerbyshireDalesDistrictCouncil.py @@ -24,12 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py index 796364c9da..4651f7ed03 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastLindseyDistrictCouncil.py @@ -20,12 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get( "https://www.e-lindsey.gov.uk/article/6714/Your-Waste-Collection-Days" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py index 95bccda22d..4743d44e74 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastRidingCouncil.py @@ -25,12 +25,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver page = f"https://www.eastriding.gov.uk/environment/bins-rubbish-recycling/bins-and-collections/bin-collection-dates/" - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py index 857247b900..066a118a7d 100644 --- a/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/EastSuffolkCouncil.py @@ -20,12 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get("https://my.eastsuffolk.gov.uk/service/Bin_collection_dates_finder") # Wait for iframe to load and switch to it diff --git a/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py index 309ef13925..7e98093f0c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ForestOfDeanDistrictCouncil.py @@ -30,10 +30,10 @@ def parse_data(self, page: str, **kwargs) -> dict: postcode = kwargs.get("postcode") full_address = f"{house_number}, {postcode}" web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py index c29da26896..c4ea411a87 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GatesheadCouncil.py @@ -20,12 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get( "https://www.gateshead.gov.uk/article/3150/Bin-collection-day-checker" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py b/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py index efe0fb0fc0..f47f34e6cc 100644 --- a/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/GuildfordCouncil.py @@ -32,9 +32,9 @@ def parse_data(self, page: str, **kwargs) -> dict: url = "https://my.guildford.gov.uk/customers/s/view-bin-collections" web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 120) diff --git a/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py index c4935e17d5..87dd79a2f2 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HaltonBoroughCouncil.py @@ -25,11 +25,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver page = f"https://webapp.halton.gov.uk/PublicWebForms/WasteServiceSearchv1.aspx" - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py index a1895f4603..a05f232c0f 100644 --- a/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/HighPeakCouncil.py @@ -61,10 +61,10 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Hide Cookies diff --git a/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py index 29663230c3..822de2a3b7 100644 --- a/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/LeedsCityCouncil.py @@ -29,7 +29,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver @@ -37,7 +37,7 @@ def parse_data(self, page: str, **kwargs) -> dict: f"https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day" ) - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py b/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py index 3bc52d06e8..407c0456cc 100644 --- a/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py +++ b/uk_bin_collection/uk_bin_collection/councils/LondonBoroughRedbridge.py @@ -31,8 +31,8 @@ def parse_data(self, page: str, **kwargs) -> dict: uprn = kwargs.get("uprn") postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") - driver = create_webdriver(web_driver,headless) + headless = kwargs.get("headless") + driver = create_webdriver(web_driver, headless) driver.get(kwargs.get("url")) wait = WebDriverWait(driver, 60) diff --git a/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py index 2f3c1f418b..28e153d487 100644 --- a/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/MidAndEastAntrimBoroughCouncil.py @@ -24,12 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_postcode = kwargs.get("postcode") # not used: user_paon = kwargs.get("paon") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") # Create Selenium webdriver options = webdriver.ChromeOptions() options.add_experimental_option("excludeSwitches", ["enable-logging"]) - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) diff --git a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py index b94efc35ac..0d01f9c20c 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NeathPortTalbotCouncil.py @@ -22,12 +22,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get("https://www.npt.gov.uk/2195") # Accept cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py index ac87e014d7..c8b552d1d6 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthEastDerbyshireDistrictCouncil.py @@ -27,11 +27,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py index 5e7c664e1c..19c6e4c337 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthNorfolkDistrictCouncil.py @@ -24,12 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py b/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py index 3103c1b1cb..b1c5b811dc 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthWestLeicestershire.py @@ -25,7 +25,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver @@ -33,7 +33,7 @@ def parse_data(self, page: str, **kwargs) -> dict: f"https://my.nwleics.gov.uk/my-property-finder?address={user_postcode}&go=1" ) - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py index 46e73c4a8a..e614af7168 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NorthumberlandCouncil.py @@ -32,12 +32,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) time.sleep(1) diff --git a/uk_bin_collection/uk_bin_collection/councils/NottinghamCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/NottinghamCityCouncil.py index 5f42b2bda2..f7b7d41e09 100644 --- a/uk_bin_collection/uk_bin_collection/councils/NottinghamCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/NottinghamCityCouncil.py @@ -26,9 +26,7 @@ def parse_data(self, page: str, **kwargs) -> dict: for collection in next_collections: bin_type = collection["collectionType"] - next_collection_date = datetime.fromisoformat( - collection["collectionDate"] - ) + next_collection_date = datetime.fromisoformat(collection["collectionDate"]) dict_data = { "type": bin_type, "collectionDate": next_collection_date.strftime(date_format), diff --git a/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py index a540a06468..1a74bbac26 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PortsmouthCityCouncil.py @@ -28,11 +28,11 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py index a858f8140f..2f12cc4784 100644 --- a/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/PrestonCityCouncil.py @@ -28,12 +28,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # If you bang in the house number (or property name) and postcode in the box it should find your property diff --git a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py index 5f09957045..8554e8d211 100644 --- a/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/ReigateAndBansteadBoroughCouncil.py @@ -18,13 +18,13 @@ class CouncilClass(AbstractGetBinDataClass): def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) # Pad UPRN with 0's at the start for any that aren't 12 chars user_uprn = user_uprn.zfill(12) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get( f"https://my.reigate-banstead.gov.uk/en/service/Bins_and_recycling___collections_calendar?uprn={user_uprn}" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py index bc3c77f092..c62f530db5 100644 --- a/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/RushcliffeBoroughCouncil.py @@ -24,12 +24,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Populate postcode field diff --git a/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py index 5fc3051bff..a6ee66a138 100644 --- a/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/SevenoaksDistrictCouncil.py @@ -22,7 +22,7 @@ def wait_for_element_conditions(self, driver, conditions, timeout: int = 5): def parse_data(self, page: str, **kwargs) -> dict: web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") page = "https://sevenoaks-dc-host01.oncreate.app/w/webpage/waste-collection-day" # Assign user info @@ -30,7 +30,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Enter postcode diff --git a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py index 25f728c7d2..aed54973b2 100644 --- a/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/StaffordshireMoorlandsDistrictCouncil.py @@ -21,12 +21,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_uprn = kwargs.get("uprn") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_uprn(user_uprn) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get("https://www.staffsmoorlands.gov.uk/findyourbinday") # Close cookies banner diff --git a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py index 4a339ccd3c..a7757e4cd4 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestLothianCouncil.py @@ -20,12 +20,12 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") user_postcode = kwargs.get("postcode") web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") check_paon(user_paon) check_postcode(user_postcode) # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get( "https://www.westlothian.gov.uk/article/31528/Bin-Collection-Calendar-Dates" ) diff --git a/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py b/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py index 5fbd4f8a24..8cf0c8a335 100644 --- a/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py +++ b/uk_bin_collection/uk_bin_collection/councils/WestSuffolkCouncil.py @@ -23,7 +23,7 @@ def wait_for_element_conditions(self, driver, conditions, timeout: int = 5): def parse_data(self, page: str, **kwargs) -> dict: web_driver = kwargs.get("web_driver") - headless= kwargs.get("headless") + headless = kwargs.get("headless") page = "https://westsuffolk-self.achieveservice.com/service/WSS_EX_Inf_Bin_Collection_Postcode_Lookup" # Assign user info @@ -31,7 +31,7 @@ def parse_data(self, page: str, **kwargs) -> dict: user_paon = kwargs.get("paon") # Create Selenium webdriver - driver = create_webdriver(web_driver,headless) + driver = create_webdriver(web_driver, headless) driver.get(page) # Click the cookie button diff --git a/uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py b/uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py index 33a3a5cf61..82d099948f 100644 --- a/uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py +++ b/uk_bin_collection/uk_bin_collection/councils/council_class_template/councilclasstemplate.py @@ -1,7 +1,6 @@ from bs4 import BeautifulSoup from uk_bin_collection.uk_bin_collection.common import * -from uk_bin_collection.uk_bin_collection.get_bin_data import \ - AbstractGetBinDataClass +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass # import the wonderful Beautiful Soup and the URL grabber @@ -25,7 +24,9 @@ def parse_data(self, page: str, **kwargs) -> dict: if bin_collection: dict_data = { "type": bin_type, - "collectionDate": datetime.strptime(bin_collection.get_text(strip=True), "%A, %d %B %Y") + "collectionDate": datetime.strptime( + bin_collection.get_text(strip=True), "%A, %d %B %Y" + ), } data["bins"].append(dict_data) diff --git a/wiki/generate_wiki.py b/wiki/generate_wiki.py index 9bd7a53e74..90f638a154 100644 --- a/wiki/generate_wiki.py +++ b/wiki/generate_wiki.py @@ -4,15 +4,17 @@ def main(): # initial markdown content - md = "\n" \ - "\n\n" \ - "This Markdown document provides a list of commands and parameters for use with this script.\n\n" \ - "As a reminder, most scripts only need a module name and a URL to run, but others need more parameters " \ - "depending on how the data is scraped.\n\n" \ - "For scripts that need postcodes, these should be provided in double quotes and with a space, " \ - "e.g. `\"AA1 2BB\"` rather than `AA12BB`.\n\n" \ - "This document is still a work in progress, don't worry if your council isn't listed - it will be soon!\n\n" \ - "## Contents\n" + md = ( + "\n" + "\n\n" + "This Markdown document provides a list of commands and parameters for use with this script.\n\n" + "As a reminder, most scripts only need a module name and a URL to run, but others need more parameters " + "depending on how the data is scraped.\n\n" + "For scripts that need postcodes, these should be provided in double quotes and with a space, " + 'e.g. `"AA1 2BB"` rather than `AA12BB`.\n\n' + "This document is still a work in progress, don't worry if your council isn't listed - it will be soon!\n\n" + "## Contents\n" + ) # get input.json cwd = os.getcwd() @@ -23,13 +25,20 @@ def main(): entries = "" for council, council_details in json_data.items(): if council != "" and council_details.get("wiki_name", council) != "": - # add contents entry to markdown content md += "- [" + council_details.get("wiki_name", council) + "]" - md += "(#" + council_details.get("wiki_name", council).lower().replace(" ", "-") + ")\n" + md += ( + "(#" + + council_details.get("wiki_name", council) + .lower() + .replace(" ", "-") + + ")\n" + ) # get additional arguments - command = council_details.get("wiki_command_url_override", council_details.get("url", "")) + command = council_details.get( + "wiki_command_url_override", council_details.get("url", "") + ) additional_parameters = "" if "skip_get_url" in council_details: command += " -s" @@ -38,7 +47,7 @@ def main(): command += " -u XXXXXXXX" additional_parameters += "- `-u` - UPRN\n" if "postcode" in council_details: - command += " -p \"XXXX XXX\"" + command += ' -p "XXXX XXX"' additional_parameters += "- `-p` - postcode\n" if "house_number" in council_details: command += " -n XX" From 6c1a063bbde4c9facc25c9fabbfbcdb53343fa27 Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 6 Jan 2024 21:17:29 +0000 Subject: [PATCH 3/5] feat: Update config_flow.py --- custom_components/uk_bin_collection/config_flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/custom_components/uk_bin_collection/config_flow.py b/custom_components/uk_bin_collection/config_flow.py index 2008ff6317..9bf3446dce 100644 --- a/custom_components/uk_bin_collection/config_flow.py +++ b/custom_components/uk_bin_collection/config_flow.py @@ -57,7 +57,7 @@ async def get_council_schema(self, council=str) -> vol.Schema: {vol.Required("web_driver", default=""): cv.string} ) council_schema = council_schema.extend( - {vol.Required("headless", default=True): cv.boolean} + {vol.Optional("headless", default=True): cv.boolean} ) return council_schema From d918e3f86b09a243a6655b0a4b91c30bd6e1ba24 Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 6 Jan 2024 21:18:06 +0000 Subject: [PATCH 4/5] feat: Update en.json --- custom_components/uk_bin_collection/translations/en.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/custom_components/uk_bin_collection/translations/en.json b/custom_components/uk_bin_collection/translations/en.json index 398a69e244..a4d46157f5 100644 --- a/custom_components/uk_bin_collection/translations/en.json +++ b/custom_components/uk_bin_collection/translations/en.json @@ -19,6 +19,7 @@ "number": "House number of the address", "usrn": "USRN (Unique Street Reference Number)", "web_driver": "URL of the remote Selenium web driver to use", + "headless": "Run Selenium in headless mode?", "submit": "Submit" }, "description": "Please refer to your councils [wiki](https://github.com/robbrad/UKBinCollectionData/wiki/Councils) entry for details on what to enter" @@ -29,4 +30,4 @@ "council": "Please select a council" } } -} \ No newline at end of file +} From 3539e92a8e4e6cd0951ab9c513b4845c796be16d Mon Sep 17 00:00:00 2001 From: Robert Bradley Date: Sat, 6 Jan 2024 21:18:36 +0000 Subject: [PATCH 5/5] feat: Update strings.json --- custom_components/uk_bin_collection/strings.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/custom_components/uk_bin_collection/strings.json b/custom_components/uk_bin_collection/strings.json index 398a69e244..a4d46157f5 100644 --- a/custom_components/uk_bin_collection/strings.json +++ b/custom_components/uk_bin_collection/strings.json @@ -19,6 +19,7 @@ "number": "House number of the address", "usrn": "USRN (Unique Street Reference Number)", "web_driver": "URL of the remote Selenium web driver to use", + "headless": "Run Selenium in headless mode?", "submit": "Submit" }, "description": "Please refer to your councils [wiki](https://github.com/robbrad/UKBinCollectionData/wiki/Councils) entry for details on what to enter" @@ -29,4 +30,4 @@ "council": "Please select a council" } } -} \ No newline at end of file +}