diff --git a/uk_bin_collection/tests/features/validate_council_outputs.feature b/uk_bin_collection/tests/features/validate_council_outputs.feature index 1218f3095c..93f39e84f2 100644 --- a/uk_bin_collection/tests/features/validate_council_outputs.feature +++ b/uk_bin_collection/tests/features/validate_council_outputs.feature @@ -402,6 +402,11 @@ Feature: Test each council output matches expected results | council | selenium_url | selenium_mode | | NorthWestLeicestershire | http://selenium:4444 | local | + @NorthYorkshire + Examples: NorthYorkshire + | council | selenium_url | selenium_mode | + | NorthYorkshire | None | None | + @NottinghamCityCouncil Examples: NottinghamCityCouncil | council | selenium_url | selenium_mode | diff --git a/uk_bin_collection/tests/input.json b/uk_bin_collection/tests/input.json index 3eeefbe8b6..4f04da8587 100644 --- a/uk_bin_collection/tests/input.json +++ b/uk_bin_collection/tests/input.json @@ -6,14 +6,6 @@ "wiki_name": "Aylesbury Vale Council (Buckinghamshire)", "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN. Returns all published collections in the past, present, future." }, - "BarnsleyMBCouncil": { - "skip_get_url": true, - "postcode": "S36 9AN", - "uprn": "2007004502", - "url": "https://waste.barnsley.gov.uk/ViewCollection/Collections", - "wiki_name": "Barnsley Metropolitan Borough Council", - "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)." - }, "BCPCouncil": { "skip_get_url": true, "uprn": "100040810214", @@ -21,12 +13,20 @@ "wiki_name": "BCP Council" }, "BarnetCouncil": { + "house_number": "HA8 7NA, 2, MANOR PARK GARDENS, EDGWARE, BARNET", + "postcode": "HA8 7NA", "skip_get_url": true, + "url": "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day", "web_driver": "http://selenium:4444", - "postcode": "HA8 7NA", - "house_number": "HA8 7NA, 2, MANOR PARK GARDENS, EDGWARE, BARNET", - "wiki_name": "Barnet Council", - "url": "https://www.barnet.gov.uk/recycling-and-waste/bin-collections/find-your-bin-collection-day" + "wiki_name": "Barnet Council" + }, + "BarnsleyMBCouncil": { + "postcode": "S36 9AN", + "skip_get_url": true, + "uprn": "2007004502", + "url": "https://waste.barnsley.gov.uk/ViewCollection/Collections", + "wiki_name": "Barnsley Metropolitan Borough Council", + "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search)." }, "BasingstokeCouncil": { "skip_get_url": true, @@ -41,20 +41,20 @@ "wiki_name": "Bath and North East Somerset Council" }, "BedfordshireCouncil": { + "postcode": "SG19 2UP", "skip_get_url": true, + "uprn": "10000802040", "url": "https://www.centralbedfordshire.gov.uk/info/163/bins_and_waste_collections_-_check_bin_collection_day", "wiki_name": "Bedfordshire Council", - "wiki_note": "In order to use this parser, you must provide a valid postcode and a uprn retrieved from the councils website for your specific address", - "postcode": "SG19 2UP", - "uprn": "10000802040" + "wiki_note": "In order to use this parser, you must provide a valid postcode and a uprn retrieved from the councils website for your specific address" }, "BexleyCouncil": { + "house_number": "1 Dorchester Avenue, Bexley", + "postcode": "DA5 3AH", "skip_get_url": true, + "uprn": "100020196143", "url": "https://mybexley.bexley.gov.uk/service/When_is_my_collection_day", "web_driver": "http://selenium:4444", - "postcode": "DA5 3AH", - "uprn": "100020196143", - "house_number": "1 Dorchester Avenue, Bexley", "wiki_name": "Bexley Council", "wiki_note": "In order to use this parser, you will need to sign up to [Bexley's @Home app](https://www.bexley.gov.uk/services/rubbish-and-recycling/bexley-home-recycling-app/about-app) (available for [iOS](https://apps.apple.com/gb/app/home-collection-reminder/id1050703690) and [Android](https://play.google.com/store/apps/details?id=com.contender.athome.android)).\nComplete the setup by entering your email and setting your address with postcode and address line.\nOnce you can see the calendar, you _should_ be good to run the parser.\nJust pass the email you used in quotes in the UPRN parameter.\n" }, @@ -62,28 +62,28 @@ "skip_get_url": true, "uprn": "100010733027", "url": "https://mybins.blackburn.gov.uk/api/mybins/getbincollectiondays?uprn=100010733027&month=8&year=2022", + "web_driver": "http://selenium:4444", "wiki_command_url_override": "https://www.blackburn.gov.uk", - "wiki_name": "Blackburn Council", - "web_driver": "http://selenium:4444" + "wiki_name": "Blackburn Council" }, "BoltonCouncil": { - "skip_get_url": true, "postcode": "BL1 5PQ", + "skip_get_url": true, "uprn": "100010886936", "url": "https://carehomes.bolton.gov.uk/bins.aspx", + "web_driver": "http://selenium:4444", "wiki_name": "Bolton Council", - "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Previously required single field that was UPRN and full address, now requires UPRN and postcode as separate fields.", - "web_driver": "http://selenium:4444" + "wiki_note": "To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search). Previously required single field that was UPRN and full address, now requires UPRN and postcode as separate fields." }, "BrightonandHoveCityCouncil": { + "house_number": "44 Carden Avenue, Brighton, BN1 8NE", + "postcode": "BN1 8NE", "skip_get_url": true, "uprn": "22060199", "url": "https://cityclean.brighton-hove.gov.uk/link/collections", + "web_driver": "http://selenium:4444", "wiki_name": "Brighton and Hove City Council", - "wiki_note": "Use the full address as it appears on the drop down on the site when you search by postcode", - "postcode": "BN1 8NE", - "house_number": "44 Carden Avenue, Brighton, BN1 8NE", - "web_driver": "http://selenium:4444" + "wiki_note": "Use the full address as it appears on the drop down on the site when you search by postcode" }, "BristolCityCouncil": { "skip_get_url": true, @@ -93,10 +93,10 @@ }, "BromleyBoroughCouncil": { "url": "https://recyclingservices.bromley.gov.uk/waste/6087017", + "web_driver": "http://selenium:4444", "wiki_command_url_override": "https://recyclingservices.bromley.gov.uk/waste/XXXXXXX", "wiki_name": "Bromley Borough Council", - "wiki_note": "Follow the instructions [here](https://recyclingservices.bromley.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command.", - "web_driver": "http://selenium:4444" + "wiki_note": "Follow the instructions [here](https://recyclingservices.bromley.gov.uk/waste) until the \"Your bin days\" page then copy the URL and replace the URL in the command." }, "BroxtoweBoroughCouncil": { "postcode": "NG16 2LY", @@ -128,8 +128,8 @@ "skip_get_url": true, "uprn": "010035034598", "url": "https://www.calderdale.gov.uk/environment/waste/household-collections/collectiondayfinder.jsp", - "wiki_name": "Calderdale Council", - "web_driver": "http://selenium:4444" + "web_driver": "http://selenium:4444", + "wiki_name": "Calderdale Council" }, "CannockChaseDistrictCouncil": { "postcode": "WS15 1JA", @@ -149,8 +149,8 @@ "skip_get_url": true, "uprn": "4525", "url": "https://apps.castlepoint.gov.uk/cpapps/index.cfm?fa=wastecalendar", - "wiki_name": "Castlepoint District Council", - "web_driver": "http://selenium:4444" + "web_driver": "http://selenium:4444", + "wiki_name": "Castlepoint District Council" }, "CharnwoodBoroughCouncil": { "url": "https://my.charnwood.gov.uk/location?put=cbc10070067259&rememberme=0&redirect=%2F", @@ -159,12 +159,12 @@ "wiki_note": "Replace XXXXXXXX with UPRN keeping \"cbc\" before it." }, "ChelmsfordCityCouncil": { + "house_number": "1 Celeborn Street, South Woodham Ferrers, Chelmsford, CM3 7AE", + "postcode": "CM3 7AE", "url": "https://www.chelmsford.gov.uk/myhome/", - "wiki_name": "Chelmsford City Council", - "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field.", "web_driver": "http://selenium:4444", - "postcode": "CM3 7AE", - "house_number": "1 Celeborn Street, South Woodham Ferrers, Chelmsford, CM3 7AE" + "wiki_name": "Chelmsford City Council", + "wiki_note": "Follow the instructions [here](https://www.chelmsford.gov.uk/myhome/) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field." }, "CheshireEastCouncil": { "url": "https://online.cheshireeast.gov.uk/MyCollectionDay/SearchByAjax/GetBartecJobList?uprn=100012791226&onelineaddress=3%20COBBLERS%20YARD,%20SK9%207DZ&_=1689413260149", @@ -174,8 +174,8 @@ }, "ChorleyCouncil": { "postcode": "PR6 7PG", - "uprn": "UPRN100010382247", "skip_get_url": true, + "uprn": "UPRN100010382247", "url": "https://myaccount.chorley.gov.uk/wastecollections.aspx", "web_driver": "http://selenium:4444", "wiki_name": "Chorley Council", @@ -251,11 +251,11 @@ "wiki_note": "Pass the house name/number in the house number parameter, wrapped in double quotes" }, "EastRidingCouncil": { - "postcode": "HU17 8LG", "house_number": "14 THE LEASES BEVERLEY HU17 8LG", + "postcode": "HU17 8LG", "skip_get_url": true, - "web_driver": "http://selenium:4444", "url": "https://wasterecyclingapi.eastriding.gov.uk", + "web_driver": "http://selenium:4444", "wiki_name": "East Riding Council", "wiki_note": "Put the full address as it displays on the council website dropdown when you do the check manually" }, @@ -293,13 +293,13 @@ "wiki_name": "Fenland District Council" }, "ForestOfDeanDistrictCouncil": { - "skip_get_url": true, "house_number": "ELMOGAL, PARKEND ROAD, BREAM, LYDNEY", "postcode": "GL15 6JT", + "skip_get_url": true, "url": "https://community.fdean.gov.uk/s/waste-collection-enquiry", + "web_driver": "http://selenium:4444", "wiki_name": "Forest of Dean District Council", - "wiki_note": "Pass the full address in the house number and postcode in", - "web_driver": "http://selenium:4444" + "wiki_note": "Pass the full address in the house number and postcode in" }, "GatesheadCouncil": { "house_number": "Bracken Cottage", @@ -324,23 +324,23 @@ "wiki_note": "Replace XXXXXXXX with UPRN." }, "GuildfordCouncil": { + "house_number": "THE LODGE, PUTTENHAM HILL HOUSE, PUTTENHAM HILL, PUTTENHAM, GUILDFORD, GU3 1AH", + "postcode": "GU3 1AH", "skip_get_url": true, "uprn": "100061372691", "url": "https://my.guildford.gov.uk/customers/s/view-bin-collections", - "postcode": "GU3 1AH", "web_driver": "http://selenium:4444", - "house_number": "THE LODGE, PUTTENHAM HILL HOUSE, PUTTENHAM HILL, PUTTENHAM, GUILDFORD, GU3 1AH", "wiki_name": "Guildford Council", "wiki_note": "If the bin day is 'today' then the collectionDate will only show today's date if before 7AM, else the date will be in 'previousCollectionDate'. To get the UPRN, you will need to use [FindMyAddress](https://www.findmyaddress.co.uk/search) to find the UPRN." }, "HaltonBoroughCouncil": { - "skip_get_url": true, "house_number": "12", "postcode": "WA7 4HA", + "skip_get_url": true, "url": "https://webapp.halton.gov.uk/PublicWebForms/WasteServiceSearchv1.aspx#collections", + "web_driver": "http://selenium:4444", "wiki_name": "Halton Borough Council", - "wiki_note": "Pass the House number and post code", - "web_driver": "http://selenium:4444" + "wiki_note": "Pass the House number and post code" }, "HaringeyCouncil": { "skip_get_url": true, @@ -386,12 +386,12 @@ }, "LeedsCityCouncil": { "house_number": "1", - "uprn": "72506983", "postcode": "LS6 2SE", "skip_get_url": true, + "uprn": "72506983", "url": "https://www.leeds.gov.uk/residents/bins-and-recycling/check-your-bin-day", - "wiki_name": "Leeds City Council", - "web_driver": "http://selenium:4444" + "web_driver": "http://selenium:4444", + "wiki_name": "Leeds City Council" }, "LisburnCastlereaghCityCouncil": { "house_number": "97", @@ -411,14 +411,14 @@ "uprn": "100021577765", "url": "https://www.hounslow.gov.uk/homepage/86/recycling_and_waste_collection_day_finder", "wiki_name": "London Borough Hounslow" - }, + }, "LondonBoroughRedbridge": { + "postcode": "IG2 6LQ", + "uprn": "10023770353", "url": "https://my.redbridge.gov.uk/RecycleRefuse", - "wiki_name": "London Borough Redbridge", - "wiki_note": "Follow the instructions [here](https://my.redbridge.gov.uk/RecycleRefuse) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field.", "web_driver": "http://selenium:4444", - "postcode": "IG2 6LQ", - "uprn": "10023770353" + "wiki_name": "London Borough Redbridge", + "wiki_note": "Follow the instructions [here](https://my.redbridge.gov.uk/RecycleRefuse) until you get the page listing your \"Address\" then copy the entire address text and use that in the house number field." }, "MaldonDistrictCouncil": { "skip_get_url": true, @@ -495,12 +495,12 @@ "wiki_name": "Newport City Council" }, "NorthEastDerbyshireDistrictCouncil": { + "postcode": "S42 5RB", "skip_get_url": true, "uprn": "010034492221", - "postcode": "S42 5RB", "url": "https://myselfservice.ne-derbyshire.gov.uk/service/Check_your_Bin_Day", - "wiki_name": "North East Derbyshire District Council", - "web_driver": "http://selenium:4444" + "web_driver": "http://selenium:4444", + "wiki_name": "North East Derbyshire District Council" }, "NorthEastLincs": { "skip_get_url": true, @@ -528,14 +528,6 @@ "url": "https://www.northlincs.gov.uk/bins-waste-and-recycling/bin-and-box-collection-dates/", "wiki_name": "North Lincolnshire Council" }, - "NorthWestLeicestershire": { - "skip_get_url": true, - "uprn": "100030572613", - "postcode": "DE74 2FZ", - "url": "https://www.nwleics.gov.uk/pages/collection_information", - "wiki_name": "North West Leicestershire Council", - "web_driver": "http://selenium:4444" - }, "NorthNorfolkDistrictCouncil": { "house_number": "1 Morston Mews", "postcode": "NR25 6BH", @@ -565,6 +557,20 @@ "url": "https://my.northtyneside.gov.uk/category/81/bin-collection-dates", "wiki_name": "North Tyneside Council" }, + "NorthWestLeicestershire": { + "postcode": "DE74 2FZ", + "skip_get_url": true, + "uprn": "100030572613", + "url": "https://www.nwleics.gov.uk/pages/collection_information", + "web_driver": "http://selenium:4444", + "wiki_name": "North West Leicestershire Council" + }, + "NorthYorkshire": { + "skip_get_url": true, + "uprn": "10093091235", + "url": "https://www.northyorks.gov.uk/bin-calendar/lookup", + "wiki_name": "NorthYorkshire" + }, "NorthumberlandCouncil": { "house_number": "22", "postcode": "NE46 1UQ", @@ -572,7 +578,7 @@ "url": "https://www.northumberland.gov.uk/Waste/Bins/Bin-Calendars.aspx", "web_driver": "http://selenium:4444", "wiki_name": "Northumberland Council" - }, + }, "NottinghamCityCouncil": { "skip_get_url": true, "uprn": "100031540180", @@ -585,12 +591,12 @@ "wiki_note": "Replace UPRN in URL with your own UPRN." }, "PortsmouthCityCouncil": { + "postcode": "PO4 0LE", "skip_get_url": true, "uprn": "1775027504", - "postcode": "PO4 0LE", "url": "https://my.portsmouth.gov.uk/en/AchieveForms/?form_uri=sandbox-publish://AF-Process-26e27e70-f771-47b1-a34d-af276075cede/AF-Stage-cd7cc291-2e59-42cc-8c3f-1f93e132a2c9/definition.json&redirectlink=%2F&cancelRedirectLink=%2F", - "wiki_name": "Portsmouth City Council", - "web_driver": "http://selenium:4444" + "web_driver": "http://selenium:4444", + "wiki_name": "Portsmouth City Council" }, "PrestonCityCouncil": { "house_number": "Town Hall", @@ -808,9 +814,9 @@ }, "ValeofWhiteHorseCouncil": { "custom_component_show_url_field": false, + "skip_get_url": true, "uprn": "100121391443", "url": "https://eform.whitehorsedc.gov.uk/ebase/BINZONE_DESKTOP.eb", - "skip_get_url": true, "wiki_name": "Vale of White Horse Council" }, "WakefieldCityCouncil": { @@ -885,8 +891,8 @@ "WiltshireCouncil": { "postcode": "SN83TE", "skip_get_url": true, - "url": "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/index", "uprn": "100120982570", + "url": "https://ilambassadorformsprod.azurewebsites.net/wastecollectiondays/index", "wiki_name": "Wiltshire Council" }, "WindsorAndMaidenheadCouncil": { @@ -910,4 +916,4 @@ "url": "https://waste-api.york.gov.uk/api/Collections/GetBinCollectionDataForUprn/", "wiki_name": "York Council" } -} +} \ No newline at end of file diff --git a/uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py b/uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py new file mode 100644 index 0000000000..7ec32f01ca --- /dev/null +++ b/uk_bin_collection/uk_bin_collection/councils/NorthYorkshire.py @@ -0,0 +1,58 @@ +from bs4 import BeautifulSoup +from uk_bin_collection.uk_bin_collection.common import * +from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass + + +class CouncilClass(AbstractGetBinDataClass): + """ + Concrete classes have to implement all abstract operations of the + base class. They can also override some operations with a default + implementation. + """ + + def parse_data(self, page: str, **kwargs) -> dict: + uprn = kwargs.get("uprn") + check_uprn(uprn) + + # Figure bin data URL from UPRN + url = "https://www.northyorks.gov.uk/bin-calendar/lookup" + + payload = { + "selected_address": uprn, + "submit": "Continue", + "form_id": "bin_calendar_lookup_form", + } + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + # This endpoint redirects to the data url. + response = requests.request("POST", url, headers=headers, data=payload) + bin_data_url = f"{response.url}/ajax" + + # Get bin data + response = requests.request("GET", bin_data_url) + bin_data = response.json() + + # Parse bin data + soup = BeautifulSoup(bin_data[1]["data"], "html.parser") + + # All collection info is in the table + table = ( + soup.find("div", {"id": "upcoming-collection"}).find("table").find("tbody") + ) + rows = table.find_all("tr") + + data = {"bins": []} + for row in rows: + cols = row.find_all("td") + # First column is date + bin_date = datetime.strptime(cols[0].text.strip(), "%d %B %Y") + # Third column is type + bin_type = cols[2].text.strip() + # This bin + this_bin = { + "type": bin_type, + "collectionDate": bin_date.strftime(date_format), + } + data["bins"].append(this_bin) + + return data