-
Notifications
You must be signed in to change notification settings - Fork 111
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #1031 from m26dvd/master
feat: Council Pack 18
- Loading branch information
Showing
10 changed files
with
534 additions
and
8 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
93 changes: 93 additions & 0 deletions
93
uk_bin_collection/uk_bin_collection/councils/CumberlandAllerdaleCouncil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| import requests | ||
| from bs4 import BeautifulSoup | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_postcode = kwargs.get("postcode") | ||
| user_paon = kwargs.get("paon") | ||
| check_postcode(user_postcode) | ||
| check_paon(user_paon) | ||
| bindata = {"bins": []} | ||
|
|
||
| URI = "https://abc-wrp.whitespacews.com/" | ||
|
|
||
| session = requests.Session() | ||
|
|
||
| # get link from first page as has some kind of unique hash | ||
| r = session.get( | ||
| URI, | ||
| ) | ||
| r.raise_for_status() | ||
| soup = BeautifulSoup(r.text, features="html.parser") | ||
|
|
||
| alink = soup.find("a", text="View My Collections") | ||
|
|
||
| if alink is None: | ||
| raise Exception("Initial page did not load correctly") | ||
|
|
||
| # greplace 'seq' query string to skip next step | ||
| nextpageurl = alink["href"].replace("seq=1", "seq=2") | ||
|
|
||
| data = { | ||
| "address_name_number": user_paon, | ||
| "address_postcode": user_postcode, | ||
| } | ||
|
|
||
| # get list of addresses | ||
| r = session.post(nextpageurl, data) | ||
| r.raise_for_status() | ||
|
|
||
| soup = BeautifulSoup(r.text, features="html.parser") | ||
|
|
||
| # get first address (if you don't enter enough argument values this won't find the right address) | ||
| alink = soup.find("div", id="property_list").find("a") | ||
|
|
||
| if alink is None: | ||
| raise Exception("Address not found") | ||
|
|
||
| nextpageurl = URI + alink["href"] | ||
|
|
||
| # get collection page | ||
| r = session.get( | ||
| nextpageurl, | ||
| ) | ||
| r.raise_for_status() | ||
| soup = BeautifulSoup(r.text, features="html.parser") | ||
|
|
||
| if soup.find("span", id="waste-hint"): | ||
| raise Exception("No scheduled services at this address") | ||
|
|
||
| u1s = soup.find("section", id="scheduled-collections").find_all("u1") | ||
|
|
||
| for u1 in u1s: | ||
| lis = u1.find_all("li", recursive=False) | ||
|
|
||
| date = lis[1].text.replace("\n", "") | ||
| bin_type = lis[2].text.replace("\n", "") | ||
|
|
||
| dict_data = { | ||
| "type": bin_type, | ||
| "collectionDate": datetime.strptime( | ||
| date, | ||
| "%d/%m/%Y", | ||
| ).strftime(date_format), | ||
| } | ||
| bindata["bins"].append(dict_data) | ||
|
|
||
| bindata["bins"].sort( | ||
| key=lambda x: datetime.strptime(x.get("collectionDate"), date_format) | ||
| ) | ||
|
|
||
| return bindata |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
83 changes: 83 additions & 0 deletions
83
uk_bin_collection/uk_bin_collection/councils/HartlepoolBoroughCouncil.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| import time | ||
|
|
||
| import requests | ||
| from bs4 import BeautifulSoup | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
| bindata = {"bins": []} | ||
|
|
||
| SESSION_URL = "https://online.hartlepool.gov.uk/authapi/isauthenticated?uri=https%253A%252F%252Fonline.hartlepool.gov.uk%252Fservice%252FRefuse_and_recycling___check_bin_day&hostname=online.hartlepool.gov.uk&withCredentials=true" | ||
|
|
||
| API_URL = "https://online.hartlepool.gov.uk/apibroker/runLookup" | ||
|
|
||
| headers = { | ||
| "Content-Type": "application/json", | ||
| "Accept": "application/json", | ||
| "User-Agent": "Mozilla/5.0", | ||
| "X-Requested-With": "XMLHttpRequest", | ||
| "Referer": "https://online.hartlepool.gov.uk/fillform/?iframe_id=fillform-frame-1&db_id=", | ||
| } | ||
| s = requests.session() | ||
| r = s.get(SESSION_URL) | ||
| r.raise_for_status() | ||
| session_data = r.json() | ||
| sid = session_data["auth-session"] | ||
| params = { | ||
| "id": "5ec67e019ffdd", | ||
| "repeat_against": "", | ||
| "noRetry": "true", | ||
| "getOnlyTokens": "undefined", | ||
| "log_id": "", | ||
| "app_name": "AF-Renderer::Self", | ||
| # unix_timestamp | ||
| "_": str(int(time.time() * 1000)), | ||
| "sid": sid, | ||
| } | ||
|
|
||
| data = { | ||
| "formValues": { | ||
| "Section 1": { | ||
| "collectionLocationUPRN": { | ||
| "value": user_uprn, | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| r = s.post(API_URL, json=data, headers=headers, params=params) | ||
| r.raise_for_status() | ||
|
|
||
| data = r.json() | ||
| rows_data = data["integration"]["transformed"]["rows_data"]["0"] | ||
| if not isinstance(rows_data, dict): | ||
| raise ValueError("Invalid data returned from API") | ||
|
|
||
| soup = BeautifulSoup(rows_data["HTMLCollectionDatesText"], "html.parser") | ||
|
|
||
| # Find all div elements containing the bin schedule | ||
| for div in soup.find_all("div"): | ||
| # Extract bin type and date from the span tag | ||
| text = div.find("span").text.strip() | ||
| bin_type, date = text.split(" ", 1) | ||
| dict_data = { | ||
| "type": bin_type, | ||
| "collectionDate": date, | ||
| } | ||
| bindata["bins"].append(dict_data) | ||
|
|
||
| return bindata |
75 changes: 75 additions & 0 deletions
75
uk_bin_collection/uk_bin_collection/councils/LondonBoroughHavering.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,75 @@ | ||
| import time | ||
|
|
||
| import requests | ||
| from bs4 import BeautifulSoup | ||
|
|
||
| from uk_bin_collection.uk_bin_collection.common import * | ||
| from uk_bin_collection.uk_bin_collection.get_bin_data import AbstractGetBinDataClass | ||
|
|
||
|
|
||
| # import the wonderful Beautiful Soup and the URL grabber | ||
| class CouncilClass(AbstractGetBinDataClass): | ||
| """ | ||
| Concrete classes have to implement all abstract operations of the | ||
| base class. They can also override some operations with a default | ||
| implementation. | ||
| """ | ||
|
|
||
| def parse_data(self, page: str, **kwargs) -> dict: | ||
|
|
||
| user_uprn = kwargs.get("uprn") | ||
| check_uprn(user_uprn) | ||
| bindata = {"bins": []} | ||
|
|
||
| URI = "https://lbhapiprod.azure-api.net" | ||
| endpoint = f"{URI}/whitespace/GetCollectionByUprnAndDate" | ||
| subscription_key = "2ea6a75f9ea34bb58d299a0c9f84e72e" | ||
|
|
||
| # Get today's date in 'YYYY-MM-DD' format | ||
| collection_date = datetime.now().strftime("%Y-%m-%d") | ||
|
|
||
| # Define the request headers | ||
| headers = { | ||
| "Content-Type": "application/json", | ||
| "Ocp-Apim-Subscription-Key": subscription_key, | ||
| } | ||
|
|
||
| # Define the request body | ||
| data = { | ||
| "getCollectionByUprnAndDate": { | ||
| "getCollectionByUprnAndDateInput": { | ||
| "uprn": user_uprn, | ||
| "nextCollectionFromDate": collection_date, | ||
| } | ||
| } | ||
| } | ||
| # Make the POST request | ||
| response = requests.post(endpoint, headers=headers, data=json.dumps(data)) | ||
| response.raise_for_status() # Raise an exception for HTTP errors | ||
|
|
||
| # Parse the JSON response | ||
| response_data = response.json() | ||
|
|
||
| collections = ( | ||
| response_data.get("getCollectionByUprnAndDateResponse", {}) | ||
| .get("getCollectionByUprnAndDateResult", {}) | ||
| .get("Collections", []) | ||
| ) | ||
|
|
||
| for collection in collections: | ||
| bin_type = collection["service"] | ||
| collection_date = collection["date"] | ||
|
|
||
| dict_data = { | ||
| "type": bin_type, | ||
| "collectionDate": datetime.strptime( | ||
| collection_date, | ||
| "%d/%m/%Y %H:%M:%S", | ||
| ).strftime(date_format), | ||
| } | ||
| bindata["bins"].append(dict_data) | ||
| bindata["bins"].sort( | ||
| key=lambda x: datetime.strptime(x.get("collectionDate"), date_format) | ||
| ) | ||
|
|
||
| return bindata |
Oops, something went wrong.