From 84e7948853e75a6f0abf2a620051676186ecf1fd Mon Sep 17 00:00:00 2001 From: Elliot <3186037+elliot-100@users.noreply.github.com> Date: Wed, 20 Dec 2023 16:48:19 +0000 Subject: [PATCH 1/3] Mark all private symbols with _ --- britishcycling_clubs/manager.py | 21 +++++++++++++++------ britishcycling_clubs/profile.py | 10 +++++----- 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/britishcycling_clubs/manager.py b/britishcycling_clubs/manager.py index 0e2268a..dabdce2 100644 --- a/britishcycling_clubs/manager.py +++ b/britishcycling_clubs/manager.py @@ -11,7 +11,7 @@ if TYPE_CHECKING: from typing_extensions import TypeGuard -MANAGER_BASE_URL = "https://www.britishcycling.org.uk/uac/connect?success_url=/dashboard/club/membership?club_id=" +_MANAGER_VIA_LOGIN_BASE_URL = "https://www.britishcycling.org.uk/uac/connect?success_url=/dashboard/club/membership?club_id=" log = logging.getLogger(__name__) @@ -66,8 +66,6 @@ def get_manager_member_counts( values hadn't populated correctly. """ - club_manager_url = f"{MANAGER_BASE_URL}{club_id}/" - start_time = time.time() _log_info("Started timer for Playwright operations", start_time) @@ -77,7 +75,7 @@ def get_manager_member_counts( page = browser.new_page() # login page - page.goto(club_manager_url) + page.goto(club_manager_url_via_login(club_id)) page.locator("id=username2").fill(username) page.locator("id=password2").fill(password) page.locator("id=login_button").click() @@ -104,6 +102,17 @@ def get_manager_member_counts( return _process_manager_member_counts(raw_member_counts) +def club_manager_url_via_login(club_id: str) -> str: + """Return URL of club's Club Manager page. + + Parameters + ---------- + club_id + From the URL used to access club pages. + """ + return f"{_MANAGER_VIA_LOGIN_BASE_URL}{club_id}/" + + def _process_manager_member_counts(member_counts: dict[str, str]) -> MemberCounts: """Process raw values. @@ -126,12 +135,12 @@ def _process_manager_member_counts(member_counts: dict[str, str]) -> MemberCount ) raise ValueError(error_message) - if not is_membercounts(processed_member_counts): + if not _is_membercounts(processed_member_counts): raise TypeError return processed_member_counts -def is_membercounts(val: object) -> TypeGuard[MemberCounts]: +def _is_membercounts(val: object) -> TypeGuard[MemberCounts]: """Check return type.""" if isinstance(val, dict): return all(isinstance(v, int) for v in val.values()) diff --git a/britishcycling_clubs/profile.py b/britishcycling_clubs/profile.py index 646b2f2..b3b7c5b 100644 --- a/britishcycling_clubs/profile.py +++ b/britishcycling_clubs/profile.py @@ -5,8 +5,8 @@ import requests from bs4 import BeautifulSoup, Tag -PROFILE_BASE_URL = "https://www.britishcycling.org.uk/club/profile/" -REQUESTS_TIMEOUT = 10 # For `requests` library operations +_PROFILE_BASE_URL = "https://www.britishcycling.org.uk/club/profile/" +_REQUESTS_TIMEOUT = 10 # For `requests` library operations class ProfileInfo(TypedDict): @@ -26,10 +26,10 @@ def get_profile_info(club_id: str) -> ProfileInfo: Returns ------- - ProfileInfo + _ProfileInfo """ - profile_url = f"{PROFILE_BASE_URL}{club_id}/" - r = requests.get(profile_url, timeout=REQUESTS_TIMEOUT) + profile_url = f"{_PROFILE_BASE_URL}{club_id}/" + r = requests.get(profile_url, timeout=_REQUESTS_TIMEOUT) r.raise_for_status() if r.url != profile_url: error_message = f"Redirected to unexpected URL {r.url}. Is `club_id` valid?" From 61ebf3e825f2bca0ba379d1e6ff29f50e4dfce0e Mon Sep 17 00:00:00 2001 From: Elliot <3186037+elliot-100@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:21:17 +0000 Subject: [PATCH 2/3] build (dev-deps): bump black, ruff, types-beautifulsoup4 --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0d57adf..4d2f7b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,15 +17,15 @@ requests = "^2.23.1" playwright = "^1.39.0" [tool.poetry.group.dev.dependencies] -black = "^23.11.0" +black = "^23.12.0" isort = "^5.13.0" -ruff = "^0.1.6" +ruff = "^0.1.8" [tool.poetry.group.test.dependencies] mypy = "^1.7.1" pytest = "^7.4.2" types-requests = "^2.31.0.10" -types-beautifulsoup4 = "^4.12.0.6" +types-beautifulsoup4 = "^4.12.0.7" [tool.black] line-length = 88 From 669cb2e0c2cc258d014259251ebe619bddb115da Mon Sep 17 00:00:00 2001 From: Elliot <3186037+elliot-100@users.noreply.github.com> Date: Wed, 20 Dec 2023 17:22:41 +0000 Subject: [PATCH 3/3] `club_profile_url` and `club_manager_url_via_login()` --- CHANGELOG.md | 15 ++++++++--- README.md | 45 ++++++++++++++++++++++---------- britishcycling_clubs/__init__.py | 4 +++ britishcycling_clubs/profile.py | 17 ++++++++++-- tests/test_manager.py | 13 ++++++++- tests/test_profile.py | 9 +++++++ 6 files changed, 82 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c45ea6..61f1dde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,12 +8,19 @@ and this project tries to adhere to [Semantic Versioning](https://semver.org/spe Historic and pre-release versions aren't necessarily included. +## [UNRELEASED] - TBC + +### Added + +- `club_profile_url` and `club_manager_url_via_login()` + + ## [0.8.1] - 2023-12-19 ### Fixed - `get_manager_member_counts()` still returned `["pending"]` instead of - `["new"]`. + `["new"]` ## [0.8.0] - 2023-12-19 @@ -22,12 +29,11 @@ Historic and pre-release versions aren't necessarily included. - **BREAKING CHANGES**: Functions renamed to `get_profile_info()` and `get_manager_member_counts()`. `get_manager_member_counts()` returns `["new"]` - instead of `["pending"]`. + instead of `["pending"]` - Update dev/test dependencies: isort, mypy; CI dependencies actions/checkout, actions/setup-python - ### Added - `get_profile_info()`: raise exception on redirect; better error messages; basic @@ -57,7 +63,8 @@ Historic and pre-release versions aren't necessarily included. ### Changed -- `get_private_member_counts()`: raise exception if zero 'active members' would be returned. +- `get_private_member_counts()`: raise exception if zero 'active members' would be + returned - Update dev/test dependencies: mypy, ruff diff --git a/README.md b/README.md index 86d4031..838332a 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,15 @@ See also https://playwright.dev/python/docs/browsers#install-system-dependencies ## Usage +### Club profile URL -### Get info from a club's profile page +``` +britishcycling_clubs.club_profile_url( + club_id: str +) -> str +``` + +### Get info from a club's profile ``` britishcycling_clubs.get_profile_info( @@ -49,17 +56,27 @@ britishcycling_clubs.get_profile_info( ``` Return information from the club's public profile page; doesn't require login. -Specifically, returns these values: +Specifically, return a dict with these keys and corresponding values: -- Club name -- Total club members +- `"club_name"`: Club name +- `"total_members"`: Total club members Example script `example_profile_info.py` loads club ID from `config.ini` (you'll -need to copy `config_dist.ini`, populate club ID only and rename). It -then retrieves and prints the club name and total member count. +need to copy `config_dist.ini`, populate club ID only and rename). +It then retrieves and prints the club name and total member count. + + +### Club manager URL (via login) +``` +britishcycling_clubs.club_manager_url_via_login( + club_id: str +) -> str +``` +URL which redirects to Club Manager URL, via login if needed. -### Get member counts from a club's Club Manager pages + +### Get member counts from Club Manager ``` britishcycling_clubs.get_manager_member_counts( @@ -71,17 +88,17 @@ britishcycling_clubs.get_manager_member_counts( ``` Get numbers of active, new, expired members from the club manager page. -Specifically, returns the counts from these tabs: +Specifically, return a dict with these keys, and values from badges on corresponding +tabs: -- Active Club Members -- New Club Subscriptions -- Expired Club Members +- `"active"`: Active Club Members +- `"expired"`: Expired Club Members +- `"new"`: New Club Subscriptions This takes about 10s. Example script `example_manager_member_counts.py` loads club ID and credentials from `config.ini` (you'll need to copy `config_dist.ini`, populate and rename to -`config.ini`). It then retrieves and prints the number of active, expired and new +`config.ini`). +It then retrieves and prints the number of active, expired and new club member counts from the club's Club Manager pages. - - diff --git a/britishcycling_clubs/__init__.py b/britishcycling_clubs/__init__.py index 70fdf25..79c1483 100644 --- a/britishcycling_clubs/__init__.py +++ b/britishcycling_clubs/__init__.py @@ -1,6 +1,10 @@ """Module with functions to retrieve information about a club.""" +from britishcycling_clubs.manager import ( + club_manager_url_via_login as club_manager_url_via_login, +) from britishcycling_clubs.manager import ( get_manager_member_counts as get_manager_member_counts, ) +from britishcycling_clubs.profile import club_profile_url as club_profile_url from britishcycling_clubs.profile import get_profile_info as get_profile_info diff --git a/britishcycling_clubs/profile.py b/britishcycling_clubs/profile.py index b3b7c5b..6000bc5 100644 --- a/britishcycling_clubs/profile.py +++ b/britishcycling_clubs/profile.py @@ -26,9 +26,11 @@ def get_profile_info(club_id: str) -> ProfileInfo: Returns ------- - _ProfileInfo + dict[str, str | int] + keys: 'club_name', 'total_members' + values: corresponding str or int """ - profile_url = f"{_PROFILE_BASE_URL}{club_id}/" + profile_url = club_profile_url(club_id) r = requests.get(profile_url, timeout=_REQUESTS_TIMEOUT) r.raise_for_status() if r.url != profile_url: @@ -41,6 +43,17 @@ def get_profile_info(club_id: str) -> ProfileInfo: } +def club_profile_url(club_id: str) -> str: + """Return URL of club's profile page. + + Parameters + ---------- + club_id + From the URL used to access club pages. + """ + return f"{_PROFILE_BASE_URL}{club_id}/" + + def _club_name_from_profile(soup: BeautifulSoup) -> str: """Return the club's name from profile page soup.""" club_name_h1 = soup.find("h1", class_="article__header__title-body__text") diff --git a/tests/test_manager.py b/tests/test_manager.py index 4d065f0..0d542c6 100644 --- a/tests/test_manager.py +++ b/tests/test_manager.py @@ -1,7 +1,18 @@ """Tests for 'manager' functions.""" import pytest -from britishcycling_clubs.manager import _process_manager_member_counts +from britishcycling_clubs.manager import ( + _process_manager_member_counts, + club_manager_url_via_login, +) + + +def test_club_manager_url_via_login__happy_path() -> None: + """Test that correct URL is returned.""" + assert ( + club_manager_url_via_login("000") + == "https://www.britishcycling.org.uk/uac/connect?success_url=/dashboard/club/membership?club_id=000/" + ) def test__process_manager_member_counts__happy_path() -> None: diff --git a/tests/test_profile.py b/tests/test_profile.py index 45e4f74..383a280 100644 --- a/tests/test_profile.py +++ b/tests/test_profile.py @@ -4,8 +4,17 @@ from britishcycling_clubs.profile import ( _club_name_from_profile, _total_members_from_profile, + club_profile_url, ) + +def test_club_profile_url__happy_path() -> None: + """Test that correct URL is returned.""" + assert ( + club_profile_url("000") == "https://www.britishcycling.org.uk/club/profile/000/" + ) + + # Partial extract from actual page PROFILE_PAGE_EXTRACT = """