Skip to content

Commit

Permalink
feat: use aiohttp_client_cache
Browse files Browse the repository at this point in the history
  • Loading branch information
34j committed Oct 10, 2023
1 parent 4779c65 commit 71ef609
Show file tree
Hide file tree
Showing 3 changed files with 67 additions and 6 deletions.
53 changes: 52 additions & 1 deletion poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ aiofiles = "^23.2.1"
tqdm = "^4.66.1"
pandas = "^2.1.1"
rich = "^13.6.0"
aiohttp-client-cache = "^0.9.1"

[tool.poetry.group.dev.dependencies]
pre-commit = ">=3"
Expand Down
19 changes: 14 additions & 5 deletions src/ut_course_catalog/ja.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@

import aiofiles
import aiohttp
from aiohttp_client_cache import SQLiteBackend
from aiohttp_client_cache.session import CachedSession
from bs4 import BeautifulSoup, ResultSet, Tag
from pandas import DataFrame
from tenacity import WrappedFn, retry
Expand Down Expand Up @@ -675,17 +677,24 @@ class UTCourseCatalog:
_rate_limitter: RateLimitter

def __init__(
self, logger_level: int = 0, min_interval: timedelta | int = 1
self,
logger_level: int = 0,
min_interval: timedelta | int = 1,
session: aiohttp.ClientSession | None = None,
) -> None:
self.session = None
self.session = session
self._logger = getLogger(__name__)
self._logger.setLevel(logger_level)
self._rate_limitter = RateLimitter(min_interval=min_interval)

async def __aenter__(self) -> Self:
if self.session:
raise RuntimeError("__aenter__ called twice")
self.session = aiohttp.ClientSession()
if self.session is None:
self.session = CachedSession(
cache=SQLiteBackend(
cache_name="~/.cache/ut_course_catalog/cache.sqlite"
),
logger=self._logger,
)
await self.session.__aenter__()
return self

Expand Down

0 comments on commit 71ef609

Please sign in to comment.