Skip to content

Commit

Permalink
Simplify heartbeat_refresh (#98)
Browse files Browse the repository at this point in the history
* build: do not handle aioqzone extras

* feat: heartbeat is simplified

* ci: fix extras not installed to poetry venv

* test: fix unexpected param

---------

Co-authored-by: JamzumSum <zzzzss990315@gmail.com>
  • Loading branch information
github-actions[bot] and JamzumSum authored Mar 27, 2023
1 parent f66ed5c commit cb76f1c
Show file tree
Hide file tree
Showing 8 changed files with 93 additions and 251 deletions.
1 change: 1 addition & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ jobs:
run: |
echo "::group::Install Dependencies"
poetry install -n -vv
poetry run pip install aioqzone[captcha,lxml] -q
echo "::endgroup::"
echo "::group::pytest outputs"
Expand Down
35 changes: 12 additions & 23 deletions doc/source/locale/zh_CN/LC_MESSAGES/api/heartbeat.po
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: aioqzone-feed \n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-03-09 14:12+0800\n"
"POT-Creation-Date: 2023-03-27 09:41+0800\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language: zh_CN\n"
Expand All @@ -29,24 +29,15 @@ msgid ""
"timer that circularly calls `.heartbeat_refresh`."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat
#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh of
#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat of
msgid "参数"
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:5 of
msgid "max retry times when some exceptions occurs, defaults to 5."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:7 of
msgid "retry interval, defaults to 5."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:8 of
msgid "heartbeat interval, defaults to 300."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:10 of
#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:7 of
msgid "timer name"
msgstr ""

Expand All @@ -55,32 +46,30 @@ msgstr ""
msgid "返回"
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:11 of
#: aioqzone_feed.api.heartbeat.HeartbeatApi.add_heartbeat:8 of
msgid "the heartbeat task"
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:1 of
msgid ""
"A wrapper function that calls :external:meth:`aioqzone.api.QzoneWebAPI.get_feeds_count` and "
"handles all kinds of excpetions raised during heartbeat."
"A wrapper function that calls :obj:`hb_api` and handles all kinds of excpetions raised during "
"heartbeat."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:5 of
msgid ""
"This method calls heartbeat **ONLY ONCE** so it should be called circularly by using "
"This method calls heartbeat **ONLY ONCE** so it should be called periodically by using "
"`.add_heartbeat` or other timer/scheduler."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:9 of
msgid "retry times on QzoneError, default as 2."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:11 of
msgid "retry interval on QzoneError"
#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:10 of
msgid "do not retry, just call heartbeat once"
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.heartbeat_refresh:12 of
msgid "whether the timer should stop, means heartbeat will always fail until something is changed."
msgid ""
"whether the timer is suggested to be stopped, means heartbeat might not success even after a "
"retry, until underlying causes are solved."
msgstr ""

#: aioqzone_feed.api.heartbeat.HeartbeatApi.stop:1 of
Expand Down
10 changes: 7 additions & 3 deletions doc/source/locale/zh_CN/LC_MESSAGES/event/index.po
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ msgid ""
msgstr ""
"Project-Id-Version: aioqzone-feed \n"
"Report-Msgid-Bugs-To: \n"
"POT-Creation-Date: 2023-03-09 14:23+0800\n"
"POT-Creation-Date: 2023-03-27 09:44+0800\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language: zh_CN\n"
Expand Down Expand Up @@ -94,8 +94,8 @@ msgstr ""

#: aioqzone_feed.event.heartbeat.HeartbeatEvent.HeartbeatFailed:1 of
msgid ""
"The HeartbeatFailed function is called when the heartbeat fails. It can be used to log an error "
"and call :meth:`aioqzone_feed.api.feed.FeedApi.add_heartbeat` again if possible."
"The HeartbeatFailed function is called when the heartbeat is skipped/stopped. It can be used to "
"log an error and call :meth:`aioqzone_feed.api.feed.FeedApi.add_heartbeat` again if possible."
msgstr ""

#: aioqzone_feed.event.heartbeat.HeartbeatEvent.HeartbeatFailed:6 of
Expand All @@ -104,6 +104,10 @@ msgid ""
"failure."
msgstr ""

#: aioqzone_feed.event.heartbeat.HeartbeatEvent.HeartbeatFailed:10 of
msgid "`exc` is not optional"
msgstr ""

#: aioqzone_feed.event.heartbeat.HeartbeatEvent.HeartbeatRefresh:1 of
msgid ""
"This event is triggered after a heartbeat succeeded and there are new feeds. Use this event to "
Expand Down
146 changes: 2 additions & 144 deletions poetry.lock

Large diffs are not rendered by default.

10 changes: 2 additions & 8 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "aioqzone-feed"
version = "0.13.3.dev1"
version = "0.13.4.dev1"
description = "aioqzone plugin providing higher level api for processing feed."
authors = ["aioqzone <zzzzss990315@gmail.com>"]
license = "AGPL-3.0"
Expand All @@ -16,17 +16,11 @@ documentation = "https://aioqzone.github.io/aioqzone-feed"
python = "^3.8"
aioqzone = { version = "^0.12.11.dev2", source = "PyPI", allow-prereleases = true }
exceptiongroup = { version = ">=1.1.1", python = "<3.11" }

lxml = { version = "*", optional = false }
cssselect = { version = "*", optional = false }

numpy = { version = "*", optional = false }
pillow = { version = "*", optional = false }

# prepare for aioqzone v13
[tool.poetry.extras]
captcha = ["numpy", "pillow"] # equals to aioqzone[captcha]
web = ["lxml", "cssselect"] # equals to aioqzone[lxml]
web = ["lxml"]

# dependency groups
[tool.poetry.group.test]
Expand Down
128 changes: 61 additions & 67 deletions src/aioqzone_feed/api/heartbeat.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
import logging
import sys
from functools import partial, singledispatch
from typing import Optional, Union

Expand All @@ -16,6 +17,9 @@
from aioqzone_feed.event import HeartbeatEvent
from aioqzone_feed.utils.task import AsyncTimer

if sys.version_info < (3, 11):
from exceptiongroup import ExceptionGroup

log = logging.getLogger(__name__)


Expand Down Expand Up @@ -46,93 +50,83 @@ def __init__(self, api: Union[QzoneH5API, QzoneWebAPI]) -> None:
else:
raise TypeError("wrong api instance:", type(api))

async def heartbeat_refresh(self, *, retry: int = 2, retry_intv: float = 5):
"""A wrapper function that calls :external:meth:`aioqzone.api.QzoneWebAPI.get_feeds_count`
and handles all kinds of excpetions raised during heartbeat.
async def heartbeat_refresh(self):
"""A wrapper function that calls :obj:`hb_api` and handles all kinds of excpetions
raised during heartbeat.
.. note::
This method calls heartbeat **ONLY ONCE** so it should be called circularly by using
This method calls heartbeat **ONLY ONCE** so it should be called periodically by using
`.add_heartbeat` or other timer/scheduler.
:param retry: retry times on QzoneError, default as 2.
:param retry_intv: retry interval on QzoneError
:return: whether the timer should stop, means heartbeat will always fail until something is changed.
"""
exc = last_fail_hook = None
r = False
for i in range(retry):
try:
cnt = new_feed_cnt(await self.hb_api())
log.debug("heartbeat: new_feed_cnt=%d", cnt)
if cnt:
self.add_hook_ref("hook", self.hook.HeartbeatRefresh(cnt))
return False # don't stop
except (
QzoneError,
HTTPStatusError,
) as e:
# retry at once
exc, excname = e, e.__class__.__name__
log.warning("%s captured in heartbeat, retry at once (%d)", excname, i)
log.debug(excname, exc_info=e)
except HookError as e:
if e.hook.__qualname__ == last_fail_hook:
# if the same hook raises exception twice, we assume it is systematically broken
# so we should stop heartbeat at once.
r = True
break
last_fail_hook = e.hook.__qualname__
log.error("HookError captured in heartbeat, retry at once (%d)", i)
except (
HTTPError,
SkipLoginInterrupt,
KeyboardInterrupt,
UserBreak,
asyncio.CancelledError,
) as e:
# retry in next trigger
exc, excname = e, e.__class__.__name__
log.warning("%s captured in heartbeat, retry in next trigger", excname)
log.debug(excname, exc_info=e)
break
except LoginError as e:
if LoginMethod.up in e.methods_tried:
# login error means all methods failed.
# we should stop HB if up login will fail.
r = True
break
except BaseException as e:
exc, r = e, True
log.error("Uncaught error in heartbeat.", exc_info=e)
break
await asyncio.sleep(retry_intv)
else:
log.error("Max retry exceeds (%d)", retry)
.. versionchanged:: 0.13.4
if r:
log.warning(f"Heartbeat stopped.")
do not retry, just call heartbeat once
self.add_hook_ref("hook", self.hook.HeartbeatFailed(exc))
return r # stop at once
:return: whether the timer is suggested to be stopped,
means heartbeat might not success even after a retry, until underlying causes are solved.
"""
fail = lambda exc: self.add_hook_ref("hook", self.hook.HeartbeatFailed(exc))
try:
cnt = new_feed_cnt(await self.hb_api())
log.debug("heartbeat: new_feed_cnt=%d", cnt)
if cnt:
self.add_hook_ref("hook", self.hook.HeartbeatRefresh(cnt))
return False # don't stop
except QzoneError as e:
fail(e)
log.warning(e)
if e.code == -3000 and "登录" in e.msg:
return True
return False
except HTTPStatusError as e:
fail(e)
log.warning(e)
log.debug(e.request, exc_info=e)
if e.response.status_code in [403, 302]:
return True
return False
except HookError as e:
fail(e)
log.error("HookError in heartbeat, stop at once")
log.debug(e)
return True
except (
HTTPError,
SkipLoginInterrupt,
KeyboardInterrupt,
UserBreak,
asyncio.CancelledError,
) as e:
fail(e)
log.warning(f"{e.__class__.__name__}in heartbeat, retry in next trigger")
log.debug(e)
return False
except LoginError as e:
fail(e)
if LoginMethod.up in e.methods_tried:
# login error means all methods failed.
# we should stop HB if up login will fail.
return True
return False
except BaseException as e:
fail(e)
log.error("Uncaught error in heartbeat.", exc_info=e)
return True

def add_heartbeat(
self,
*,
retry: int = 5,
retry_intv: float = 5,
hb_intv: float = 300,
name: Optional[str] = None,
):
"""A helper function that creates a heartbeat task and keep a ref of it.
A heartbeat task is a timer that circularly calls `.heartbeat_refresh`.
:param retry: max retry times when some exceptions occurs, defaults to 5.
:param hb_intv: retry interval, defaults to 5.
:param hb_intv: heartbeat interval, defaults to 300.
:param name: timer name
:return: the heartbeat task
"""
heartbeat_refresh = partial(self.heartbeat_refresh, retry=retry, retry_intv=retry_intv)
heartbeat_refresh = partial(self.heartbeat_refresh)
self.hb_timer = AsyncTimer(
hb_intv, heartbeat_refresh, delay=hb_intv, name=name or "heartbeat"
)
Expand Down
10 changes: 6 additions & 4 deletions src/aioqzone_feed/event/heartbeat.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,18 @@
from typing import Optional

from qqqr.event import Event


class HeartbeatEvent(Event):
async def HeartbeatFailed(self, exc: Optional[BaseException] = None):
async def HeartbeatFailed(self, exc: BaseException):
"""
The HeartbeatFailed function is called when the heartbeat fails.
The HeartbeatFailed function is called when the heartbeat is skipped/stopped.
It can be used to log an error and call :meth:`aioqzone_feed.api.feed.FeedApi.add_heartbeat`
again if possible.
:param exc: Used to pass an exception object that can be used to determine the cause of the heartbeat failure.
.. versionchanged:: 0.13.4
`exc` is not optional
"""

pass
Expand Down
4 changes: 2 additions & 2 deletions test/api/test_heartbeat.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,9 @@ async def api(client: ClientAdapter, man: MixedLoginMan):
(LoginError("mock", [LoginMethod.qr]), True),
(ConnectError("mock"), True),
(TimeoutException("mock"), True),
(HTTPStatusError("mock", request=..., response=...), True), # type: ignore
(HTTPError("mock"), True),
(QzoneError(-3000), True),
(QzoneError(-3000, "请先登录"), False),
(SkipLoginInterrupt(), True),
(UserBreak(), True),
(asyncio.CancelledError(), True),
Expand All @@ -46,7 +46,7 @@ async def api(client: ClientAdapter, man: MixedLoginMan):
)
async def test_heartbeat_exc(api: HeartbeatApi, exc2r: Type[BaseException], should_alive: bool):
with patch.object(api, "hb_api", side_effect=exc2r):
api.add_heartbeat(retry=2, hb_intv=0.1, retry_intv=0)
api.add_heartbeat(hb_intv=0.1)
assert api.hb_timer
await asyncio.sleep(0.4)
assert (api.hb_timer.state == "PENDING") is should_alive

0 comments on commit cb76f1c

Please sign in to comment.