From 3f09439ab1c161b8d438cba99f2cdfd42b1d0683 Mon Sep 17 00:00:00 2001 From: bralbra Date: Thu, 26 Sep 2024 11:02:13 -0400 Subject: [PATCH] kick --- README.MD | 10 +- src/bot/dialogs/user/channel/add/constants.py | 2 +- src/constants.py | 2 +- .../data_fetcher/__init__.py | 7 +- .../data_fetcher/kick/__init__.py | 4 + .../data_fetcher/kick/constants.py | 4 + .../data_fetcher/kick/fetcher.py | 97 ++++++++ .../telegram_notify_job/notifier/notify.py | 220 ++++++++++-------- src/utils.py | 15 +- 9 files changed, 250 insertions(+), 111 deletions(-) create mode 100644 src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/__init__.py create mode 100644 src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/constants.py create mode 100644 src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/fetcher.py diff --git a/README.MD b/README.MD index eec85d8..ded7a7c 100644 --- a/README.MD +++ b/README.MD @@ -6,11 +6,11 @@ Simple LiveStreams notifier in telegram based on [aiogram](https://github.com/ai ### Available platforms -| Platform | Based on | Status | -|-----------------|-------------------------------------------------|----------------------------------------------------------------------| -| Youtube | [youtube-dlp](https://github.com/yt-dlp/yt-dlp) | ✅ | -| Twitch | [twitch-api](https://github.com/Teekeks/pyTwitchAPI) | ✅ | - +| Platform | Based on | Status | +|----------|------------------------------------------------------|------------------------------------------------------| +| Youtube | [youtube-dlp](https://github.com/yt-dlp/yt-dlp) | ✅ | +| Twitch | [twitch-api](https://github.com/Teekeks/pyTwitchAPI) | ✅ | +| Kick | [aiohttp](https://github.com/aio-libs/aiohttp) | ❌ (Until cloudflare-403 fix or creating official api | Use this bot to receive periodic reports on live broadcasts on stream platforms and generate and send the report to telegram. The current version of the bot works in the telegram channel [НАСРАНО](https://t.me/HACPAH1). diff --git a/src/bot/dialogs/user/channel/add/constants.py b/src/bot/dialogs/user/channel/add/constants.py index f124e4e..e74e27d 100644 --- a/src/bot/dialogs/user/channel/add/constants.py +++ b/src/bot/dialogs/user/channel/add/constants.py @@ -12,7 +12,7 @@ url_examples: dict = { ChannelType.YOUTUBE: "https://www.youtube.com/@username", ChannelType.TWITCH: "https://www.twitch.tv/username", - ChannelType.KICK: "empty", + ChannelType.KICK: "https://www.kick.com/username", } __all__ = ["url_examples", "url_validators"] diff --git a/src/constants.py b/src/constants.py index 7f2926a..5f698bc 100755 --- a/src/constants.py +++ b/src/constants.py @@ -8,7 +8,7 @@ SQLITE_DATABASE_FILE_PATH: str = os.environ.get( "SQLITE_DATABASE_FILE_PATH", os.path.join(ROOT_DIR, "youtube-notifier-bot.db") ) -VERSION: str = "2024-09-26.07" +VERSION: str = "2024-09-26.11" __all__ = [ "CONFIG_FILE_PATH", diff --git a/src/scheduler/jobs/telegram_notify_job/data_fetcher/__init__.py b/src/scheduler/jobs/telegram_notify_job/data_fetcher/__init__.py index c4af938..043f0e5 100644 --- a/src/scheduler/jobs/telegram_notify_job/data_fetcher/__init__.py +++ b/src/scheduler/jobs/telegram_notify_job/data_fetcher/__init__.py @@ -1,4 +1,9 @@ +from .kick import async_kick_fetch_livestreams from .twitch import async_twitch_fetch_livestreams from .youtube import async_youtube_fetch_livestreams -__all__ = ["async_twitch_fetch_livestreams", "async_youtube_fetch_livestreams"] +__all__ = [ + "async_kick_fetch_livestreams", + "async_twitch_fetch_livestreams", + "async_youtube_fetch_livestreams", +] diff --git a/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/__init__.py b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/__init__.py new file mode 100644 index 0000000..166c670 --- /dev/null +++ b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/__init__.py @@ -0,0 +1,4 @@ +from .fetcher import async_kick_fetch_livestreams + + +__all__ = ["async_kick_fetch_livestreams"] diff --git a/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/constants.py b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/constants.py new file mode 100644 index 0000000..b0950ee --- /dev/null +++ b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/constants.py @@ -0,0 +1,4 @@ +KICK_API_URL = "https://kick.com/api/v2/channels/" + + +__all__ = ["KICK_API_URL"] diff --git a/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/fetcher.py b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/fetcher.py new file mode 100644 index 0000000..1110192 --- /dev/null +++ b/src/scheduler/jobs/telegram_notify_job/data_fetcher/kick/fetcher.py @@ -0,0 +1,97 @@ +import asyncio +import operator +from datetime import datetime +from typing import Optional + +from aiohttp import ClientSession +from dateutil.tz import tzutc + +from .constants import KICK_API_URL +from src.db.models import ChannelModel +from src.logger import logger +from src.scheduler.jobs.telegram_notify_job.data_fetcher.utils import make_time_readable +from src.scheduler.jobs.telegram_notify_job.dto import ErrorVideoInfo +from src.scheduler.jobs.telegram_notify_job.dto import VideoInfo +from src.utils import extract_kick_username + + +async def async_fetch_livestream( + channel: ChannelModel, session: ClientSession +) -> Optional[VideoInfo] | ErrorVideoInfo: + """ + :param session: + :param channel: + :return: + """ + await logger.ainfo(channel.model_dump_json()) + + live_stream = None + try: + + username = extract_kick_username(channel.url) + if not username: + raise Exception(f"Cannot extract username for {channel.url}") + + async with session.get(f"{KICK_API_URL}{username}") as resp: + + raw_data = await resp.json() + + livestream: Optional[dict] = raw_data.get("livestream", None) + if livestream: + is_live = livestream.get("is_live", None) + + if is_live: + + concurrent_view_count = livestream["viewer_count"] + + duration = make_time_readable( + ( + datetime.now(tz=tzutc()) + - datetime.strptime( + livestream["start_time"], "%Y-%m-%d %H:%M:%S" + ) + ).seconds + ) + + live_stream = VideoInfo( + url=channel.url, + label=channel.label, + concurrent_view_count=concurrent_view_count, + duration=duration, + ) + + except Exception as ex: + await logger.aerror(f"Fetching info error: {channel.url} {ex}") + return ErrorVideoInfo(channel=channel.model_dump(), ex_message=str(ex)) + + return live_stream + + +async def async_kick_fetch_livestreams( + channels: list[ChannelModel], +) -> tuple[list[VideoInfo], list[ErrorVideoInfo]]: + """ + :param channels: + :return: + """ + + async with ClientSession() as client: + tasks = [ + async_fetch_livestream(channel=channel, session=client) + for channel in channels + ] + + data = await asyncio.gather(*tasks) + + errors = [stream for stream in data if isinstance(stream, ErrorVideoInfo)] + + live_streams = [stream for stream in data if isinstance(stream, VideoInfo)] + + live_streams = sorted( + live_streams, key=operator.attrgetter("concurrent_view_count"), reverse=True + ) + + return live_streams, errors + + +__all__ = ["async_kick_fetch_livestreams"] diff --git a/src/scheduler/jobs/telegram_notify_job/notifier/notify.py b/src/scheduler/jobs/telegram_notify_job/notifier/notify.py index 3c0930e..ab3511b 100644 --- a/src/scheduler/jobs/telegram_notify_job/notifier/notify.py +++ b/src/scheduler/jobs/telegram_notify_job/notifier/notify.py @@ -9,6 +9,7 @@ from sulguk import SULGUK_PARSE_MODE from twitchAPI.twitch import Twitch +from ..data_fetcher import async_kick_fetch_livestreams from ..data_fetcher import async_twitch_fetch_livestreams from ..data_fetcher import async_youtube_fetch_livestreams from ..dto import ErrorVideoInfo @@ -46,78 +47,120 @@ async def notify( # get channels channels = await dal.get_channels(enabled=True) - youtube_channels = [ - channel for channel in channels if channel.type.type == ChannelType.YOUTUBE - ] - twitch_channels = [ - channel for channel in channels if channel.type.type == ChannelType.TWITCH - ] + if channels: + youtube_channels = [ + channel for channel in channels if channel.type.type == ChannelType.YOUTUBE + ] + twitch_channels = [ + channel for channel in channels if channel.type.type == ChannelType.TWITCH + ] + kick_channels = [ + channel for channel in channels if channel.type.type == ChannelType.KICK + ] + + data: tuple[list[VideoInfo], list[ErrorVideoInfo]] = ( + await async_youtube_fetch_livestreams(channels=youtube_channels, ydl=ydl) + ) + + live_list, errors = data - data: tuple[list[VideoInfo], list[ErrorVideoInfo]] = ( - await async_youtube_fetch_livestreams(channels=youtube_channels, ydl=ydl) - ) + if twitch and twitch_channels: + _twitch = await twitch - live_list, errors = data + twitch_data: tuple[list[VideoInfo], list[ErrorVideoInfo]] = ( + await async_twitch_fetch_livestreams( + channels=twitch_channels, twitch=_twitch + ) + ) - if twitch: - _twitch = await twitch + twitch_live_list, twitch_errors = twitch_data + live_list.extend(twitch_live_list) + errors.extend(twitch_errors) - twitch_data: tuple[list[VideoInfo], list[ErrorVideoInfo]] = ( - await async_twitch_fetch_livestreams( - channels=twitch_channels, twitch=_twitch + if kick_channels: + kick_data: tuple[list[VideoInfo], list[ErrorVideoInfo]] = ( + await async_kick_fetch_livestreams(channels=kick_channels) ) + + kick_live_list, kick_errors = kick_data + live_list.extend(kick_live_list) + errors.extend(kick_errors) + + # logging errors + for error in errors: + await logger.aerror(f"Error with {error.channel['id']}: {error.ex_message}") + + await logger.ainfo(f"Live list length {len(live_list)}") + + message_text: Optional[str] = generate_jinja_report( + data=live_list, + report_template=report_template, + empty_template=empty_template, ) - twitch_live_list, twitch_errors = twitch_data - live_list.extend(twitch_live_list) - errors.extend(twitch_errors) - - # logging errors - for error in errors: - await logger.aerror(f"Error with {error.channel['id']}: {error.ex_message}") - - await logger.ainfo(f"Live list length {len(live_list)}") - - message_text: Optional[str] = generate_jinja_report( - data=live_list, report_template=report_template, empty_template=empty_template - ) - - if message_text: - message_id: Optional[int] = await dal.get_last_published_message_id() - - if message_id is not None: - try: - async with ChatActionSender( - bot=bot, chat_id=chat_id, action=ChatAction.TYPING - ): - is_needed_send = await check_if_need_send_instead_of_edit( - message_id=message_id, - delta_messages=3, - bot=bot, - from_chat_id=chat_id, - to_chat_id=temp_chat_id, - ) - - if not is_needed_send: - msg = await bot.edit_message_text( - chat_id=chat_id, - text=message_text, + if message_text: + message_id: Optional[int] = await dal.get_last_published_message_id() + + if message_id is not None: + try: + async with ChatActionSender( + bot=bot, chat_id=chat_id, action=ChatAction.TYPING + ): + is_needed_send = await check_if_need_send_instead_of_edit( message_id=message_id, - parse_mode=SULGUK_PARSE_MODE, - disable_web_page_preview=True, + delta_messages=3, + bot=bot, + from_chat_id=chat_id, + to_chat_id=temp_chat_id, ) - message_id = msg.message_id - await logger.ainfo(f"Msg: {message_id} edited") - else: - try: - await bot.delete_message( - chat_id=chat_id, message_id=message_id + + if not is_needed_send: + msg = await bot.edit_message_text( + chat_id=chat_id, + text=message_text, + message_id=message_id, + parse_mode=SULGUK_PARSE_MODE, + disable_web_page_preview=True, ) - except TelegramBadRequest as ex: - await logger.aerror( - f"Msg: {message_id} cannot be deleted {ex}" + message_id = msg.message_id + await logger.ainfo(f"Msg: {message_id} edited") + else: + try: + await bot.delete_message( + chat_id=chat_id, message_id=message_id + ) + except TelegramBadRequest as ex: + await logger.aerror( + f"Msg: {message_id} cannot be deleted {ex}" + ) + + msg = await bot.send_message( + text=message_text, + chat_id=chat_id, + parse_mode=SULGUK_PARSE_MODE, + disable_web_page_preview=True, ) + message_id = msg.message_id + + await logger.ainfo(f"Msg: {message_id} sent") + + except TelegramNetworkError as ex: + await logger.aerror(f"Exc: TelegramNetworkError finish cycle: {ex}") + return + + except TelegramBadRequest as ex: + if ( + str(ex).find( + "specified new message content and reply markup are exactly the same as a current" + " content and reply markup of the message" + ) + > -1 + ): + await logger.aerror(f"Same message: {ex}") + else: + await logger.aerror(f"Editing: {ex}") + msg = await bot.send_message( text=message_text, chat_id=chat_id, @@ -129,51 +172,24 @@ async def notify( await logger.ainfo(f"Msg: {message_id} sent") - except TelegramNetworkError as ex: - await logger.aerror(f"Exc: TelegramNetworkError finish cycle: {ex}") - return - - except TelegramBadRequest as ex: - if ( - str(ex).find( - "specified new message content and reply markup are exactly the same as a current" - " content and reply markup of the message" - ) - > -1 - ): - await logger.aerror(f"Same message: {ex}") - else: - await logger.aerror(f"Editing: {ex}") - - msg = await bot.send_message( - text=message_text, - chat_id=chat_id, - parse_mode=SULGUK_PARSE_MODE, - disable_web_page_preview=True, - ) - - message_id = msg.message_id - - await logger.ainfo(f"Msg: {message_id} sent") - - else: - await logger.ainfo(f"Message_id is None, sending") - - msg = await bot.send_message( - text=message_text, - chat_id=chat_id, - parse_mode=SULGUK_PARSE_MODE, - disable_web_page_preview=True, - ) + else: + await logger.ainfo(f"Message_id is None, sending") - message_id = msg.message_id + msg = await bot.send_message( + text=message_text, + chat_id=chat_id, + parse_mode=SULGUK_PARSE_MODE, + disable_web_page_preview=True, + ) - await logger.ainfo(f"Msg: {message_id} sent") + message_id = msg.message_id - if message_id: - await dal.create_message( - obj=MessageLogModel(message_id=message_id, text=message_text) - ) + await logger.ainfo(f"Msg: {message_id} sent") + + if message_id: + await dal.create_message( + obj=MessageLogModel(message_id=message_id, text=message_text) + ) __all__ = ["notify"] diff --git a/src/utils.py b/src/utils.py index 59da213..983f8a7 100644 --- a/src/utils.py +++ b/src/utils.py @@ -9,6 +9,10 @@ r"^https?://(?:www\.)?twitch\.tv/([\w-]+)/?$" ) +KICK_USERNAME_CHANNEL_LINK_PATTERN = re.compile( + r"^https?://(?:www\.)?kick\.com/([\w-]+)/?$" +) + def youtube_channel_url_validator(link: str) -> bool: match = re.match(YOUTUBE_USERNAME_CHANNEL_LINK_PATTERN, link) @@ -28,10 +32,19 @@ def extract_twitch_username(link: str) -> Optional[str]: def kick_channel_url_validator(link: str) -> bool: - return False + match = re.match(KICK_USERNAME_CHANNEL_LINK_PATTERN, link) + return bool(match) + + +def extract_kick_username(link: str) -> Optional[str]: + match = re.match(KICK_USERNAME_CHANNEL_LINK_PATTERN, link) + if match: + return match.group(1) + return None __all__ = [ + "extract_kick_username", "extract_twitch_username", "kick_channel_url_validator", "twitch_channel_url_validator",