Merge branch 'hikka-io:main' into feat/moderation-log

.gitignore

@@ -24,3 +24,4 @@ pyrightconfig.json
 # VsCode
 .vscode/*
 
+.profile_info

app/__init__.py

@@ -1,11 +1,12 @@
 from prometheus_fastapi_instrumentator import Instrumentator
+from app.middlewares import register_profiling_middleware
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from contextlib import asynccontextmanager
 from app.database import sessionmanager
 from app.utils import TimeoutMiddleware
-import fastapi.openapi.utils as fu
 from app.utils import get_settings
+import fastapi.openapi.utils as fu
 from fastapi import FastAPI
 from app import errors
 
@@ -24,9 +25,7 @@ async def lifespan(app: FastAPI):
             if sessionmanager._engine is not None:
                 await sessionmanager.close()
 
-    fu.validation_error_response_definition = (
-        errors.ErrorResponse.model_json_schema()
-    )
+    fu.validation_error_response_definition = errors.ErrorResponse.model_json_schema()
 
     app = FastAPI(
         title="Hikka API",
@@ -73,6 +72,8 @@ async def lifespan(app: FastAPI):
         allow_headers=["*"],
     )
 
+    register_profiling_middleware(app)
+
     app.add_exception_handler(errors.Abort, errors.abort_handler)
     app.add_exception_handler(
         RequestValidationError,

app/constants.py

@@ -230,8 +230,8 @@
 UPLOAD_COVER = "cover"
 
 # Todo types
-TODO_ANIME_SYNOPSIS_UA = "synopsis_ua"
-TODO_ANIME_TITLE_UA = "title_ua"
+TODO_SYNOPSIS_UA = "synopsis_ua"
+TODO_TITLE_UA = "title_ua"
 
 # Log types
 LOG_FAVOURITE = "favourite_add"

app/edit/router.py

@@ -1,4 +1,6 @@
 from sqlalchemy.ext.asyncio import AsyncSession
+from app.manga.schemas import MangaPaginationResponse
+from app.novel.schemas import NovelPaginationResponse
 from app.schemas import AnimePaginationResponse
 from fastapi import APIRouter, Depends
 from app.database import get_session
@@ -40,10 +42,11 @@
 )
 
 from .schemas import (
+    ContentToDoEnum,
+    EditContentToDoEnum,
     EditContentTypeEnum,
     EditListResponse,
     EditSearchArgs,
-    AnimeToDoEnum,
     EditResponse,
     EditArgs,
 )
@@ -135,21 +138,27 @@ async def deny_edit(
     return await service.deny_pending_edit(session, edit, moderator)
 
 
-@router.get("/todo/anime/{todo_type}", response_model=AnimePaginationResponse)
-async def get_edit_todo(
-    todo_type: AnimeToDoEnum,
+@router.get(
+    "/todo/{content_type}/{todo_type}",
+    response_model=AnimePaginationResponse
+    | MangaPaginationResponse
+    | NovelPaginationResponse,
+)
+async def get_content_edit_todo(
+    content_type: EditContentToDoEnum,
+    todo_type: ContentToDoEnum,
     session: AsyncSession = Depends(get_session),
     request_user: User | None = Depends(auth_required(optional=True)),
     page: int = Depends(get_page),
     size: int = Depends(get_size),
 ):
     limit, offset = pagination(page, size)
-    total = await service.anime_todo_total(session, todo_type)
-    anime = await service.anime_todo(
-        session, todo_type, request_user, limit, offset
+    total = await service.content_todo_total(session, content_type, todo_type)
+    content = await service.content_todo(
+        session, content_type, todo_type, request_user, limit, offset
     )
 
     return {
         "pagination": pagination_dict(total, page, limit),
-        "list": anime.unique().all(),
+        "list": content.unique().all(),
     }

app/edit/schemas.py

@@ -16,9 +16,15 @@
 
 
 # Enums
-class AnimeToDoEnum(str, Enum):
-    synopsis_ua = constants.TODO_ANIME_SYNOPSIS_UA
-    title_ua = constants.TODO_ANIME_TITLE_UA
+class ContentToDoEnum(str, Enum):
+    synopsis_ua = constants.TODO_SYNOPSIS_UA
+    title_ua = constants.TODO_TITLE_UA
+
+
+class EditContentToDoEnum(str, Enum):
+    content_anime = constants.CONTENT_ANIME
+    content_manga = constants.CONTENT_MANGA
+    content_novel = constants.CONTENT_NOVEL
 
 
 class EditContentTypeEnum(str, Enum):

app/edit/service.py

@@ -4,6 +4,8 @@
 from sqlalchemy.sql.selectable import Select
 from sqlalchemy.orm import with_expression
 from sqlalchemy.orm import joinedload
+
+from app.models.list.read import MangaRead, NovelRead
 from .utils import calculate_before
 from app.utils import utcnow
 from app import constants
@@ -17,9 +19,10 @@
 )
 
 from .schemas import (
+    ContentToDoEnum,
+    EditContentToDoEnum,
     EditContentTypeEnum,
     EditSearchArgs,
-    AnimeToDoEnum,
     EditArgs,
 )
 
@@ -140,7 +143,8 @@ async def edits_search_filter(
         query = query.filter(Edit.status == args.status)
 
     query = query.filter(
-        Edit.system_edit == False, Edit.hidden == False  # noqa: E712
+        Edit.system_edit == False,  # noqa: E712
+        Edit.hidden == False,  # noqa: E712
     )
 
     return query
@@ -405,54 +409,79 @@ async def deny_pending_edit(
     return edit
 
 
-async def anime_todo_total(
+async def content_todo_total(
     session: AsyncSession,
-    todo_type: AnimeToDoEnum,
+    content_type: EditContentToDoEnum,
+    todo_type: ContentToDoEnum,
 ):
-    query = select(func.count(Anime.id)).filter(
-        ~Anime.media_type.in_([constants.MEDIA_TYPE_MUSIC]),
-        Anime.deleted == False,  # noqa: E712
+    match content_type:
+        case constants.CONTENT_ANIME:
+            content_type = Anime
+        case constants.CONTENT_MANGA:
+            content_type = Manga
+        case constants.CONTENT_NOVEL:
+            content_type = Novel
+
+    query = select(func.count(content_type.id)).filter(
+        ~content_type.media_type.in_([constants.MEDIA_TYPE_MUSIC]),
+        content_type.deleted == False,  # noqa: E712
     )
 
-    if todo_type == constants.TODO_ANIME_TITLE_UA:
-        query = query.filter(Anime.title_ua == None)  # noqa: E711
+    if todo_type == constants.TODO_TITLE_UA:
+        query = query.filter(content_type.title_ua == None)  # noqa: E711
 
-    if todo_type == constants.TODO_ANIME_SYNOPSIS_UA:
-        query = query.filter(Anime.synopsis_ua == None)  # noqa: E711
+    if todo_type == constants.TODO_SYNOPSIS_UA:
+        query = query.filter(content_type.synopsis_ua == None)  # noqa: E711
 
     return await session.scalar(query)
 
 
-async def anime_todo(
+async def content_todo(
     session: AsyncSession,
-    todo_type: AnimeToDoEnum,
+    content_type: EditContentTypeEnum,
+    todo_type: ContentToDoEnum,
     request_user: User | None,
     limit: int,
     offset: int,
 ):
+    match content_type:
+        case constants.CONTENT_ANIME:
+            content_type = Anime
+            option = AnimeWatch
+        case constants.CONTENT_MANGA:
+            content_type = Manga
+            option = MangaRead
+        case constants.CONTENT_NOVEL:
+            content_type = Novel
+            option = NovelRead
+
     # Load request user watch statuses here
     load_options = [
-        joinedload(Anime.watch),
+        joinedload(
+            content_type.read if content_type != Anime else content_type.watch
+        ),
         with_loader_criteria(
-            AnimeWatch,
-            AnimeWatch.user_id == request_user.id if request_user else None,
+            option,
+            option.user_id == request_user.id if request_user else None,
         ),
     ]
 
-    query = select(Anime).filter(
-        ~Anime.media_type.in_([constants.MEDIA_TYPE_MUSIC]),
-        Anime.deleted == False,  # noqa: E712
+    query = select(content_type).filter(
+        ~content_type.media_type.in_([constants.MEDIA_TYPE_MUSIC]),
+        content_type.deleted == False,  # noqa: E712
     )
 
-    if todo_type == constants.TODO_ANIME_TITLE_UA:
-        query = query.filter(Anime.title_ua == None)  # noqa: E711
+    if todo_type == constants.TODO_TITLE_UA:
+        query = query.filter(content_type.title_ua == None)  # noqa: E711
 
-    if todo_type == constants.TODO_ANIME_SYNOPSIS_UA:
-        query = query.filter(Anime.synopsis_ua == None)  # noqa: E711
+    if todo_type == constants.TODO_SYNOPSIS_UA:
+        query = query.filter(content_type.synopsis_ua == None)  # noqa: E711
 
     return await session.scalars(
         query.order_by(
-            desc(Anime.score), desc(Anime.scored_by), desc(Anime.content_id)
+            desc(content_type.score),
+            desc(content_type.scored_by),
+            desc(content_type.content_id),
         )
         .options(*load_options)
         .limit(limit)

app/middlewares.py

@@ -0,0 +1,49 @@
+from pyinstrument.renderers.html import HTMLRenderer
+from fastapi import FastAPI, Request
+from app.utils import get_settings
+from pyinstrument import Profiler
+from datetime import datetime
+from typing import Callable
+from pathlib import Path
+
+
+# https://blog.balthazar-rouberol.com/how-to-profile-a-fastapi-asynchronous-request
+def register_profiling_middleware(app: FastAPI):
+    settings = get_settings()
+
+    if not settings.profiling.enabled:
+        return
+
+    if settings.profiling.trigger not in ["query", "all"]:
+        return
+
+    @app.middleware("http")
+    async def profile_request(request: Request, call_next: Callable):
+        """
+        Profile the current request
+        Taken from https://pyinstrument.readthedocs.io/en/latest/guide.html#profile-a-web-request-in-fastapi
+        with small improvements.
+        """
+
+        if (
+            settings.profiling.trigger == "all"
+            or settings.profiling.trigger == "query"
+            and request.query_params.get("profiling_flag", False)
+            and request.query_params.get("profiling_secret", None) == settings.profiling.profiling_secret
+        ):
+            with Profiler(interval=0.001, async_mode="enabled") as profiler:
+                response = await call_next(request)
+
+            path = f"{settings.profiling.path}/{request.url.path}"
+            path = path.replace("//", "/")
+            Path(path).mkdir(parents=True, exist_ok=True)
+
+            timestamp = datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
+
+            with open(f"{path}/{timestamp}_profile.html", "w") as out:
+                out.write(profiler.output(renderer=HTMLRenderer()))
+
+            return response
+
+        # Proceed without profiling
+        return await call_next(request)

app/related/service.py

@@ -1,5 +1,6 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import with_loader_criteria
+from sqlalchemy.orm import subqueryload
 from sqlalchemy.orm import joinedload
 from sqlalchemy import select
 
@@ -24,21 +25,21 @@ async def get_franchise(
         select(Franchise)
         .filter(Franchise.id == content.franchise_id)
         .options(
-            joinedload(Franchise.anime).joinedload(Anime.watch),
+            subqueryload(Franchise.anime).joinedload(Anime.watch),
             with_loader_criteria(
                 AnimeWatch,
                 AnimeWatch.user_id == request_user.id if request_user else None,
             ),
         )
         .options(
-            joinedload(Franchise.manga).joinedload(Manga.read),
+            subqueryload(Franchise.manga).joinedload(Manga.read),
             with_loader_criteria(
                 MangaRead,
                 MangaRead.user_id == request_user.id if request_user else None,
             ),
         )
         .options(
-            joinedload(Franchise.novel).joinedload(Novel.read),
+            subqueryload(Franchise.novel).joinedload(Novel.read),
             with_loader_criteria(
                 NovelRead,
                 NovelRead.user_id == request_user.id if request_user else None,

docs/settings.example.toml

@@ -42,6 +42,12 @@
     endpoint = "https://endpoint.s3.provider.com"
     bucket = "hikka"
 
+    [default.profiling]
+    enabled = true
+    trigger = "query"  # [query/all]
+    path = ".profile_info"
+    profiling_secret = "secret"
+
 [testing]
     [testing.database]
     endpoint = "postgresql+asyncpg://user:password@localhost:5432/database-tests"
@@ -73,3 +79,9 @@
     secret = "FAKE_SECRET"
     endpoint = "https://fake.s3.example.com"
     bucket = "hikka-test"
+
+    [testing.profiling]
+    enabled = false
+    trigger = "query"  # [query/all]
+    path = ".profile_info"
+    profiling_secret = "secret"