From faf1207a642fa151e359aa9a40c592bf434de10f Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sat, 6 Jul 2024 14:05:58 +0300 Subject: [PATCH 01/12] feat(moderation-log): initial code --- ...22-46bff46ba582_added_moderation_models.py | 36 +++++++++++ app/__init__.py | 3 + app/models/__init__.py | 3 + app/models/moderation/moderation.py | 38 ++++++++++++ app/moderation/__init__.py | 3 + app/moderation/router.py | 42 +++++++++++++ app/moderation/schemas.py | 23 +++++++ app/moderation/service.py | 26 ++++++++ app/sync/__init__.py | 3 + app/sync/moderation/__init__.py | 62 +++++++++++++++++++ app/sync/moderation/generate/__init__.py | 5 ++ app/sync/moderation/generate/edit_action.py | 43 +++++++++++++ app/sync/moderation/service.py | 24 +++++++ sync.py | 2 + 14 files changed, 313 insertions(+) create mode 100644 alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py create mode 100644 app/models/moderation/moderation.py create mode 100644 app/moderation/__init__.py create mode 100644 app/moderation/router.py create mode 100644 app/moderation/schemas.py create mode 100644 app/moderation/service.py create mode 100644 app/sync/moderation/__init__.py create mode 100644 app/sync/moderation/generate/__init__.py create mode 100644 app/sync/moderation/generate/edit_action.py create mode 100644 app/sync/moderation/service.py diff --git a/alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py b/alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py new file mode 100644 index 00000000..c541b257 --- /dev/null +++ b/alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py @@ -0,0 +1,36 @@ +"""Added moderation models + +Revision ID: 46bff46ba582 +Revises: 4c13fdf8868d +Create Date: 2024-07-06 13:22:44.814126 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = '46bff46ba582' +down_revision = '4c13fdf8868d' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('service_moderation', + sa.Column('user_id', sa.Uuid(), nullable=True), + sa.Column('content_type', sa.String(), nullable=False), + sa.Column('content_id', sa.Uuid(), nullable=False), + sa.Column('id', sa.Uuid(), nullable=False), + sa.Column('created', sa.DateTime(), nullable=False), + sa.ForeignKeyConstraint(['user_id'], ['service_users.id'], ), + sa.PrimaryKeyConstraint('id') + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('service_moderation') + # ### end Alembic commands ### diff --git a/app/__init__.py b/app/__init__.py index 09052c3f..fcb5d18f 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -47,6 +47,7 @@ async def lifespan(app: FastAPI): {"name": "Read"}, {"name": "Related"}, {"name": "Edit"}, + {"name": "Moderation"}, {"name": "Settings"}, {"name": "Schedule"}, {"name": "Upload"}, @@ -82,6 +83,7 @@ async def lifespan(app: FastAPI): from .integrations import router as integrations_router from .collections import router as collections_router from .characters import router as characters_router + from .moderation import router as moderation_router from .companies import router as companies_router from .favourite import router as favourite_router from .settings import router as settings_router @@ -109,6 +111,7 @@ async def lifespan(app: FastAPI): app.include_router(integrations_router) app.include_router(collections_router) app.include_router(characters_router) + app.include_router(moderation_router) app.include_router(companies_router) app.include_router(favourite_router) app.include_router(settings_router) diff --git a/app/models/__init__.py b/app/models/__init__.py index cfa4be1b..4a227557 100644 --- a/app/models/__init__.py +++ b/app/models/__init__.py @@ -24,6 +24,8 @@ from .edit.edit import NovelEdit from .edit.edit import Edit +from .moderation.moderation import Moderation + from .comments.comment import CollectionComment from .comments.vote import CommentVoteLegacy from .comments.comment import AnimeComment @@ -109,6 +111,7 @@ "ReadImportHistory", "WatchHistory", "History", + "Moderation", "UserOAuth", "Follow", "User", diff --git a/app/models/moderation/moderation.py b/app/models/moderation/moderation.py new file mode 100644 index 00000000..f527d1c0 --- /dev/null +++ b/app/models/moderation/moderation.py @@ -0,0 +1,38 @@ +from sqlalchemy.orm import Mapped, mapped_column, relationship +from ..mixins import CreatedMixin +from sqlalchemy import ForeignKey +from ..base import Base +from uuid import UUID + + +class Moderation(Base, CreatedMixin): + __tablename__ = "service_moderation" + __mapper_args__ = { + "polymorphic_identity": "default", + "polymorphic_on": "content_type", + } + + content_type: Mapped[str] + content_id: Mapped[UUID] + + user_id = mapped_column(ForeignKey("service_users.id")) + user: Mapped["User"] = relationship(foreign_keys=[user_id]) + + +class EditModeration(Moderation): + __mapper_args__ = { + "polymorphic_identity": "edit", + "eager_defaults": True, + } + + content_id = mapped_column( + ForeignKey("service_edits.id", ondelete="CASCADE"), + use_existing_column=True, + index=True, + ) + + content: Mapped["Edit"] = relationship( + primaryjoin="Edit.id == EditModeration.content_id", + foreign_keys=[content_id], + lazy="immediate", + ) diff --git a/app/moderation/__init__.py b/app/moderation/__init__.py new file mode 100644 index 00000000..5bc0c2e6 --- /dev/null +++ b/app/moderation/__init__.py @@ -0,0 +1,3 @@ +from .router import router + +__all__ = ["router"] diff --git a/app/moderation/router.py b/app/moderation/router.py new file mode 100644 index 00000000..ec271ed2 --- /dev/null +++ b/app/moderation/router.py @@ -0,0 +1,42 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from fastapi import APIRouter, Depends +from app.database import get_session +from . import service + + +from .schemas import ( + ModerationPaginationResponse, +) + +from app.utils import ( + pagination_dict, + pagination, +) + +from app.dependencies import ( + get_page, + get_size, +) + + +router = APIRouter(prefix="/moderation", tags=["Moderation"]) + + +@router.get( + "/history", + response_model=ModerationPaginationResponse, + summary="Moderation history", +) +async def moderation_history( + session: AsyncSession = Depends(get_session), + page: int = Depends(get_page), + size: int = Depends(get_size), +): + limit, offset = pagination(page, size) + total = await service.get_moderation_count(session) + history = await service.get_moderation(session, limit, offset) + + return { + "pagination": pagination_dict(total, page, limit), + "list": history.all(), + } diff --git a/app/moderation/schemas.py b/app/moderation/schemas.py new file mode 100644 index 00000000..f33b0f8d --- /dev/null +++ b/app/moderation/schemas.py @@ -0,0 +1,23 @@ +from app.edit.schemas import EditResponse +from app.schemas import datetime_pd + +from app.schemas import ( + PaginationResponse, + UserResponse, + CustomModel, +) + + +# Responses +class ModerationResponse(CustomModel): + content: EditResponse | None = None + created: datetime_pd + user: UserResponse + history_type: str + reference: str + data: dict + + +class ModerationPaginationResponse(CustomModel): + pagination: PaginationResponse + list: list[ModerationResponse] diff --git a/app/moderation/service.py b/app/moderation/service.py new file mode 100644 index 00000000..e2d0c5e2 --- /dev/null +++ b/app/moderation/service.py @@ -0,0 +1,26 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, desc, func +from sqlalchemy.orm import joinedload + +from app.models import ( + Moderation, + User, +) + + +async def get_moderation_count(session: AsyncSession) -> int: + return await session.scalar(select(func.count(Moderation.id))) + + +async def get_moderation( + session: AsyncSession, limit: int, offset: int +) -> User: + """Get moderation history""" + + return await session.scalars( + select(Moderation) + .options(joinedload(Moderation.user)) + .order_by(desc(Moderation.created)) + .limit(limit) + .offset(offset) + ) diff --git a/app/sync/__init__.py b/app/sync/__init__.py index ee834e05..87ccf63f 100644 --- a/app/sync/__init__.py +++ b/app/sync/__init__.py @@ -21,6 +21,8 @@ from .history import update_history +from .moderation import update_moderation + from .weights import update_weights from .sitemap import update_sitemap @@ -47,6 +49,7 @@ "update_ranking_all", "aggregator_people", "aggregator_genres", + "update_moderation", "aggregator_anime", "aggregator_manga", "aggregator_novel", diff --git a/app/sync/moderation/__init__.py b/app/sync/moderation/__init__.py new file mode 100644 index 00000000..2905958d --- /dev/null +++ b/app/sync/moderation/__init__.py @@ -0,0 +1,62 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import SystemTimestamp, Log +from datetime import datetime, timedelta +from app.database import sessionmanager +from sqlalchemy import select, asc +from app import constants + +from .generate import ( + generate_edit_action, +) + + +async def generate_moderation(session: AsyncSession): + edit_delta = timedelta(hours=3) + + # Get system timestamp for latest moderation update + if not ( + system_timestamp := await session.scalar( + select(SystemTimestamp).filter(SystemTimestamp.name == "moderation") + ) + ): + system_timestamp = SystemTimestamp( + **{ + "timestamp": datetime(2024, 1, 13), + "name": "moderation", + } + ) + + # Get new logs that were created since last update + logs = await session.scalars( + select(Log) + .filter( + Log.log_type.in_( + [ + constants.LOG_EDIT_ACCEPT, + constants.LOG_EDIT_DENY, + ] + ) + ) + .filter(Log.created > system_timestamp.timestamp) + .order_by(asc(Log.created)) + ) + + for log in logs: + # We set timestamp here because after thay it won't be set due to continue + system_timestamp.timestamp = log.created + + if log.log_type in [ + constants.LOG_EDIT_ACCEPT, + constants.LOG_EDIT_DENY, + ]: + await generate_edit_action(session, log, edit_delta) + + session.add(system_timestamp) + await session.commit() + + +async def update_moderation(): + """Generate moderation history from logs""" + + async with sessionmanager.session() as session: + await generate_moderation(session) diff --git a/app/sync/moderation/generate/__init__.py b/app/sync/moderation/generate/__init__.py new file mode 100644 index 00000000..576d1b43 --- /dev/null +++ b/app/sync/moderation/generate/__init__.py @@ -0,0 +1,5 @@ +from .edit_action import generate_edit_action + +__all__ = [ + "generate_edit_action", +] diff --git a/app/sync/moderation/generate/edit_action.py b/app/sync/moderation/generate/edit_action.py new file mode 100644 index 00000000..1a9393d1 --- /dev/null +++ b/app/sync/moderation/generate/edit_action.py @@ -0,0 +1,43 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from datetime import timedelta +from app import constants +from ...moderation import service + + +async def generate_edit_action( + session: AsyncSession, + log: Log, + edit_delta: timedelta, +): + threshold = log.created - edit_delta + + history_type = { + constants.EDIT_ACCEPTED: constants.HISTORY_EDIT_ACCEPT, + constants.EDIT_DENIED: constants.HISTORY_EDIT_DENY, + }.get(log.data["content_type"]) + + if not history_type: + return + + moderation = await service.get_moderation( + session, + history_type, + log.target_id, + log.user_id, + threshold, + ) + + if not moderation: + moderation = Moderation( + **{ + "history_type": history_type, + "used_logs": [str(log.id)], + "target_id": log.target_id, + "user_id": log.user_id, + "created": log.created, + } + ) + + session.add(moderation) + await session.commit() diff --git a/app/sync/moderation/service.py b/app/sync/moderation/service.py new file mode 100644 index 00000000..019bfb96 --- /dev/null +++ b/app/sync/moderation/service.py @@ -0,0 +1,24 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy import select, desc +from app.models import Moderation +# from datetime import datetime +# from uuid import UUID + + +async def get_moderation( + session: AsyncSession, + # content_type: str, + # content_id: UUID, + # user_id: UUID, + # threshold: datetime, +): + return await session.scalar( + select(Moderation) + # .filter( + # Moderation.content_type == content_type, + # Moderation.content_id == content_id, + # Moderation.user_id == user_id, + # Moderation.created > threshold, + # ) + .order_by(desc(Moderation.created)) + ) diff --git a/sync.py b/sync.py index d6ae5711..b3bd9fc3 100644 --- a/sync.py +++ b/sync.py @@ -6,6 +6,7 @@ from app.sync import ( update_notifications, update_ranking_all, + update_moderation, update_activity, update_schedule, update_ranking, @@ -22,6 +23,7 @@ def init_scheduler(): sessionmanager.init(settings.database.endpoint) scheduler.add_job(update_notifications, "interval", seconds=10) + scheduler.add_job(update_moderation, "interval", seconds=10) scheduler.add_job(update_ranking_all, "interval", hours=1) scheduler.add_job(update_activity, "interval", seconds=10) scheduler.add_job(update_schedule, "interval", minutes=5) From 5e15121d0ff30120204e28f45974855485dea358 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 00:58:36 +0300 Subject: [PATCH 02/12] fix: make code to work --- ...06-3038cd0ae3b0_added_moderation_model.py} | 17 ++++--- app/models/moderation/moderation.py | 34 ++++---------- app/moderation/router.py | 42 +++++++++++++++--- app/moderation/schemas.py | 12 +---- app/moderation/service.py | 26 +++++++++-- app/sync/moderation/__init__.py | 22 ++++------ app/sync/moderation/generate/__init__.py | 7 ++- app/sync/moderation/generate/edit_accept.py | 44 +++++++++++++++++++ app/sync/moderation/generate/edit_action.py | 43 ------------------ app/sync/moderation/generate/edit_deny.py | 44 +++++++++++++++++++ app/sync/moderation/service.py | 36 ++++++++------- 11 files changed, 202 insertions(+), 125 deletions(-) rename alembic/versions/{2024_07_06_1322-46bff46ba582_added_moderation_models.py => 2024_07_07_0006-3038cd0ae3b0_added_moderation_model.py} (57%) create mode 100644 app/sync/moderation/generate/edit_accept.py delete mode 100644 app/sync/moderation/generate/edit_action.py create mode 100644 app/sync/moderation/generate/edit_deny.py diff --git a/alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py b/alembic/versions/2024_07_07_0006-3038cd0ae3b0_added_moderation_model.py similarity index 57% rename from alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py rename to alembic/versions/2024_07_07_0006-3038cd0ae3b0_added_moderation_model.py index c541b257..894f61f3 100644 --- a/alembic/versions/2024_07_06_1322-46bff46ba582_added_moderation_models.py +++ b/alembic/versions/2024_07_07_0006-3038cd0ae3b0_added_moderation_model.py @@ -1,16 +1,16 @@ -"""Added moderation models +"""Added moderation model -Revision ID: 46bff46ba582 +Revision ID: 3038cd0ae3b0 Revises: 4c13fdf8868d -Create Date: 2024-07-06 13:22:44.814126 +Create Date: 2024-07-07 00:06:45.747702 """ from alembic import op import sqlalchemy as sa - +from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '46bff46ba582' +revision = '3038cd0ae3b0' down_revision = '4c13fdf8868d' branch_labels = None depends_on = None @@ -19,18 +19,21 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.create_table('service_moderation', + sa.Column('target_type', sa.String(length=64), nullable=False), + sa.Column('data', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('log_id', sa.Uuid(), nullable=True), sa.Column('user_id', sa.Uuid(), nullable=True), - sa.Column('content_type', sa.String(), nullable=False), - sa.Column('content_id', sa.Uuid(), nullable=False), sa.Column('id', sa.Uuid(), nullable=False), sa.Column('created', sa.DateTime(), nullable=False), sa.ForeignKeyConstraint(['user_id'], ['service_users.id'], ), sa.PrimaryKeyConstraint('id') ) + op.create_index(op.f('ix_service_moderation_target_type'), 'service_moderation', ['target_type'], unique=False) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### + op.drop_index(op.f('ix_service_moderation_target_type'), table_name='service_moderation') op.drop_table('service_moderation') # ### end Alembic commands ### diff --git a/app/models/moderation/moderation.py b/app/models/moderation/moderation.py index f527d1c0..556decbe 100644 --- a/app/models/moderation/moderation.py +++ b/app/models/moderation/moderation.py @@ -1,38 +1,20 @@ -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.dialects.postgresql import JSONB from ..mixins import CreatedMixin +from sqlalchemy.orm import mapped_column +from sqlalchemy.orm import relationship +from sqlalchemy.orm import Mapped from sqlalchemy import ForeignKey +from sqlalchemy import String from ..base import Base from uuid import UUID class Moderation(Base, CreatedMixin): __tablename__ = "service_moderation" - __mapper_args__ = { - "polymorphic_identity": "default", - "polymorphic_on": "content_type", - } - content_type: Mapped[str] - content_id: Mapped[UUID] + target_type: Mapped[str] = mapped_column(String(64), index=True) + data: Mapped[dict] = mapped_column(JSONB, default={}) + log_id: Mapped[UUID] = mapped_column(nullable=True) user_id = mapped_column(ForeignKey("service_users.id")) user: Mapped["User"] = relationship(foreign_keys=[user_id]) - - -class EditModeration(Moderation): - __mapper_args__ = { - "polymorphic_identity": "edit", - "eager_defaults": True, - } - - content_id = mapped_column( - ForeignKey("service_edits.id", ondelete="CASCADE"), - use_existing_column=True, - index=True, - ) - - content: Mapped["Edit"] = relationship( - primaryjoin="Edit.id == EditModeration.content_id", - foreign_keys=[content_id], - lazy="immediate", - ) diff --git a/app/moderation/router.py b/app/moderation/router.py index ec271ed2..0d88f451 100644 --- a/app/moderation/router.py +++ b/app/moderation/router.py @@ -1,6 +1,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from fastapi import APIRouter, Depends +from app import constants from app.database import get_session +from app.models import User from . import service @@ -16,27 +18,57 @@ from app.dependencies import ( get_page, get_size, + get_user, ) +class ModerationError(Exception): + def __init__(self, content: dict, status_code: int): + self.status_code = status_code + self.content = content + + router = APIRouter(prefix="/moderation", tags=["Moderation"]) @router.get( - "/history", + "/log", response_model=ModerationPaginationResponse, - summary="Moderation history", + summary="Moderation log", ) -async def moderation_history( +async def moderation_log( session: AsyncSession = Depends(get_session), page: int = Depends(get_page), size: int = Depends(get_size), ): limit, offset = pagination(page, size) total = await service.get_moderation_count(session) - history = await service.get_moderation(session, limit, offset) + moderation = await service.get_moderation(session, limit, offset) + + return { + "pagination": pagination_dict(total, page, limit), + "list": moderation.all(), + } + + +@router.get( + "/{username}/log", + response_model=ModerationPaginationResponse, + summary="User moderation log", +) +async def moderation_user_log( + session: AsyncSession = Depends(get_session), + user: User = Depends(get_user), + page: int = Depends(get_page), + size: int = Depends(get_size), +): + limit, offset = pagination(page, size) + total = await service.get_user_moderation_count(session, user.id) + moderation = await service.get_user_moderation( + session, user.id, limit, offset + ) return { "pagination": pagination_dict(total, page, limit), - "list": history.all(), + "list": moderation.all(), } diff --git a/app/moderation/schemas.py b/app/moderation/schemas.py index f33b0f8d..ce91bb0e 100644 --- a/app/moderation/schemas.py +++ b/app/moderation/schemas.py @@ -1,19 +1,11 @@ -from app.edit.schemas import EditResponse +from app.schemas import PaginationResponse, CustomModel from app.schemas import datetime_pd -from app.schemas import ( - PaginationResponse, - UserResponse, - CustomModel, -) - # Responses class ModerationResponse(CustomModel): - content: EditResponse | None = None + target_type: str created: datetime_pd - user: UserResponse - history_type: str reference: str data: dict diff --git a/app/moderation/service.py b/app/moderation/service.py index e2d0c5e2..416eb0be 100644 --- a/app/moderation/service.py +++ b/app/moderation/service.py @@ -1,6 +1,5 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, desc, func -from sqlalchemy.orm import joinedload from app.models import ( Moderation, @@ -15,11 +14,32 @@ async def get_moderation_count(session: AsyncSession) -> int: async def get_moderation( session: AsyncSession, limit: int, offset: int ) -> User: - """Get moderation history""" + """Get moderation log""" return await session.scalars( select(Moderation) - .options(joinedload(Moderation.user)) + .order_by(desc(Moderation.created)) + .limit(limit) + .offset(offset) + ) + + +async def get_user_moderation_count( + session: AsyncSession, user_id: User.id +) -> int: + return await session.scalar( + select(func.count(Moderation.id)).filter(Moderation.user_id == user_id) + ) + + +async def get_user_moderation( + session: AsyncSession, user_id: User.id, limit: int, offset: int +) -> User: + """Get user moderation log""" + + return await session.scalars( + select(Moderation) + .filter(Moderation.user_id == user_id) .order_by(desc(Moderation.created)) .limit(limit) .offset(offset) diff --git a/app/sync/moderation/__init__.py b/app/sync/moderation/__init__.py index 2905958d..bc348716 100644 --- a/app/sync/moderation/__init__.py +++ b/app/sync/moderation/__init__.py @@ -1,19 +1,17 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.models import SystemTimestamp, Log -from datetime import datetime, timedelta from app.database import sessionmanager from sqlalchemy import select, asc +from datetime import datetime from app import constants from .generate import ( - generate_edit_action, + generate_edit_accept, + generate_edit_deny, ) async def generate_moderation(session: AsyncSession): - edit_delta = timedelta(hours=3) - - # Get system timestamp for latest moderation update if not ( system_timestamp := await session.scalar( select(SystemTimestamp).filter(SystemTimestamp.name == "moderation") @@ -26,7 +24,6 @@ async def generate_moderation(session: AsyncSession): } ) - # Get new logs that were created since last update logs = await session.scalars( select(Log) .filter( @@ -42,21 +39,20 @@ async def generate_moderation(session: AsyncSession): ) for log in logs: - # We set timestamp here because after thay it won't be set due to continue system_timestamp.timestamp = log.created - if log.log_type in [ - constants.LOG_EDIT_ACCEPT, - constants.LOG_EDIT_DENY, - ]: - await generate_edit_action(session, log, edit_delta) + if log.log_type == constants.LOG_EDIT_ACCEPT: + await generate_edit_accept(session, log) + + if log.log_type == constants.LOG_EDIT_DENY: + await generate_edit_deny(session, log) session.add(system_timestamp) await session.commit() async def update_moderation(): - """Generate moderation history from logs""" + """Generate moderation log from logs""" async with sessionmanager.session() as session: await generate_moderation(session) diff --git a/app/sync/moderation/generate/__init__.py b/app/sync/moderation/generate/__init__.py index 576d1b43..c6a33a3b 100644 --- a/app/sync/moderation/generate/__init__.py +++ b/app/sync/moderation/generate/__init__.py @@ -1,5 +1,8 @@ -from .edit_action import generate_edit_action +from .edit_accept import generate_edit_accept +from .edit_deny import generate_edit_deny + __all__ = [ - "generate_edit_action", + "generate_edit_accept", + "generate_edit_deny", ] diff --git a/app/sync/moderation/generate/edit_accept.py b/app/sync/moderation/generate/edit_accept.py new file mode 100644 index 00000000..2dc32034 --- /dev/null +++ b/app/sync/moderation/generate/edit_accept.py @@ -0,0 +1,44 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_edit_accept(session: AsyncSession, log: Log): + target_type = constants.MODERATION_EDIT_ACCEPTED + + if not (edit := await service.get_edit(session, log.target_id)): + return + + if not edit.author: + return + + if edit.author_id == edit.moderator_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "description": edit.description, + "edit_id": edit.edit_id, + "username": log.user.username, + "avatar": log.user.avatar, + }, + } + ) + + session.add(moderation) diff --git a/app/sync/moderation/generate/edit_action.py b/app/sync/moderation/generate/edit_action.py deleted file mode 100644 index 1a9393d1..00000000 --- a/app/sync/moderation/generate/edit_action.py +++ /dev/null @@ -1,43 +0,0 @@ -from sqlalchemy.ext.asyncio import AsyncSession -from app.models import Moderation, Log -from datetime import timedelta -from app import constants -from ...moderation import service - - -async def generate_edit_action( - session: AsyncSession, - log: Log, - edit_delta: timedelta, -): - threshold = log.created - edit_delta - - history_type = { - constants.EDIT_ACCEPTED: constants.HISTORY_EDIT_ACCEPT, - constants.EDIT_DENIED: constants.HISTORY_EDIT_DENY, - }.get(log.data["content_type"]) - - if not history_type: - return - - moderation = await service.get_moderation( - session, - history_type, - log.target_id, - log.user_id, - threshold, - ) - - if not moderation: - moderation = Moderation( - **{ - "history_type": history_type, - "used_logs": [str(log.id)], - "target_id": log.target_id, - "user_id": log.user_id, - "created": log.created, - } - ) - - session.add(moderation) - await session.commit() diff --git a/app/sync/moderation/generate/edit_deny.py b/app/sync/moderation/generate/edit_deny.py new file mode 100644 index 00000000..fcc8aa7a --- /dev/null +++ b/app/sync/moderation/generate/edit_deny.py @@ -0,0 +1,44 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_edit_deny(session: AsyncSession, log: Log): + target_type = constants.MODERATION_EDIT_DENIED + + if not (edit := await service.get_edit(session, log.target_id)): + return + + if not edit.author: + return + + if edit.author_id == edit.moderator_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "description": edit.description, + "username": log.user.username, + "avatar": log.user.avatar, + "edit_id": edit.edit_id, + }, + } + ) + + session.add(moderation) diff --git a/app/sync/moderation/service.py b/app/sync/moderation/service.py index 019bfb96..ef1acf60 100644 --- a/app/sync/moderation/service.py +++ b/app/sync/moderation/service.py @@ -1,24 +1,28 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, desc -from app.models import Moderation -# from datetime import datetime -# from uuid import UUID +from app.models import Moderation, Edit +from datetime import datetime +from uuid import UUID async def get_moderation( session: AsyncSession, - # content_type: str, - # content_id: UUID, - # user_id: UUID, - # threshold: datetime, + user_id: UUID, + log_id: UUID, + target_type: str | None = None, ): - return await session.scalar( - select(Moderation) - # .filter( - # Moderation.content_type == content_type, - # Moderation.content_id == content_id, - # Moderation.user_id == user_id, - # Moderation.created > threshold, - # ) - .order_by(desc(Moderation.created)) + query = select(Moderation).filter( + Moderation.user_id == user_id, + Moderation.log_id == log_id, ) + + if target_type: + query = query.filter( + Moderation.target_type == target_type, + ) + + return await session.scalar(query.order_by(desc(Moderation.created))) + + +async def get_edit(session, content_id): + return await session.scalar(select(Edit).filter(Edit.id == content_id)) From f30fbb258a971cff9210422b49bfb826742a9a8d Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 00:59:21 +0300 Subject: [PATCH 03/12] fix: add types for moderation --- app/constants.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/constants.py b/app/constants.py index c649d9ae..593ab0d4 100644 --- a/app/constants.py +++ b/app/constants.py @@ -331,3 +331,7 @@ COLLECTION_PUBLIC = "public" COLLECTION_UNLISTED = "unlisted" COLLECTION_PRIVATE = "private" + +# Moderation types +MODERATION_EDIT_ACCEPTED = "edit_accepted" +MODERATION_EDIT_DENIED = "edit_denied" From c55866df34827e1e0697c955c81e253309ca4178 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 01:00:04 +0300 Subject: [PATCH 04/12] add some examples to manga --- app/manga/schemas.py | 41 +++++++++++++++++++++-------------------- app/schemas.py | 26 +++++++++++++------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/app/manga/schemas.py b/app/manga/schemas.py index e8a3d3d2..245dac41 100644 --- a/app/manga/schemas.py +++ b/app/manga/schemas.py @@ -1,3 +1,4 @@ +from pydantic import Field from app.schemas import datetime_pd from app.schemas import ( @@ -23,28 +24,28 @@ class MangaInfoResponse(CustomModel, DataTypeMixin): authors: list[ContentAuthorResponse] magazines: list[MagazineResponse] external: list[ExternalResponse] - start_date: datetime_pd | None - end_date: datetime_pd | None + start_date: datetime_pd | None = Field(examples=[786585600]) + end_date: datetime_pd | None = Field(examples=[1008806400]) genres: list[GenreResponse] - title_original: str | None + title_original: str | None = Field(examples=["Monster"]) stats: ReadStatsResponse - synopsis_en: str | None - synopsis_ua: str | None - media_type: str | None - chapters: int | None - title_en: str | None - title_ua: str | None + synopsis_en: str | None = Field(examples=["..."]) + synopsis_ua: str | None = Field(examples=["..."]) + media_type: str | None = Field(examples=["manga"]) + chapters: int | None = Field(examples=[162]) + title_en: str | None = Field(examples=["Monster"]) + title_ua: str | None = Field(examples=["Монстр"]) updated: datetime_pd synonyms: list[str] comments_count: int - has_franchise: bool - translated_ua: bool - volumes: int | None - status: str | None - image: str | None - year: int | None - scored_by: int - score: float - mal_id: int - nsfw: bool - slug: str + has_franchise: bool = Field(examples=[True]) + translated_ua: bool = Field(examples=[True]) + volumes: int | None = Field(examples=[18]) + status: str | None = Field(examples=["finished"]) + image: str | None = Field(examples=["https://cdn.hikka.io/hikka.jpg"]) + year: int | None = Field(examples=[1994]) + scored_by: int = Field(examples=[99368]) + score: float = Field(examples=[9.16]) + mal_id: int = Field(examples=[1]) + nsfw: bool = Field(examples=[False]) + slug: str = Field(examples=["monster-54bb37"]) diff --git a/app/schemas.py b/app/schemas.py index 32cd6e21..df046556 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -382,19 +382,19 @@ class AnimeResponse(CustomModel, DataTypeMixin): class MangaResponse(CustomModel, DataTypeMixin): - title_original: str | None - media_type: str | None - title_ua: str | None - title_en: str | None - chapters: int | None - volumes: int | None - translated_ua: bool - status: str | None - image: str | None - year: int | None - scored_by: int - score: float - slug: str + title_original: str | None = Field(examples=["Monster"]) + media_type: str | None = Field(examples=["manga"]) + title_ua: str | None = Field(examples=["Монстр"]) + title_en: str | None = Field(examples=["Monster"]) + chapters: int | None = Field(examples=[162]) + volumes: int | None = Field(examples=[18]) + translated_ua: bool = Field(True) + status: str | None = Field(examples=["finished"]) + image: str | None = Field(examples=["https://cdn.hikka.io/hikka.jpg"]) + year: int | None = Field(examples=[1994]) + scored_by: int = Field(examples=[99368]) + score: float = Field(examples=[9.16]) + slug: str = Field(examples=["monster-54bb37"]) class NovelResponse(CustomModel, DataTypeMixin): From 7cacbc8f3d48aa8c6a7f01c64cb8a069cd40e840 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 11:33:03 +0300 Subject: [PATCH 05/12] feat(moderation-log): add comment_hide support --- app/constants.py | 1 + app/sync/moderation/__init__.py | 5 +++ app/sync/moderation/generate/__init__.py | 2 + app/sync/moderation/generate/comment_hide.py | 44 ++++++++++++++++++++ app/sync/moderation/service.py | 9 +++- 5 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 app/sync/moderation/generate/comment_hide.py diff --git a/app/constants.py b/app/constants.py index 593ab0d4..79b96a89 100644 --- a/app/constants.py +++ b/app/constants.py @@ -335,3 +335,4 @@ # Moderation types MODERATION_EDIT_ACCEPTED = "edit_accepted" MODERATION_EDIT_DENIED = "edit_denied" +MODERATION_COMMENT_HIDE = "comment_hide" diff --git a/app/sync/moderation/__init__.py b/app/sync/moderation/__init__.py index bc348716..d4ac80c2 100644 --- a/app/sync/moderation/__init__.py +++ b/app/sync/moderation/__init__.py @@ -6,6 +6,7 @@ from app import constants from .generate import ( + generate_comment_hide, generate_edit_accept, generate_edit_deny, ) @@ -31,6 +32,7 @@ async def generate_moderation(session: AsyncSession): [ constants.LOG_EDIT_ACCEPT, constants.LOG_EDIT_DENY, + constants.LOG_COMMENT_HIDE, ] ) ) @@ -47,6 +49,9 @@ async def generate_moderation(session: AsyncSession): if log.log_type == constants.LOG_EDIT_DENY: await generate_edit_deny(session, log) + if log.log_type == constants.LOG_COMMENT_HIDE: + await generate_comment_hide(session, log) + session.add(system_timestamp) await session.commit() diff --git a/app/sync/moderation/generate/__init__.py b/app/sync/moderation/generate/__init__.py index c6a33a3b..caebe8f4 100644 --- a/app/sync/moderation/generate/__init__.py +++ b/app/sync/moderation/generate/__init__.py @@ -1,8 +1,10 @@ from .edit_accept import generate_edit_accept from .edit_deny import generate_edit_deny +from .comment_hide import generate_comment_hide __all__ = [ + "generate_comment_hide", "generate_edit_accept", "generate_edit_deny", ] diff --git a/app/sync/moderation/generate/comment_hide.py b/app/sync/moderation/generate/comment_hide.py new file mode 100644 index 00000000..398b29c2 --- /dev/null +++ b/app/sync/moderation/generate/comment_hide.py @@ -0,0 +1,44 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_comment_hide(session: AsyncSession, log: Log): + target_type = constants.MODERATION_COMMENT_HIDE + + if not (comment := await service.get_comment(session, log.target_id)): + return + + if not comment: + return + + if comment.author_id == comment.hidden_by_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "content_type": comment.content_type, + "content_slug": comment.preview["slug"], + "username": log.user.username, + "avatar": log.user.avatar, + }, + } + ) + + session.add(moderation) diff --git a/app/sync/moderation/service.py b/app/sync/moderation/service.py index ef1acf60..9e888d93 100644 --- a/app/sync/moderation/service.py +++ b/app/sync/moderation/service.py @@ -1,7 +1,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, desc -from app.models import Moderation, Edit -from datetime import datetime +from app.models import Moderation, Edit, Comment from uuid import UUID @@ -26,3 +25,9 @@ async def get_moderation( async def get_edit(session, content_id): return await session.scalar(select(Edit).filter(Edit.id == content_id)) + + +async def get_comment(session, content_id): + return await session.scalar( + select(Comment).filter(Comment.id == content_id) + ) From 4ecd2b4c4c647694b3eb495c9a9bdea91415b269 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 13:45:41 +0300 Subject: [PATCH 06/12] feat(moderation-log): add edit_update logging --- app/constants.py | 3 +- app/sync/moderation/__init__.py | 5 +++ app/sync/moderation/generate/__init__.py | 2 + app/sync/moderation/generate/edit_update.py | 45 +++++++++++++++++++++ 4 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 app/sync/moderation/generate/edit_update.py diff --git a/app/constants.py b/app/constants.py index 79b96a89..b62e3492 100644 --- a/app/constants.py +++ b/app/constants.py @@ -335,4 +335,5 @@ # Moderation types MODERATION_EDIT_ACCEPTED = "edit_accepted" MODERATION_EDIT_DENIED = "edit_denied" -MODERATION_COMMENT_HIDE = "comment_hide" +MODERATION_EDIT_UPDATED = "edit_updated" +MODERATION_COMMENT_HIDE = "comment_hidden" diff --git a/app/sync/moderation/__init__.py b/app/sync/moderation/__init__.py index d4ac80c2..ddd59b15 100644 --- a/app/sync/moderation/__init__.py +++ b/app/sync/moderation/__init__.py @@ -8,6 +8,7 @@ from .generate import ( generate_comment_hide, generate_edit_accept, + generate_edit_update, generate_edit_deny, ) @@ -32,6 +33,7 @@ async def generate_moderation(session: AsyncSession): [ constants.LOG_EDIT_ACCEPT, constants.LOG_EDIT_DENY, + constants.LOG_EDIT_UPDATE, constants.LOG_COMMENT_HIDE, ] ) @@ -49,6 +51,9 @@ async def generate_moderation(session: AsyncSession): if log.log_type == constants.LOG_EDIT_DENY: await generate_edit_deny(session, log) + if log.log_type == constants.LOG_EDIT_UPDATE: + await generate_edit_update(session, log) + if log.log_type == constants.LOG_COMMENT_HIDE: await generate_comment_hide(session, log) diff --git a/app/sync/moderation/generate/__init__.py b/app/sync/moderation/generate/__init__.py index caebe8f4..75264b7a 100644 --- a/app/sync/moderation/generate/__init__.py +++ b/app/sync/moderation/generate/__init__.py @@ -1,10 +1,12 @@ from .edit_accept import generate_edit_accept from .edit_deny import generate_edit_deny +from .edit_update import generate_edit_update from .comment_hide import generate_comment_hide __all__ = [ "generate_comment_hide", "generate_edit_accept", + "generate_edit_update", "generate_edit_deny", ] diff --git a/app/sync/moderation/generate/edit_update.py b/app/sync/moderation/generate/edit_update.py new file mode 100644 index 00000000..022c8e01 --- /dev/null +++ b/app/sync/moderation/generate/edit_update.py @@ -0,0 +1,45 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_edit_update(session: AsyncSession, log: Log): + target_type = constants.MODERATION_EDIT_UPDATED + + if not (edit := await service.get_edit(session, log.target_id)): + return + + if not edit.author: + return + + if edit.author_id == log.user_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "updated_edit": log.data["updated_edit"], + "old_edit": log.data["old_edit"], + "username": log.user.username, + "avatar": log.user.avatar, + "edit_id": edit.edit_id, + }, + } + ) + + session.add(moderation) From b098c8c7a4e904c22a079dab12998632cc49c564 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 22:31:28 +0300 Subject: [PATCH 07/12] feat(moderation-log): add collection delete & update logging --- app/constants.py | 2 + app/sync/moderation/__init__.py | 10 +++++ app/sync/moderation/generate/__init__.py | 6 +++ .../moderation/generate/collection_delete.py | 43 ++++++++++++++++++ .../moderation/generate/collection_update.py | 45 +++++++++++++++++++ app/sync/moderation/service.py | 8 +++- 6 files changed, 113 insertions(+), 1 deletion(-) create mode 100644 app/sync/moderation/generate/collection_delete.py create mode 100644 app/sync/moderation/generate/collection_update.py diff --git a/app/constants.py b/app/constants.py index b62e3492..9e54d182 100644 --- a/app/constants.py +++ b/app/constants.py @@ -337,3 +337,5 @@ MODERATION_EDIT_DENIED = "edit_denied" MODERATION_EDIT_UPDATED = "edit_updated" MODERATION_COMMENT_HIDE = "comment_hidden" +MODERATION_COLLECTION_DELETE = "collection_deleted" +MODERATION_COLLECTION_UPDATE = "collection_updated" diff --git a/app/sync/moderation/__init__.py b/app/sync/moderation/__init__.py index ddd59b15..a2d73922 100644 --- a/app/sync/moderation/__init__.py +++ b/app/sync/moderation/__init__.py @@ -6,6 +6,8 @@ from app import constants from .generate import ( + generate_collection_delete, + generate_collection_update, generate_comment_hide, generate_edit_accept, generate_edit_update, @@ -35,6 +37,8 @@ async def generate_moderation(session: AsyncSession): constants.LOG_EDIT_DENY, constants.LOG_EDIT_UPDATE, constants.LOG_COMMENT_HIDE, + constants.LOG_COLLECTION_DELETE, + constants.LOG_COLLECTION_UPDATE, ] ) ) @@ -57,6 +61,12 @@ async def generate_moderation(session: AsyncSession): if log.log_type == constants.LOG_COMMENT_HIDE: await generate_comment_hide(session, log) + if log.log_type == constants.LOG_COLLECTION_DELETE: + await generate_collection_delete(session, log) + + if log.log_type == constants.LOG_COLLECTION_UPDATE: + await generate_collection_update(session, log) + session.add(system_timestamp) await session.commit() diff --git a/app/sync/moderation/generate/__init__.py b/app/sync/moderation/generate/__init__.py index 75264b7a..f57da93f 100644 --- a/app/sync/moderation/generate/__init__.py +++ b/app/sync/moderation/generate/__init__.py @@ -1,10 +1,16 @@ from .edit_accept import generate_edit_accept from .edit_deny import generate_edit_deny from .edit_update import generate_edit_update + from .comment_hide import generate_comment_hide +from .collection_delete import generate_collection_delete +from .collection_update import generate_collection_update + __all__ = [ + "generate_collection_delete", + "generate_collection_update", "generate_comment_hide", "generate_edit_accept", "generate_edit_update", diff --git a/app/sync/moderation/generate/collection_delete.py b/app/sync/moderation/generate/collection_delete.py new file mode 100644 index 00000000..48e0b3d5 --- /dev/null +++ b/app/sync/moderation/generate/collection_delete.py @@ -0,0 +1,43 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_collection_delete(session: AsyncSession, log: Log): + target_type = constants.MODERATION_COLLECTION_DELETE + + if not (collection := await service.get_collection(session, log.target_id)): + return + + if not collection.author: + return + + if collection.author_id == log.user_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "username": log.user.username, + "avatar": log.user.avatar, + "collection_id": str(collection.id), + }, + } + ) + + session.add(moderation) diff --git a/app/sync/moderation/generate/collection_update.py b/app/sync/moderation/generate/collection_update.py new file mode 100644 index 00000000..703ad3e2 --- /dev/null +++ b/app/sync/moderation/generate/collection_update.py @@ -0,0 +1,45 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.models import Moderation, Log +from app import constants +from .. import service + + +async def generate_collection_update(session: AsyncSession, log: Log): + target_type = constants.MODERATION_COLLECTION_UPDATE + + if not (collection := await service.get_collection(session, log.target_id)): + return + + if not collection.author: + return + + if collection.author_id == log.user_id: + return + + if await service.get_moderation( + session, + log.user_id, + log.id, + target_type, + ): + return + + await session.refresh(log, attribute_names=["user"]) + + moderation = Moderation( + **{ + "target_type": target_type, + "user_id": log.user_id, + "created": log.created, + "log_id": log.id, + "data": { + "updated_collection": log.data["updated_collection"], + "old_collection": log.data["old_collection"], + "username": log.user.username, + "avatar": log.user.avatar, + "collection_id": str(collection.id), + }, + } + ) + + session.add(moderation) diff --git a/app/sync/moderation/service.py b/app/sync/moderation/service.py index 9e888d93..8502cacc 100644 --- a/app/sync/moderation/service.py +++ b/app/sync/moderation/service.py @@ -1,6 +1,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy import select, desc -from app.models import Moderation, Edit, Comment +from app.models import Moderation, Edit, Comment, Collection from uuid import UUID @@ -31,3 +31,9 @@ async def get_comment(session, content_id): return await session.scalar( select(Comment).filter(Comment.id == content_id) ) + + +async def get_collection(session, content_id): + return await session.scalar( + select(Collection).filter(Collection.id == content_id) + ) From 0d33ea6d3a5299e9b1dda1d69740bd2e92fc539f Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 22:33:35 +0300 Subject: [PATCH 08/12] add comment_path to data of comment log --- app/sync/moderation/generate/comment_hide.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/sync/moderation/generate/comment_hide.py b/app/sync/moderation/generate/comment_hide.py index 398b29c2..38664866 100644 --- a/app/sync/moderation/generate/comment_hide.py +++ b/app/sync/moderation/generate/comment_hide.py @@ -35,6 +35,7 @@ async def generate_comment_hide(session: AsyncSession, log: Log): "data": { "content_type": comment.content_type, "content_slug": comment.preview["slug"], + "comment_path": comment.path, "username": log.user.username, "avatar": log.user.avatar, }, From de52f7a26e025b5140cd0af9124b180577b9ec7e Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Sun, 7 Jul 2024 23:59:21 +0300 Subject: [PATCH 09/12] feat(moderation-log): add filter & sort --- app/constants.py | 6 +- app/moderation/dependencies.py | 21 ++++++ app/moderation/router.py | 17 ++--- app/moderation/schemas.py | 39 ++++++++++ app/moderation/service.py | 72 ++++++++++++------- .../moderation/generate/collection_delete.py | 2 +- .../moderation/generate/collection_update.py | 2 +- app/sync/moderation/generate/comment_hide.py | 2 +- 8 files changed, 118 insertions(+), 43 deletions(-) create mode 100644 app/moderation/dependencies.py diff --git a/app/constants.py b/app/constants.py index 9e54d182..79ae6224 100644 --- a/app/constants.py +++ b/app/constants.py @@ -336,6 +336,6 @@ MODERATION_EDIT_ACCEPTED = "edit_accepted" MODERATION_EDIT_DENIED = "edit_denied" MODERATION_EDIT_UPDATED = "edit_updated" -MODERATION_COMMENT_HIDE = "comment_hidden" -MODERATION_COLLECTION_DELETE = "collection_deleted" -MODERATION_COLLECTION_UPDATE = "collection_updated" +MODERATION_COMMENT_HIDDEN = "comment_hidden" +MODERATION_COLLECTION_DELETED = "collection_deleted" +MODERATION_COLLECTION_UPDATED = "collection_updated" diff --git a/app/moderation/dependencies.py b/app/moderation/dependencies.py new file mode 100644 index 00000000..b00e099e --- /dev/null +++ b/app/moderation/dependencies.py @@ -0,0 +1,21 @@ +from sqlalchemy.ext.asyncio import AsyncSession +from app.database import get_session +from app.errors import Abort +from fastapi import Depends + +from .schemas import ModerationSearchArgs + +from app.service import ( + get_user_by_username, +) + + +async def validate_moderation_search_args( + args: ModerationSearchArgs, + session: AsyncSession = Depends(get_session), +): + if args.author: + if not await get_user_by_username(session, args.author): + raise Abort("edit", "author-not-found") + + return args diff --git a/app/moderation/router.py b/app/moderation/router.py index 0d88f451..4c452ebf 100644 --- a/app/moderation/router.py +++ b/app/moderation/router.py @@ -1,6 +1,5 @@ from sqlalchemy.ext.asyncio import AsyncSession from fastapi import APIRouter, Depends -from app import constants from app.database import get_session from app.models import User from . import service @@ -8,8 +7,11 @@ from .schemas import ( ModerationPaginationResponse, + ModerationSearchArgs, ) +from .dependencies import validate_moderation_search_args + from app.utils import ( pagination_dict, pagination, @@ -22,28 +24,23 @@ ) -class ModerationError(Exception): - def __init__(self, content: dict, status_code: int): - self.status_code = status_code - self.content = content - - router = APIRouter(prefix="/moderation", tags=["Moderation"]) -@router.get( +@router.post( "/log", response_model=ModerationPaginationResponse, summary="Moderation log", ) async def moderation_log( + args: ModerationSearchArgs = Depends(validate_moderation_search_args), session: AsyncSession = Depends(get_session), page: int = Depends(get_page), size: int = Depends(get_size), ): limit, offset = pagination(page, size) - total = await service.get_moderation_count(session) - moderation = await service.get_moderation(session, limit, offset) + total = await service.get_moderation_count(session, args) + moderation = await service.get_moderation(session, args, limit, offset) return { "pagination": pagination_dict(total, page, limit), diff --git a/app/moderation/schemas.py b/app/moderation/schemas.py index ce91bb0e..b9c6d285 100644 --- a/app/moderation/schemas.py +++ b/app/moderation/schemas.py @@ -1,3 +1,6 @@ +from enum import Enum +from pydantic import field_validator +from app import constants from app.schemas import PaginationResponse, CustomModel from app.schemas import datetime_pd @@ -13,3 +16,39 @@ class ModerationResponse(CustomModel): class ModerationPaginationResponse(CustomModel): pagination: PaginationResponse list: list[ModerationResponse] + + +# Enums +class ModerationTypeEnum(str, Enum): + edit_accepted = constants.MODERATION_EDIT_ACCEPTED + edit_denied = constants.MODERATION_EDIT_DENIED + edit_updated = constants.MODERATION_EDIT_UPDATED + comment_hidden = constants.MODERATION_COMMENT_HIDDEN + collection_deleted = constants.MODERATION_COLLECTION_DELETED + collection_updated = constants.MODERATION_COLLECTION_UPDATED + + +# Args +class ModerationSearchArgs(CustomModel): + sort: str = "created:desc" + target_type: ModerationTypeEnum | None = None + author: str | None = None + + @field_validator("sort") + def validate_sort(cls, sort): + valid_orders = ["asc", "desc"] + valid_fields = [ + "created", + ] + + parts = sort.split(":") + + if len(parts) != 2: + raise ValueError(f"Invalid sort format: {sort}") + + field, order = parts + + if field not in valid_fields or order not in valid_orders: + raise ValueError(f"Invalid sort value: {sort}") + + return sort diff --git a/app/moderation/service.py b/app/moderation/service.py index 416eb0be..4156c5c3 100644 --- a/app/moderation/service.py +++ b/app/moderation/service.py @@ -1,46 +1,64 @@ from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy import select, desc, func +from sqlalchemy.sql.selectable import Select +from sqlalchemy import asc, select, desc, func from app.models import ( Moderation, User, ) +from app.service import get_user_by_username +from .schemas import ModerationSearchArgs -async def get_moderation_count(session: AsyncSession) -> int: - return await session.scalar(select(func.count(Moderation.id))) +async def moderation_search_filter( + session: AsyncSession, + args: ModerationSearchArgs, + query: Select, +): + if args.author: + author = await get_user_by_username(session, args.author) + query = query.filter(Moderation.user == author) + if args.target_type: + query = query.filter(Moderation.target_type == args.target_type) + + return query -async def get_moderation( - session: AsyncSession, limit: int, offset: int -) -> User: - """Get moderation log""" - return await session.scalars( - select(Moderation) - .order_by(desc(Moderation.created)) - .limit(limit) - .offset(offset) +def build_moderation_order_by(sort: str): + order_mapping = { + "created": Moderation.created, + } + + field, order = sort.split(":") + + order_by = ( + desc(order_mapping[field]) + if order == "desc" + else asc(order_mapping[field]) ) + return order_by + -async def get_user_moderation_count( - session: AsyncSession, user_id: User.id +async def get_moderation_count( + session: AsyncSession, args: ModerationSearchArgs ) -> int: - return await session.scalar( - select(func.count(Moderation.id)).filter(Moderation.user_id == user_id) + query = await moderation_search_filter( + session, args, select(func.count(Moderation.id)) ) + return await session.scalar(query) -async def get_user_moderation( - session: AsyncSession, user_id: User.id, limit: int, offset: int -) -> User: - """Get user moderation log""" - return await session.scalars( - select(Moderation) - .filter(Moderation.user_id == user_id) - .order_by(desc(Moderation.created)) - .limit(limit) - .offset(offset) - ) +async def get_moderation( + session: AsyncSession, args: ModerationSearchArgs, limit: int, offset: int +) -> list[Moderation]: + """Get moderation log""" + + query = await moderation_search_filter(session, args, select(Moderation)) + + query = query.order_by(build_moderation_order_by(args.sort)) + query = query.limit(limit).offset(offset) + + return await session.scalars(query) diff --git a/app/sync/moderation/generate/collection_delete.py b/app/sync/moderation/generate/collection_delete.py index 48e0b3d5..527ddd99 100644 --- a/app/sync/moderation/generate/collection_delete.py +++ b/app/sync/moderation/generate/collection_delete.py @@ -5,7 +5,7 @@ async def generate_collection_delete(session: AsyncSession, log: Log): - target_type = constants.MODERATION_COLLECTION_DELETE + target_type = constants.MODERATION_COLLECTION_DELETED if not (collection := await service.get_collection(session, log.target_id)): return diff --git a/app/sync/moderation/generate/collection_update.py b/app/sync/moderation/generate/collection_update.py index 703ad3e2..a392ab09 100644 --- a/app/sync/moderation/generate/collection_update.py +++ b/app/sync/moderation/generate/collection_update.py @@ -5,7 +5,7 @@ async def generate_collection_update(session: AsyncSession, log: Log): - target_type = constants.MODERATION_COLLECTION_UPDATE + target_type = constants.MODERATION_COLLECTION_UPDATED if not (collection := await service.get_collection(session, log.target_id)): return diff --git a/app/sync/moderation/generate/comment_hide.py b/app/sync/moderation/generate/comment_hide.py index 38664866..b117c72b 100644 --- a/app/sync/moderation/generate/comment_hide.py +++ b/app/sync/moderation/generate/comment_hide.py @@ -5,7 +5,7 @@ async def generate_comment_hide(session: AsyncSession, log: Log): - target_type = constants.MODERATION_COMMENT_HIDE + target_type = constants.MODERATION_COMMENT_HIDDEN if not (comment := await service.get_comment(session, log.target_id)): return From dba2834cfaee9d534dca5e5f1746914500c71669 Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Mon, 8 Jul 2024 00:13:57 +0300 Subject: [PATCH 10/12] add moderation perms check for moderation_log --- app/moderation/router.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/moderation/router.py b/app/moderation/router.py index 4c452ebf..56ee9d3e 100644 --- a/app/moderation/router.py +++ b/app/moderation/router.py @@ -1,5 +1,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from fastapi import APIRouter, Depends +from app import constants from app.database import get_session from app.models import User from . import service @@ -18,6 +19,7 @@ ) from app.dependencies import ( + auth_required, get_page, get_size, get_user, @@ -35,6 +37,10 @@ async def moderation_log( args: ModerationSearchArgs = Depends(validate_moderation_search_args), session: AsyncSession = Depends(get_session), + # TODO: replace with role check + user: User = Depends( + auth_required(permissions=[constants.PERMISSION_EDIT_AUTO]) + ), page: int = Depends(get_page), size: int = Depends(get_size), ): From a6eccc96db43b94ac34e9d545ec5b9f7ced44b4f Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Mon, 8 Jul 2024 01:26:31 +0300 Subject: [PATCH 11/12] remove unused import in app.moderation.service --- app/moderation/service.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/moderation/service.py b/app/moderation/service.py index 4156c5c3..ba1786ae 100644 --- a/app/moderation/service.py +++ b/app/moderation/service.py @@ -2,10 +2,7 @@ from sqlalchemy.sql.selectable import Select from sqlalchemy import asc, select, desc, func -from app.models import ( - Moderation, - User, -) +from app.models import Moderation from app.service import get_user_by_username from .schemas import ModerationSearchArgs From a134b769b3b1007cb39d952211de063d20df14db Mon Sep 17 00:00:00 2001 From: Andrii Blacksmith Date: Mon, 8 Jul 2024 12:58:34 +0300 Subject: [PATCH 12/12] make better validation of role in moderation_log --- app/errors.py | 3 +++ app/moderation/dependencies.py | 12 ++++++++++++ app/moderation/router.py | 33 +++++---------------------------- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/app/errors.py b/app/errors.py index 9c8c27ba..c9a6862c 100644 --- a/app/errors.py +++ b/app/errors.py @@ -184,6 +184,9 @@ class ErrorResponse(CustomModel): "system": { "bad-backup-token": ["Bad backup token", 401], }, + "moderation-log": { + "no-access": ["You do not have permission to access", 400] + }, } diff --git a/app/moderation/dependencies.py b/app/moderation/dependencies.py index b00e099e..b758f7e6 100644 --- a/app/moderation/dependencies.py +++ b/app/moderation/dependencies.py @@ -1,8 +1,11 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.database import get_session +from app.dependencies import auth_required from app.errors import Abort from fastapi import Depends +from app.models.user.user import User + from .schemas import ModerationSearchArgs from app.service import ( @@ -19,3 +22,12 @@ async def validate_moderation_search_args( raise Abort("edit", "author-not-found") return args + + +async def validate_moderation_role( + author: User = Depends(auth_required(optional=False)), +): + if author.role not in ["admin", "moderator"]: + raise Abort("moderation-log", "no-access") + + return author diff --git a/app/moderation/router.py b/app/moderation/router.py index 56ee9d3e..3cd4e9fc 100644 --- a/app/moderation/router.py +++ b/app/moderation/router.py @@ -11,7 +11,10 @@ ModerationSearchArgs, ) -from .dependencies import validate_moderation_search_args +from .dependencies import ( + validate_moderation_search_args, + validate_moderation_role, +) from app.utils import ( pagination_dict, @@ -37,10 +40,7 @@ async def moderation_log( args: ModerationSearchArgs = Depends(validate_moderation_search_args), session: AsyncSession = Depends(get_session), - # TODO: replace with role check - user: User = Depends( - auth_required(permissions=[constants.PERMISSION_EDIT_AUTO]) - ), + user: User = Depends(validate_moderation_role), page: int = Depends(get_page), size: int = Depends(get_size), ): @@ -52,26 +52,3 @@ async def moderation_log( "pagination": pagination_dict(total, page, limit), "list": moderation.all(), } - - -@router.get( - "/{username}/log", - response_model=ModerationPaginationResponse, - summary="User moderation log", -) -async def moderation_user_log( - session: AsyncSession = Depends(get_session), - user: User = Depends(get_user), - page: int = Depends(get_page), - size: int = Depends(get_size), -): - limit, offset = pagination(page, size) - total = await service.get_user_moderation_count(session, user.id) - moderation = await service.get_user_moderation( - session, user.id, limit, offset - ) - - return { - "pagination": pagination_dict(total, page, limit), - "list": moderation.all(), - }