Reorganize Pydantic models and types used in handlers (#17279)

Spawning from #17187 (comment) around wanting to put `SlidingSyncBody` (parse the request in the rest layer), `SlidingSyncConfig` (from the rest layer, pass to the handler), `SlidingSyncResponse` (pass the response from the handler back to the rest layer to respond) somewhere that doesn't contaminate the imports and cause circular import issues.

 - Moved Pydantic parsing models to `synapse/types/rest`
 - Moved handler types to `synapse/types/handlers`

changelog.d/17279.misc

@@ -0,0 +1 @@
+Re-organize Pydantic models and types used in handlers.

synapse/events/validator.py

@@ -47,9 +47,9 @@
     validate_canonicaljson,
 )
 from synapse.http.servlet import validate_json_object
-from synapse.rest.models import RequestBodyModel
 from synapse.storage.controllers.state import server_acl_evaluator_from_event
 from synapse.types import EventID, JsonDict, RoomID, StrCollection, UserID
+from synapse.types.rest import RequestBodyModel
 
 
 class EventValidator:

synapse/handlers/pagination.py

@@ -37,11 +37,10 @@
     JsonMapping,
     Requester,
     ScheduledTask,
-    ShutdownRoomParams,
-    ShutdownRoomResponse,
     StreamKeyType,
     TaskStatus,
 )
+from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse
 from synapse.types.state import StateFilter
 from synapse.util.async_helpers import ReadWriteLock
 from synapse.visibility import filter_events_for_client

synapse/handlers/room.py

@@ -80,15 +80,14 @@
     RoomAlias,
     RoomID,
     RoomStreamToken,
-    ShutdownRoomParams,
-    ShutdownRoomResponse,
     StateMap,
     StrCollection,
     StreamKeyType,
     StreamToken,
     UserID,
     create_requester,
 )
+from synapse.types.handlers import ShutdownRoomParams, ShutdownRoomResponse
 from synapse.types.state import StateFilter
 from synapse.util import stringutils
 from synapse.util.caches.response_cache import ResponseCache

synapse/handlers/sliding_sync.py

@@ -18,23 +18,14 @@
 #
 #
 import logging
-from enum import Enum
-from typing import TYPE_CHECKING, AbstractSet, Dict, Final, List, Optional, Tuple
+from typing import TYPE_CHECKING, AbstractSet, Dict, List, Optional
 
-import attr
 from immutabledict import immutabledict
 
-from synapse._pydantic_compat import HAS_PYDANTIC_V2
-
-if TYPE_CHECKING or HAS_PYDANTIC_V2:
-    from pydantic.v1 import Extra
-else:
-    from pydantic import Extra
-
 from synapse.api.constants import Membership
 from synapse.events import EventBase
-from synapse.rest.client.models import SlidingSyncBody
-from synapse.types import JsonMapping, Requester, RoomStreamToken, StreamToken, UserID
+from synapse.types import Requester, RoomStreamToken, StreamToken, UserID
+from synapse.types.handlers import OperationType, SlidingSyncConfig, SlidingSyncResult
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer
@@ -62,166 +53,6 @@ def filter_membership_for_sync(*, membership: str, user_id: str, sender: str) ->
     return membership != Membership.LEAVE or sender != user_id
 
 
-class SlidingSyncConfig(SlidingSyncBody):
-    """
-    Inherit from `SlidingSyncBody` since we need all of the same fields and add a few
-    extra fields that we need in the handler
-    """
-
-    user: UserID
-    device_id: Optional[str]
-
-    # Pydantic config
-    class Config:
-        # By default, ignore fields that we don't recognise.
-        extra = Extra.ignore
-        # By default, don't allow fields to be reassigned after parsing.
-        allow_mutation = False
-        # Allow custom types like `UserID` to be used in the model
-        arbitrary_types_allowed = True
-
-
-class OperationType(Enum):
-    """
-    Represents the operation types in a Sliding Sync window.
-
-    Attributes:
-        SYNC: Sets a range of entries. Clients SHOULD discard what they previous knew about
-            entries in this range.
-        INSERT: Sets a single entry. If the position is not empty then clients MUST move
-            entries to the left or the right depending on where the closest empty space is.
-        DELETE: Remove a single entry. Often comes before an INSERT to allow entries to move
-            places.
-        INVALIDATE: Remove a range of entries. Clients MAY persist the invalidated range for
-            offline support, but they should be treated as empty when additional operations
-            which concern indexes in the range arrive from the server.
-    """
-
-    SYNC: Final = "SYNC"
-    INSERT: Final = "INSERT"
-    DELETE: Final = "DELETE"
-    INVALIDATE: Final = "INVALIDATE"
-
-
-@attr.s(slots=True, frozen=True, auto_attribs=True)
-class SlidingSyncResult:
-    """
-    The Sliding Sync result to be serialized to JSON for a response.
-
-    Attributes:
-        next_pos: The next position token in the sliding window to request (next_batch).
-        lists: Sliding window API. A map of list key to list results.
-        rooms: Room subscription API. A map of room ID to room subscription to room results.
-        extensions: Extensions API. A map of extension key to extension results.
-    """
-
-    @attr.s(slots=True, frozen=True, auto_attribs=True)
-    class RoomResult:
-        """
-        Attributes:
-            name: Room name or calculated room name.
-            avatar: Room avatar
-            heroes: List of stripped membership events (containing `user_id` and optionally
-                `avatar_url` and `displayname`) for the users used to calculate the room name.
-            initial: Flag which is set when this is the first time the server is sending this
-                data on this connection. Clients can use this flag to replace or update
-                their local state. When there is an update, servers MUST omit this flag
-                entirely and NOT send "initial":false as this is wasteful on bandwidth. The
-                absence of this flag means 'false'.
-            required_state: The current state of the room
-            timeline: Latest events in the room. The last event is the most recent
-            is_dm: Flag to specify whether the room is a direct-message room (most likely
-                between two people).
-            invite_state: Stripped state events. Same as `rooms.invite.$room_id.invite_state`
-                in sync v2, absent on joined/left rooms
-            prev_batch: A token that can be passed as a start parameter to the
-                `/rooms/<room_id>/messages` API to retrieve earlier messages.
-            limited: True if their are more events than fit between the given position and now.
-                Sync again to get more.
-            joined_count: The number of users with membership of join, including the client's
-                own user ID. (same as sync `v2 m.joined_member_count`)
-            invited_count: The number of users with membership of invite. (same as sync v2
-                `m.invited_member_count`)
-            notification_count: The total number of unread notifications for this room. (same
-                as sync v2)
-            highlight_count: The number of unread notifications for this room with the highlight
-                flag set. (same as sync v2)
-            num_live: The number of timeline events which have just occurred and are not historical.
-                The last N events are 'live' and should be treated as such. This is mostly
-                useful to determine whether a given @mention event should make a noise or not.
-                Clients cannot rely solely on the absence of `initial: true` to determine live
-                events because if a room not in the sliding window bumps into the window because
-                of an @mention it will have `initial: true` yet contain a single live event
-                (with potentially other old events in the timeline).
-        """
-
-        name: str
-        avatar: Optional[str]
-        heroes: Optional[List[EventBase]]
-        initial: bool
-        required_state: List[EventBase]
-        timeline: List[EventBase]
-        is_dm: bool
-        invite_state: List[EventBase]
-        prev_batch: StreamToken
-        limited: bool
-        joined_count: int
-        invited_count: int
-        notification_count: int
-        highlight_count: int
-        num_live: int
-
-    @attr.s(slots=True, frozen=True, auto_attribs=True)
-    class SlidingWindowList:
-        """
-        Attributes:
-            count: The total number of entries in the list. Always present if this list
-                is.
-            ops: The sliding list operations to perform.
-        """
-
-        @attr.s(slots=True, frozen=True, auto_attribs=True)
-        class Operation:
-            """
-            Attributes:
-                op: The operation type to perform.
-                range: Which index positions are affected by this operation. These are
-                    both inclusive.
-                room_ids: Which room IDs are affected by this operation. These IDs match
-                    up to the positions in the `range`, so the last room ID in this list
-                    matches the 9th index. The room data is held in a separate object.
-            """
-
-            op: OperationType
-            range: Tuple[int, int]
-            room_ids: List[str]
-
-        count: int
-        ops: List[Operation]
-
-    next_pos: StreamToken
-    lists: Dict[str, SlidingWindowList]
-    rooms: Dict[str, RoomResult]
-    extensions: JsonMapping
-
-    def __bool__(self) -> bool:
-        """Make the result appear empty if there are no updates. This is used
-        to tell if the notifier needs to wait for more events when polling for
-        events.
-        """
-        return bool(self.lists or self.rooms or self.extensions)
-
-    @staticmethod
-    def empty(next_pos: StreamToken) -> "SlidingSyncResult":
-        "Return a new empty result"
-        return SlidingSyncResult(
-            next_pos=next_pos,
-            lists={},
-            rooms={},
-            extensions={},
-        )
-
-
 class SlidingSyncHandler:
     def __init__(self, hs: "HomeServer"):
         self.clock = hs.get_clock()

synapse/rest/client/account.py

@@ -56,14 +56,14 @@
 from synapse.http.site import SynapseRequest
 from synapse.metrics import threepid_send_requests
 from synapse.push.mailer import Mailer
-from synapse.rest.client.models import (
+from synapse.types import JsonDict
+from synapse.types.rest import RequestBodyModel
+from synapse.types.rest.client import (
     AuthenticationData,
     ClientSecretStr,
     EmailRequestTokenBody,
     MsisdnRequestTokenBody,
 )
-from synapse.rest.models import RequestBodyModel
-from synapse.types import JsonDict
 from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.stringutils import assert_valid_client_secret, random_string
 from synapse.util.threepids import check_3pid_allowed, validate_email

synapse/rest/client/devices.py

@@ -42,9 +42,9 @@
 )
 from synapse.http.site import SynapseRequest
 from synapse.rest.client._base import client_patterns, interactive_auth_handler
-from synapse.rest.client.models import AuthenticationData
-from synapse.rest.models import RequestBodyModel
 from synapse.types import JsonDict
+from synapse.types.rest import RequestBodyModel
+from synapse.types.rest.client import AuthenticationData
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer

synapse/rest/client/directory.py

@@ -41,8 +41,8 @@
 )
 from synapse.http.site import SynapseRequest
 from synapse.rest.client._base import client_patterns
-from synapse.rest.models import RequestBodyModel
 from synapse.types import JsonDict, RoomAlias
+from synapse.types.rest import RequestBodyModel
 
 if TYPE_CHECKING:
     from synapse.server import HomeServer

synapse/rest/client/sync.py

@@ -53,8 +53,8 @@
 )
 from synapse.http.site import SynapseRequest
 from synapse.logging.opentracing import trace_with_opname
-from synapse.rest.client.models import SlidingSyncBody
 from synapse.types import JsonDict, Requester, StreamToken
+from synapse.types.rest.client import SlidingSyncBody
 from synapse.util import json_decoder
 from synapse.util.caches.lrucache import LruCache
 

synapse/rest/key/v2/remote_key_resource.py

@@ -41,9 +41,9 @@
     parse_and_validate_json_object_from_request,
     parse_integer,
 )
-from synapse.rest.models import RequestBodyModel
 from synapse.storage.keys import FetchKeyResultForRemote
 from synapse.types import JsonDict
+from synapse.types.rest import RequestBodyModel
 from synapse.util import json_decoder
 from synapse.util.async_helpers import yieldable_gather_results
 

synapse/types/__init__.py

@@ -1279,60 +1279,3 @@ class ScheduledTask:
     result: Optional[JsonMapping]
     # Optional error that should be assigned a value when the status is FAILED
     error: Optional[str]
-
-
-class ShutdownRoomParams(TypedDict):
-    """
-    Attributes:
-        requester_user_id:
-            User who requested the action. Will be recorded as putting the room on the
-            blocking list.
-        new_room_user_id:
-            If set, a new room will be created with this user ID
-            as the creator and admin, and all users in the old room will be
-            moved into that room. If not set, no new room will be created
-            and the users will just be removed from the old room.
-        new_room_name:
-            A string representing the name of the room that new users will
-            be invited to. Defaults to `Content Violation Notification`
-        message:
-            A string containing the first message that will be sent as
-            `new_room_user_id` in the new room. Ideally this will clearly
-            convey why the original room was shut down.
-            Defaults to `Sharing illegal content on this server is not
-            permitted and rooms in violation will be blocked.`
-        block:
-            If set to `true`, this room will be added to a blocking list,
-            preventing future attempts to join the room. Defaults to `false`.
-        purge:
-            If set to `true`, purge the given room from the database.
-        force_purge:
-            If set to `true`, the room will be purged from database
-            even if there are still users joined to the room.
-    """
-
-    requester_user_id: Optional[str]
-    new_room_user_id: Optional[str]
-    new_room_name: Optional[str]
-    message: Optional[str]
-    block: bool
-    purge: bool
-    force_purge: bool
-
-
-class ShutdownRoomResponse(TypedDict):
-    """
-    Attributes:
-        kicked_users: An array of users (`user_id`) that were kicked.
-        failed_to_kick_users:
-            An array of users (`user_id`) that that were not kicked.
-        local_aliases:
-            An array of strings representing the local aliases that were
-            migrated from the old room to the new.
-        new_room_id: A string representing the room ID of the new room.
-    """
-
-    kicked_users: List[str]
-    failed_to_kick_users: List[str]
-    local_aliases: List[str]
-    new_room_id: Optional[str]