diff --git a/.env.example b/.env.example index 44c4f72..a3cf50c 100644 --- a/.env.example +++ b/.env.example @@ -14,11 +14,5 @@ INTERNAL_API_TOKEN="" # used for authenticating internal API requests (gives ac CAPTCHA_SITEKEY= CAPTCHA_SECRET= -GRPC_AUTH_ADDRESS="0.0.0.0:5000" -GRPC_AUTH_TOKEN= - -GRPC_UPLOADS_ADDRESS= -GRPC_UPLOADS_TOKEN= - CHAT_EMOJIS_LIMIT=250 CHAT_STICKERS_LIMIT=50 \ No newline at end of file diff --git a/database.py b/database.py index c1d8e14..393d5fc 100644 --- a/database.py +++ b/database.py @@ -6,7 +6,7 @@ from utils import log -CURRENT_DB_VERSION = 9 +CURRENT_DB_VERSION = 10 # Create Redis connection log("Connecting to Redis...") @@ -174,7 +174,21 @@ # Create post reactions index try: - db.post_reactions.create_index([("_id.post_id", 1), ("_id.emoji", 1)]) + db.post_reactions.create_index([("_id.post_id", pymongo.ASCENDING), ("_id.emoji", pymongo.ASCENDING)]) +except: pass + +# Create files indexes +try: + db.files.create_index([("hash", pymongo.ASCENDING)], name="hash") +except: pass +try: + db.files.create_index([("uploaded_by", pymongo.ASCENDING)], name="uploaded_by") +except: pass +try: + db.files.create_index([ + ("claimed", pymongo.ASCENDING), + ("uploaded_by", pymongo.ASCENDING) + ], name="unclaimed", partialFilterExpression={"claimed": False}) except: pass @@ -306,6 +320,16 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: "mfa_recovery_code": user["mfa_recovery_code"][:10] }}) + # Post attachments + log("[Migrator] Converting post attachments") + updates = [] + for post in db.posts.find({"attachments.id": {"$exists": True}}): + updates.append(pymongo.UpdateOne( + {"_id": post["_id"]}, + {"$set": {"attachments": [attachment["id"] for attachment in post["attachments"]]}} + )) + db.posts.bulk_write(updates) + db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/grpc_auth/auth_service_pb2.py b/grpc_auth/auth_service_pb2.py deleted file mode 100644 index 766e5c0..0000000 --- a/grpc_auth/auth_service_pb2.py +++ /dev/null @@ -1,31 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: auth_service.proto -# Protobuf Python Version: 5.26.1 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x12\x61uth_service.proto\x12\x04\x61uth\"\x1e\n\rCheckTokenReq\x12\r\n\x05token\x18\x01 \x01(\t\"0\n\x0e\x43heckTokenResp\x12\r\n\x05valid\x18\x01 \x01(\x08\x12\x0f\n\x07user_id\x18\x02 \x01(\t2?\n\x04\x41uth\x12\x37\n\nCheckToken\x12\x13.auth.CheckTokenReq\x1a\x14.auth.CheckTokenRespB\x04Z\x02./b\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'auth_service_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'Z\002./' - _globals['_CHECKTOKENREQ']._serialized_start=28 - _globals['_CHECKTOKENREQ']._serialized_end=58 - _globals['_CHECKTOKENRESP']._serialized_start=60 - _globals['_CHECKTOKENRESP']._serialized_end=108 - _globals['_AUTH']._serialized_start=110 - _globals['_AUTH']._serialized_end=173 -# @@protoc_insertion_point(module_scope) diff --git a/grpc_auth/auth_service_pb2.pyi b/grpc_auth/auth_service_pb2.pyi deleted file mode 100644 index 818ceae..0000000 --- a/grpc_auth/auth_service_pb2.pyi +++ /dev/null @@ -1,19 +0,0 @@ -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional - -DESCRIPTOR: _descriptor.FileDescriptor - -class CheckTokenReq(_message.Message): - __slots__ = ("token",) - TOKEN_FIELD_NUMBER: _ClassVar[int] - token: str - def __init__(self, token: _Optional[str] = ...) -> None: ... - -class CheckTokenResp(_message.Message): - __slots__ = ("valid", "user_id") - VALID_FIELD_NUMBER: _ClassVar[int] - USER_ID_FIELD_NUMBER: _ClassVar[int] - valid: bool - user_id: str - def __init__(self, valid: bool = ..., user_id: _Optional[str] = ...) -> None: ... diff --git a/grpc_auth/auth_service_pb2_grpc.py b/grpc_auth/auth_service_pb2_grpc.py deleted file mode 100644 index 308ce1a..0000000 --- a/grpc_auth/auth_service_pb2_grpc.py +++ /dev/null @@ -1,102 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc -import warnings - -from . import auth_service_pb2 as auth__service__pb2 - -GRPC_GENERATED_VERSION = '1.63.0' -GRPC_VERSION = grpc.__version__ -EXPECTED_ERROR_RELEASE = '1.65.0' -SCHEDULED_RELEASE_DATE = 'June 25, 2024' -_version_not_supported = False - -try: - from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) -except ImportError: - _version_not_supported = True - -if _version_not_supported: - warnings.warn( - f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in auth_service_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' - + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', - RuntimeWarning - ) - - -class AuthStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.CheckToken = channel.unary_unary( - '/auth.Auth/CheckToken', - request_serializer=auth__service__pb2.CheckTokenReq.SerializeToString, - response_deserializer=auth__service__pb2.CheckTokenResp.FromString, - _registered_method=True) - - -class AuthServicer(object): - """Missing associated documentation comment in .proto file.""" - - def CheckToken(self, request, context): - """Check & get details about a user authorization token. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_AuthServicer_to_server(servicer, server): - rpc_method_handlers = { - 'CheckToken': grpc.unary_unary_rpc_method_handler( - servicer.CheckToken, - request_deserializer=auth__service__pb2.CheckTokenReq.FromString, - response_serializer=auth__service__pb2.CheckTokenResp.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'auth.Auth', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class Auth(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def CheckToken(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/auth.Auth/CheckToken', - auth__service__pb2.CheckTokenReq.SerializeToString, - auth__service__pb2.CheckTokenResp.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) diff --git a/grpc_auth/service.py b/grpc_auth/service.py deleted file mode 100644 index e4c9181..0000000 --- a/grpc_auth/service.py +++ /dev/null @@ -1,47 +0,0 @@ -import grpc, time, os -from concurrent import futures - -from . import ( - auth_service_pb2_grpc as pb2_grpc, - auth_service_pb2 as pb2 -) - -from database import db - - -class AuthService(pb2_grpc.AuthServicer): - def __init__(self, *args, **kwargs): - pass - - def CheckToken(self, request, context): - authed = False - for key, val in context.invocation_metadata(): - if key == "x-token" and val == os.environ["GRPC_AUTH_TOKEN"]: - authed = True - break - if not authed: - context.abort(grpc.StatusCode.UNAUTHENTICATED, "Invalid or missing token") - - account = db.usersv0.find_one({"tokens": request.token}, projection={ - "_id": 1, - "ban.state": 1, - "ban.expires": 1 - }) - if account: - if account and \ - (account["ban"]["state"] == "perm_ban" or \ - (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time())): - account = None - - return pb2.CheckTokenResp( - valid=(account is not None), - user_id=(account["_id"] if account else None) - ) - - -def serve(): - server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) - pb2_grpc.add_AuthServicer_to_server(AuthService(), server) - server.add_insecure_port(os.environ["GRPC_AUTH_ADDRESS"]) - server.start() - server.wait_for_termination() diff --git a/grpc_uploads/client.py b/grpc_uploads/client.py deleted file mode 100644 index da67506..0000000 --- a/grpc_uploads/client.py +++ /dev/null @@ -1,6 +0,0 @@ -import grpc, os -from . import uploads_service_pb2_grpc as pb2_grpc -from . import uploads_service_pb2 as pb2 - -channel = grpc.insecure_channel(os.getenv("GRPC_UPLOADS_ADDRESS")) -stub = pb2_grpc.UploadsStub(channel) diff --git a/grpc_uploads/uploads_service_pb2.py b/grpc_uploads/uploads_service_pb2.py deleted file mode 100644 index 6bf1d9b..0000000 --- a/grpc_uploads/uploads_service_pb2.py +++ /dev/null @@ -1,36 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: uploads_service.proto -# Protobuf Python Version: 5.26.1 -"""Generated protocol buffer code.""" -from google.protobuf import descriptor as _descriptor -from google.protobuf import descriptor_pool as _descriptor_pool -from google.protobuf import symbol_database as _symbol_database -from google.protobuf.internal import builder as _builder -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 - - -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x15uploads_service.proto\x12\x07uploads\x1a\x1bgoogle/protobuf/empty.proto\"*\n\x0c\x43laimFileReq\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x02 \x01(\t\"h\n\rClaimFileResp\x12\n\n\x02id\x18\x01 \x01(\t\x12\x0c\n\x04mime\x18\x02 \x01(\t\x12\x10\n\x08\x66ilename\x18\x03 \x01(\t\x12\x0c\n\x04size\x18\x04 \x01(\x03\x12\r\n\x05width\x18\x05 \x01(\x05\x12\x0e\n\x06height\x18\x06 \x01(\x05\"\x1b\n\rDeleteFileReq\x12\n\n\x02id\x18\x01 \x01(\t\" \n\rClearFilesReq\x12\x0f\n\x07user_id\x18\x01 \x01(\t2\xc1\x01\n\x07Uploads\x12:\n\tClaimFile\x12\x15.uploads.ClaimFileReq\x1a\x16.uploads.ClaimFileResp\x12<\n\nDeleteFile\x12\x16.uploads.DeleteFileReq\x1a\x16.google.protobuf.Empty\x12<\n\nClearFiles\x12\x16.uploads.ClearFilesReq\x1a\x16.google.protobuf.EmptyB\x04Z\x02./b\x06proto3') - -_globals = globals() -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'uploads_service_pb2', _globals) -if not _descriptor._USE_C_DESCRIPTORS: - _globals['DESCRIPTOR']._loaded_options = None - _globals['DESCRIPTOR']._serialized_options = b'Z\002./' - _globals['_CLAIMFILEREQ']._serialized_start=63 - _globals['_CLAIMFILEREQ']._serialized_end=105 - _globals['_CLAIMFILERESP']._serialized_start=107 - _globals['_CLAIMFILERESP']._serialized_end=211 - _globals['_DELETEFILEREQ']._serialized_start=213 - _globals['_DELETEFILEREQ']._serialized_end=240 - _globals['_CLEARFILESREQ']._serialized_start=242 - _globals['_CLEARFILESREQ']._serialized_end=274 - _globals['_UPLOADS']._serialized_start=277 - _globals['_UPLOADS']._serialized_end=470 -# @@protoc_insertion_point(module_scope) diff --git a/grpc_uploads/uploads_service_pb2.pyi b/grpc_uploads/uploads_service_pb2.pyi deleted file mode 100644 index e649931..0000000 --- a/grpc_uploads/uploads_service_pb2.pyi +++ /dev/null @@ -1,42 +0,0 @@ -from google.protobuf import empty_pb2 as _empty_pb2 -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from typing import ClassVar as _ClassVar, Optional as _Optional - -DESCRIPTOR: _descriptor.FileDescriptor - -class ClaimFileReq(_message.Message): - __slots__ = ("id", "bucket") - ID_FIELD_NUMBER: _ClassVar[int] - BUCKET_FIELD_NUMBER: _ClassVar[int] - id: str - bucket: str - def __init__(self, id: _Optional[str] = ..., bucket: _Optional[str] = ...) -> None: ... - -class ClaimFileResp(_message.Message): - __slots__ = ("id", "mime", "filename", "size", "width", "height") - ID_FIELD_NUMBER: _ClassVar[int] - MIME_FIELD_NUMBER: _ClassVar[int] - FILENAME_FIELD_NUMBER: _ClassVar[int] - SIZE_FIELD_NUMBER: _ClassVar[int] - WIDTH_FIELD_NUMBER: _ClassVar[int] - HEIGHT_FIELD_NUMBER: _ClassVar[int] - id: str - mime: str - filename: str - size: int - width: int - height: int - def __init__(self, id: _Optional[str] = ..., mime: _Optional[str] = ..., filename: _Optional[str] = ..., size: _Optional[int] = ..., width: _Optional[int] = ..., height: _Optional[int] = ...) -> None: ... - -class DeleteFileReq(_message.Message): - __slots__ = ("id",) - ID_FIELD_NUMBER: _ClassVar[int] - id: str - def __init__(self, id: _Optional[str] = ...) -> None: ... - -class ClearFilesReq(_message.Message): - __slots__ = ("user_id",) - USER_ID_FIELD_NUMBER: _ClassVar[int] - user_id: str - def __init__(self, user_id: _Optional[str] = ...) -> None: ... diff --git a/grpc_uploads/uploads_service_pb2_grpc.py b/grpc_uploads/uploads_service_pb2_grpc.py deleted file mode 100644 index 44df78a..0000000 --- a/grpc_uploads/uploads_service_pb2_grpc.py +++ /dev/null @@ -1,191 +0,0 @@ -# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT! -"""Client and server classes corresponding to protobuf-defined services.""" -import grpc -import warnings - -from google.protobuf import empty_pb2 as google_dot_protobuf_dot_empty__pb2 -from . import uploads_service_pb2 as uploads__service__pb2 - -GRPC_GENERATED_VERSION = '1.63.0' -GRPC_VERSION = grpc.__version__ -EXPECTED_ERROR_RELEASE = '1.65.0' -SCHEDULED_RELEASE_DATE = 'June 25, 2024' -_version_not_supported = False - -try: - from grpc._utilities import first_version_is_lower - _version_not_supported = first_version_is_lower(GRPC_VERSION, GRPC_GENERATED_VERSION) -except ImportError: - _version_not_supported = True - -if _version_not_supported: - warnings.warn( - f'The grpc package installed is at version {GRPC_VERSION},' - + f' but the generated code in uploads_service_pb2_grpc.py depends on' - + f' grpcio>={GRPC_GENERATED_VERSION}.' - + f' Please upgrade your grpc module to grpcio>={GRPC_GENERATED_VERSION}' - + f' or downgrade your generated code using grpcio-tools<={GRPC_VERSION}.' - + f' This warning will become an error in {EXPECTED_ERROR_RELEASE},' - + f' scheduled for release on {SCHEDULED_RELEASE_DATE}.', - RuntimeWarning - ) - - -class UploadsStub(object): - """Missing associated documentation comment in .proto file.""" - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.ClaimFile = channel.unary_unary( - '/uploads.Uploads/ClaimFile', - request_serializer=uploads__service__pb2.ClaimFileReq.SerializeToString, - response_deserializer=uploads__service__pb2.ClaimFileResp.FromString, - _registered_method=True) - self.DeleteFile = channel.unary_unary( - '/uploads.Uploads/DeleteFile', - request_serializer=uploads__service__pb2.DeleteFileReq.SerializeToString, - response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - _registered_method=True) - self.ClearFiles = channel.unary_unary( - '/uploads.Uploads/ClearFiles', - request_serializer=uploads__service__pb2.ClearFilesReq.SerializeToString, - response_deserializer=google_dot_protobuf_dot_empty__pb2.Empty.FromString, - _registered_method=True) - - -class UploadsServicer(object): - """Missing associated documentation comment in .proto file.""" - - def ClaimFile(self, request, context): - """Claim a file - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def DeleteFile(self, request, context): - """Delete a file - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def ClearFiles(self, request, context): - """Clear a user's files - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_UploadsServicer_to_server(servicer, server): - rpc_method_handlers = { - 'ClaimFile': grpc.unary_unary_rpc_method_handler( - servicer.ClaimFile, - request_deserializer=uploads__service__pb2.ClaimFileReq.FromString, - response_serializer=uploads__service__pb2.ClaimFileResp.SerializeToString, - ), - 'DeleteFile': grpc.unary_unary_rpc_method_handler( - servicer.DeleteFile, - request_deserializer=uploads__service__pb2.DeleteFileReq.FromString, - response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - ), - 'ClearFiles': grpc.unary_unary_rpc_method_handler( - servicer.ClearFiles, - request_deserializer=uploads__service__pb2.ClearFilesReq.FromString, - response_serializer=google_dot_protobuf_dot_empty__pb2.Empty.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'uploads.Uploads', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - - # This class is part of an EXPERIMENTAL API. -class Uploads(object): - """Missing associated documentation comment in .proto file.""" - - @staticmethod - def ClaimFile(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/uploads.Uploads/ClaimFile', - uploads__service__pb2.ClaimFileReq.SerializeToString, - uploads__service__pb2.ClaimFileResp.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def DeleteFile(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/uploads.Uploads/DeleteFile', - uploads__service__pb2.DeleteFileReq.SerializeToString, - google_dot_protobuf_dot_empty__pb2.Empty.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) - - @staticmethod - def ClearFiles(request, - target, - options=(), - channel_credentials=None, - call_credentials=None, - insecure=False, - compression=None, - wait_for_ready=None, - timeout=None, - metadata=None): - return grpc.experimental.unary_unary( - request, - target, - '/uploads.Uploads/ClearFiles', - uploads__service__pb2.ClearFilesReq.SerializeToString, - google_dot_protobuf_dot_empty__pb2.Empty.FromString, - options, - channel_credentials, - insecure, - call_credentials, - compression, - wait_for_ready, - timeout, - metadata, - _registered_method=True) diff --git a/main.py b/main.py index b453bb1..f95f2d4 100644 --- a/main.py +++ b/main.py @@ -11,7 +11,6 @@ from cloudlink import CloudlinkServer from supporter import Supporter from security import background_tasks_loop -from grpc_auth import service as grpc_auth from rest_api import app as rest_api @@ -26,9 +25,6 @@ # Start background tasks loop Thread(target=background_tasks_loop, daemon=True).start() - # Start gRPC services - Thread(target=grpc_auth.serve, daemon=True).start() - # Initialise REST API rest_api.cl = cl rest_api.supporter = supporter diff --git a/requirements.txt b/requirements.txt index 5c8dca2..1a55046 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,8 +11,6 @@ uvicorn py-radix pydantic msgpack -grpcio -protobuf pyotp emoji websockets diff --git a/rest_api/v0/chats.py b/rest_api/v0/chats.py index be1f795..ba87eb7 100644 --- a/rest_api/v0/chats.py +++ b/rest_api/v0/chats.py @@ -6,7 +6,7 @@ import security from database import db, get_total_pages -from uploads import claim_file, delete_file +from uploads import claim_file, unclaim_file from utils import log chats_bp = Blueprint("chats_bp", __name__, url_prefix="/chats") @@ -82,7 +82,7 @@ async def create_chat(data: ChatBody): # Claim icon if data.icon: try: - claim_file(data.icon, "icons") + claim_file(data.icon, "icons", request.user) except Exception as e: log(f"Unable to claim icon: {e}") return {"error": True, "type": "unableToClaimIcon"}, 500 @@ -183,18 +183,20 @@ async def update_chat(chat_id, data: ChatBody): updated_vals = {"_id": chat_id} if data.nickname is not None and chat["nickname"] != data.nickname: updated_vals["nickname"] = data.nickname - app.supporter.create_post(chat_id, "Server", f"@{request.user} changed the nickname of the group chat to '{chat['nickname']}'.", chat_members=chat["members"]) + app.supporter.create_post(chat_id, "Server", f"@{request.user} changed the nickname of the group chat to '{data.nickname}'.", chat_members=chat["members"]) if data.icon is not None and chat["icon"] != data.icon: # Claim icon (and delete old one) if data.icon != "": try: - updated_vals["icon"] = claim_file(data.icon, "icons")["id"] + claim_file(data.icon, "icons", request.user) except Exception as e: log(f"Unable to claim icon: {e}") return {"error": True, "type": "unableToClaimIcon"}, 500 + else: + updated_vals["icon"] = data.icon if chat["icon"]: try: - delete_file(chat["icon"]) + unclaim_file(chat["icon"]) except Exception as e: log(f"Unable to delete icon: {e}") app.supporter.create_post(chat_id, "Server", f"@{request.user} changed the icon of the group chat.", chat_members=chat["members"]) @@ -270,7 +272,7 @@ async def leave_chat(chat_id): else: if chat["icon"]: try: - delete_file(chat["icon"]) + unclaim_file(chat["icon"]) except Exception as e: log(f"Unable to delete icon: {e}") db.posts.delete_many({"post_origin": chat_id, "isDeleted": False}) @@ -658,10 +660,12 @@ async def create_chat_emote(chat_id: str, emote_type: Literal["emojis", "sticker # Claim file try: - file = claim_file(emote_id, emote_type) + claim_file(emote_id, emote_type, request.user) except Exception as e: log(f"Unable to claim emote: {e}") return {"error": True, "type": "unableToClaimEmote"}, 500 + else: + file = db.files.find_one({"_id": emote_id}) # Fall back to filename if no name is specified if not data.name: @@ -783,6 +787,6 @@ async def delete_chat_emote(chat_id: str, emote_type: Literal["emojis", "sticker "_id": emote_id, "chat_id": chat_id }, usernames=chat["members"]) - delete_file(emote_id) + unclaim_file(emote_id) return {"error": False}, 200 diff --git a/rest_api/v0/home.py b/rest_api/v0/home.py index 898697c..d3a1fe8 100644 --- a/rest_api/v0/home.py +++ b/rest_api/v0/home.py @@ -92,12 +92,16 @@ async def create_home_post(data: PostBody): # Claim attachments attachments = [] - for attachment_id in set(data.attachments): + for attachment_id in data.attachments: + if attachment_id in attachments: + continue try: - attachments.append(claim_file(attachment_id, "attachments")) + claim_file(attachment_id, "attachments", request.user) except Exception as e: log(f"Unable to claim attachment: {e}") return {"error": True, "type": "unableToClaimAttachment"}, 500 + else: + attachments.append(attachment_id) # Make sure the post has text content or at least 1 attachment or at least 1 sticker if not data.content and not attachments and not data.stickers: diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index b98ac62..0d31e0d 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -3,8 +3,6 @@ from pydantic import BaseModel, Field from typing import Optional, List, Literal from copy import copy -from base64 import b64encode -from io import BytesIO import pymongo import uuid import time @@ -15,7 +13,7 @@ import security from database import db, rdb, get_total_pages -from uploads import claim_file, delete_file +from uploads import claim_file, unclaim_file from utils import log @@ -150,13 +148,13 @@ async def update_config(data: UpdateConfigBody): cur_avatar = db.usersv0.find_one({"_id": request.user}, projection={"avatar": 1})["avatar"] if new_config["avatar"] != "": try: - claim_file(new_config["avatar"], "icons") + claim_file(new_config["avatar"], "icons", request.user) except Exception as e: log(f"Unable to claim avatar: {e}") return {"error": True, "type": "unableToClaimAvatar"}, 500 if cur_avatar: try: - delete_file(cur_avatar) + unclaim_file(cur_avatar) except Exception as e: log(f"Unable to delete avatar: {e}") diff --git a/rest_api/v0/posts.py b/rest_api/v0/posts.py index 5176432..cdc41fd 100644 --- a/rest_api/v0/posts.py +++ b/rest_api/v0/posts.py @@ -8,7 +8,7 @@ import security from database import db, get_total_pages -from uploads import claim_file, delete_file +from uploads import claim_file, unclaim_file from utils import log @@ -294,13 +294,11 @@ async def delete_attachment(post_id: str, attachment_id: str): abort(403) # Delete attachment - for attachment in copy(post["attachments"]): - if attachment["id"] == attachment_id: - try: - delete_file(attachment_id) - except Exception as e: - log(f"Unable to delete attachment: {e}") - post["attachments"].remove(attachment) + if attachment_id in post["attachments"]: + post["attachments"].remove(attachment_id) + unclaim_file(attachment_id) + else: + abort(404) if post["p"] or post["attachments"] > 0: # Update post @@ -364,7 +362,7 @@ async def delete_post(query_args: PostIdQueryArgs): # Delete attachments for attachment in post["attachments"]: try: - delete_file(attachment["id"]) + unclaim_file(attachment["id"]) except Exception as e: log(f"Unable to delete attachment: {e}") @@ -460,12 +458,16 @@ async def create_chat_post(chat_id, data: PostBody): # Claim attachments attachments = [] if chat_id != "livechat": - for attachment_id in set(data.attachments): + for attachment_id in data.attachments: + if attachment_id in attachments: + continue try: - attachments.append(claim_file(attachment_id, "attachments")) + claim_file(attachment_id, "attachments", request.user) except Exception as e: log(f"Unable to claim attachment: {e}") return {"error": True, "type": "unableToClaimAttachment"}, 500 + else: + attachments.append(attachment_id) # Make sure the post has text content or at least 1 attachment or at least 1 sticker if not data.content and not attachments and not data.stickers: diff --git a/security.py b/security.py index ccea8a8..b948953 100644 --- a/security.py +++ b/security.py @@ -1,10 +1,10 @@ from hashlib import sha256 from typing import Optional -import time, requests, os, uuid, secrets, bcrypt, msgpack +import time, requests, uuid, secrets, bcrypt, msgpack from database import db, rdb from utils import log -from uploads import clear_files +from uploads import unclaim_all_files """ Meower Security Module @@ -48,6 +48,7 @@ class UserFlags: DELETED = 2 PROTECTED = 4 POST_RATELIMIT_BYPASS = 8 + ULTRA_HD_UPLOADS = 16 # joke flag class AdminPermissions: @@ -381,7 +382,7 @@ def delete_account(username, purge=False): db.authenticators.delete_many({"user": username}) # Delete uploaded files - clear_files(username) + unclaim_all_files(username) # Delete user settings db.user_settings.delete_one({"_id": username}) @@ -450,7 +451,7 @@ def get_netinfo(ip_address): netinfo = db.netinfo.find_one({"_id": ip_hash}) if not netinfo: resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") - if resp.ok: + if resp.ok and resp.json()["status"] == "success": resp_json = resp.json() netinfo = { "_id": ip_hash, diff --git a/supporter.py b/supporter.py index bd220fa..cc956ea 100644 --- a/supporter.py +++ b/supporter.py @@ -4,7 +4,6 @@ from cloudlink import CloudlinkServer from database import db, rdb -from uploads import FileDetails """ Meower Supporter Module @@ -64,7 +63,7 @@ def create_post( origin: str, author: str, content: str, - attachments: list[FileDetails] = [], + attachments: list[str] = [], stickers: list[str] = [], nonce: Optional[str] = None, chat_members: list[str] = [], @@ -207,6 +206,21 @@ def parse_posts_v0( else: post.update({"reply_to": [None for _ in post.pop("reply_to", [])]}) + # Attachments + post["attachments"] = list(db.files.aggregate([ + {"$match": {"_id": {"$in": post["attachments"]}}}, + {"$project": { + "id": "$_id", + "_id": 0, + "mime": 1, + "thumbnail_mime": 1, + "size": 1, + "filename": 1, + "width": 1, + "height": 1 + }} + ])) + # Custom emojis if post.get("emojis"): post["emojis"] = list(db.chat_emojis.find({ diff --git a/uploads.py b/uploads.py index fd90d35..005c9b6 100644 --- a/uploads.py +++ b/uploads.py @@ -1,52 +1,35 @@ -from typing import TypedDict -import os +from database import db -from grpc_uploads import client as grpc_uploads +class FileNotFoundError(Exception): pass +class FileAlreadyClaimedError(Exception): pass -class FileDetails(TypedDict): - id: str - mime: str - filename: str - size: int - width: int - height: int +def claim_file(file_id: str, bucket: str, uploader: str): + # Find file + file = db.files.find_one() + if not file: + raise FileNotFoundError -def claim_file(file_id: str, bucket: str) -> FileDetails: - resp = grpc_uploads.stub.ClaimFile( - grpc_uploads.pb2.ClaimFileReq( - id=file_id, - bucket=bucket - ), - metadata=( - ("x-token", os.getenv("GRPC_UPLOADS_TOKEN")), - ), - timeout=30 - ) - return { - "id": resp.id, - "mime": resp.mime, - "filename": resp.filename, - "size": resp.size, - "width": resp.width, - "height": resp.height - } + result = db.files.update_one({ + "_id": file_id, + "bucket": bucket, + "uploaded_by": uploader, + "claimed": False + }, {"$set": {"claimed": True}}) + if result.matched_count == 0: + raise FileNotFoundError + elif result.modified_count == 0: + raise FileAlreadyClaimedError -def delete_file(file_id: str): - grpc_uploads.stub.DeleteFile( - grpc_uploads.pb2.DeleteFileReq(id=file_id), - metadata=( - ("x-token", os.getenv("GRPC_UPLOADS_TOKEN")), - ), - timeout=30 +def unclaim_file(file_id: str): + db.files.update_many( + {"_id": file_id}, + {"$set": {"claimed": False, "uploaded_at": 0}} ) -def clear_files(user_id: str): - grpc_uploads.stub.ClearFiles( - grpc_uploads.pb2.ClearFilesReq(user_id=user_id), - metadata=( - ("x-token", os.getenv("GRPC_UPLOADS_TOKEN")), - ), - timeout=30 +def unclaim_all_files(username: str): + db.files.update_one( + {"uploaded_by": username}, + {"$set": {"claimed": False, "uploaded_at": 0}} )