diff --git a/.env.example b/.env.example index 44c4f72..f8ea401 100644 --- a/.env.example +++ b/.env.example @@ -11,9 +11,24 @@ API_ROOT= INTERNAL_API_ENDPOINT="http://127.0.0.1:3001" # used for proxying CL3 commands INTERNAL_API_TOKEN="" # used for authenticating internal API requests (gives access to any account, meant to be used by CL3) +SENTRY_DSN= + CAPTCHA_SITEKEY= CAPTCHA_SECRET= +EMAIL_SMTP_HOST= +EMAIL_SMTP_PORT= +EMAIL_SMTP_TLS= +EMAIL_SMTP_USERNAME= +EMAIL_SMTP_PASSWORD= +EMAIL_FROM_NAME= +EMAIL_FROM_ADDRESS= +EMAIL_PLATFORM_NAME="Meower" +EMAIL_PLATFORM_LOGO="" +EMAIL_PLATFORM_BRAND="Meower Media" +EMAIL_PLATFORM_FRONTEND="https://meower.org" +EMAIL_PLATFORM_SUPPORT="support@meower.org" + GRPC_AUTH_ADDRESS="0.0.0.0:5000" GRPC_AUTH_TOKEN= diff --git a/cloudlink.py b/cloudlink.py index 65ea0a7..e2cd568 100755 --- a/cloudlink.py +++ b/cloudlink.py @@ -51,7 +51,8 @@ def __init__(self): #"Kicked": "E:020 | Kicked", -- deprecated #"ChatFull": "E:023 | Chat full", -- deprecated #"LoggedOut": "I:024 | Logged out", -- deprecated - "Deleted": "E:025 | Deleted" + "Deleted": "E:025 | Deleted", + "AccountLocked": "E:026 | Account Locked" } self.commands: dict[str, function] = { # Core commands @@ -226,7 +227,8 @@ def __init__( self.server = server self.websocket = websocket - # Set username, protocol version, IP, and trusted status + # Set account session ID, username, protocol version, IP, and trusted status + self.acc_session_id: Optional[str] = None self.username: Optional[str] = None try: self.proto_version: int = int(self.req_params.get("v")[0]) @@ -255,7 +257,7 @@ def ip(self): else: return self.websocket.remote_address - def authenticate(self, account: dict[str, Any], token: str, listener: Optional[str] = None): + def authenticate(self, acc_session: dict[str, Any], token: str, account: dict[str, Any], listener: Optional[str] = None): if self.username: self.logout() @@ -265,6 +267,7 @@ def authenticate(self, account: dict[str, Any], token: str, listener: Optional[s return self.send_statuscode("Banned", listener) # Authenticate + self.acc_session_id = acc_session["_id"] self.username = account["_id"] if self.username in self.server.usernames: self.server.usernames[self.username].append(self) @@ -275,6 +278,7 @@ def authenticate(self, account: dict[str, Any], token: str, listener: Optional[s # Send auth payload self.send("auth", { "username": self.username, + "session": acc_session, "token": token, "account": account, "relationships": self.proxy_api_request("/me/relationships", "get")["autoget"], @@ -307,6 +311,7 @@ def proxy_api_request( headers.update({ "X-Internal-Token": os.environ["INTERNAL_API_TOKEN"], "X-Internal-Ip": self.ip, + "X-Internal-UA": self.websocket.request_headers.get("User-Agent"), }) if self.username: headers["X-Internal-Username"] = self.username @@ -321,8 +326,6 @@ def proxy_api_request( return resp else: match resp["type"]: - case "repairModeEnabled": - self.kick() case "ipBlocked"|"registrationBlocked": self.send_statuscode("Blocked", listener) case "badRequest": @@ -335,6 +338,8 @@ def proxy_api_request( self.send_statuscode("2FARequired", listener) case "accountDeleted": self.send_statuscode("Deleted", listener) + case "accountLocked": + self.send_statuscode("AccountLocked", listener) case "accountBanned": self.send_statuscode("Banned", listener) case "tooManyRequests": @@ -353,10 +358,8 @@ def send(self, cmd: str, val: Any, extra: Optional[dict] = None, listener: Optio def send_statuscode(self, statuscode: str, listener: Optional[str] = None): return self.send("statuscode", self.server.statuscodes[statuscode], listener=listener) - def kick(self): - async def _kick(): - await self.websocket.close() - asyncio.create_task(_kick()) + async def kick(self): + await self.websocket.close() class CloudlinkCommands: @staticmethod @@ -389,7 +392,7 @@ async def authpswd(client: CloudlinkClient, val, listener: Optional[str] = None) else: if resp and not resp["error"]: # Authenticate client - client.authenticate(resp["account"], resp["token"], listener=listener) + client.authenticate(resp["session"], resp["token"], resp["account"], listener=listener) # Tell the client it is authenticated client.send_statuscode("OK", listener) diff --git a/database.py b/database.py index c1d8e14..e66ba1a 100644 --- a/database.py +++ b/database.py @@ -1,12 +1,16 @@ import pymongo +import pymongo.errors import redis import os import secrets +import time from radix import Radix +from hashlib import sha256 +from base64 import urlsafe_b64encode from utils import log -CURRENT_DB_VERSION = 9 +CURRENT_DB_VERSION = 10 # Create Redis connection log("Connecting to Redis...") @@ -41,7 +45,9 @@ # Create usersv0 indexes try: db.usersv0.create_index([("lower_username", pymongo.ASCENDING)], name="lower_username", unique=True) except: pass -try: db.usersv0.create_index([("tokens", pymongo.ASCENDING)], name="tokens", unique=True) +try: db.usersv0.create_index([("email", pymongo.ASCENDING)], name="email", unique=True) +except: pass +try: db.usersv0.create_index([("normalized_email_hash", pymongo.ASCENDING)], name="normalized_email_hash", unique=True) except: pass try: db.usersv0.create_index([("created", pymongo.DESCENDING)], name="recent_users") except: pass @@ -60,24 +66,24 @@ try: db.authenticators.create_index([("user", pymongo.ASCENDING)], name="user") except: pass -# Create data exports indexes -try: db.data_exports.create_index([("user", pymongo.ASCENDING)], name="user") +# Create account sessions indexes +try: db.acc_sessions.create_index([("user", pymongo.ASCENDING)], name="user") except: pass - -# Create relationships indexes -try: db.relationships.create_index([("_id.from", pymongo.ASCENDING)], name="from") +try: db.acc_sessions.create_index([("ip", pymongo.ASCENDING)], name="ip") except: pass - -# Create netinfo indexes -try: db.netinfo.create_index([("last_refreshed", pymongo.ASCENDING)], name="last_refreshed") +try: db.acc_sessions.create_index([("refreshed_at", pymongo.ASCENDING)], name="refreshed_at") except: pass -# Create netlog indexes -try: db.netlog.create_index([("_id.ip", pymongo.ASCENDING)], name="ip") +# Create security log indexes +try: db.security_log.create_index([("user", pymongo.ASCENDING)], name="user") except: pass -try: db.netlog.create_index([("_id.user", pymongo.ASCENDING)], name="user") + +# Create data exports indexes +try: db.data_exports.create_index([("user", pymongo.ASCENDING)], name="user") except: pass -try: db.netlog.create_index([("last_used", pymongo.ASCENDING)], name="last_used") + +# Create relationships indexes +try: db.relationships.create_index([("_id.from", pymongo.ASCENDING)], name="from") except: pass # Create posts indexes @@ -179,39 +185,30 @@ # Create default database items -for username in ["Server", "Deleted", "Meower", "Admin", "username"]: - try: - db.usersv0.insert_one({ - "_id": username, - "lower_username": username.lower(), - "uuid": None, - "created": None, - "pfp_data": None, - "avatar": None, - "avatar_color": None, - "quote": None, - "pswd": None, - "tokens": None, - "flags": 1, - "permissions": None, - "ban": None, - "last_seen": None, - "delete_after": None - }) - except: pass try: db.config.insert_one({ "_id": "migration", "database": 1 }) -except: pass +except pymongo.errors.DuplicateKeyError: pass try: db.config.insert_one({ "_id": "status", "repair_mode": False, "registration": True }) -except: pass +except pymongo.errors.DuplicateKeyError: pass +try: + db.config.insert_one({ + "_id": "signing_keys", + "acc": secrets.token_bytes(64), + "email": secrets.token_bytes(64) + }) +except pymongo.errors.DuplicateKeyError: pass + + +# Load signing keys +signing_keys = db.config.find_one({"_id": "signing_keys"}) # Load netblocks @@ -306,6 +303,39 @@ def get_total_pages(collection: str, query: dict, page_size: int = 25) -> int: "mfa_recovery_code": user["mfa_recovery_code"][:10] }}) + # Delete system users from DB + log("[Migrator] Deleting system users from DB") + db.usersv0.delete_many({"_id": {"$in": ["Server", "Deleted", "Meower", "Admin", "username"]}}) + + # Emails + log("[Migrator] Adding email addresses") + db.usersv0.update_many({"email": {"$exists": False}}, {"$set": { + "email": "", + "normalized_email_hash": "" + }}) + + # New sessions + log("[Migrator] Adding new sessions") + for user in db.usersv0.find({ + "tokens": {"$exists": True}, + "last_seen": {"$ne": None, "$gt": int(time.time())-(86400*21)}, + }, projection={"_id": 1, "tokens": 1}): + if user["tokens"]: + for token in user["tokens"]: + rdb.set( + urlsafe_b64encode(sha256(token.encode()).digest()), + user["_id"], + ex=86400*21 # 21 days + ) + db.usersv0.update_many({}, {"$unset": {"tokens": ""}}) + try: db.usersv0.drop_index("tokens") + except: pass + + # No more netinfo and netlog + log("[Migrator] Removing netinfo and netlog") + db.netinfo.drop() + db.netlog.drop() + db.config.update_one({"_id": "migration"}, {"$set": {"database": CURRENT_DB_VERSION}}) log(f"[Migrator] Finished Migrating DB to version {CURRENT_DB_VERSION}") diff --git a/email_templates/_base.html b/email_templates/_base.html new file mode 100644 index 0000000..b1539e9 --- /dev/null +++ b/email_templates/_base.html @@ -0,0 +1,25 @@ + + + +
+
+ {{ env['EMAIL_PLATFORM_NAME'] }} Logo +
+ + + + + + + + + + {% block body %}{% endblock %} + + + + +
{{ subject }}
Hey {{ name }}!
- {{ env['EMAIL_PLATFORM_BRAND'] }}
+
+ + \ No newline at end of file diff --git a/email_templates/_base.txt b/email_templates/_base.txt new file mode 100644 index 0000000..7cd79e3 --- /dev/null +++ b/email_templates/_base.txt @@ -0,0 +1,5 @@ +Hey {{ name }}! + +{% block body %}{% endblock %} + +- {{ env['EMAIL_PLATFORM_BRAND'] }} \ No newline at end of file diff --git a/email_templates/locked.html b/email_templates/locked.html new file mode 100644 index 0000000..d9ece28 --- /dev/null +++ b/email_templates/locked.html @@ -0,0 +1,26 @@ +{% extends "_base.html" %} +{% block body %} + + + Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. + + + + + + You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. + + + + + + If you had multi-factor authentication enabled, it has been temporarily disabled as a precaution, in case it was modified by someone attempting to lock you out of your account. + + + + + + If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. + + +{% endblock %} \ No newline at end of file diff --git a/email_templates/locked.txt b/email_templates/locked.txt new file mode 100644 index 0000000..7650678 --- /dev/null +++ b/email_templates/locked.txt @@ -0,0 +1,10 @@ +{% extends "_base.txt" %} +{% block body %} +Your {{ env['EMAIL_PLATFORM_NAME'] }} account has been locked because we believe it may have been compromised. This can happen if your {{ env['EMAIL_PLATFORM_NAME'] }} password is weak, you used the same password on another website and that website was hacked, or you accidentally gave an access token to someone else. + +You will be required to reset your password using this email address ({{ address }}) before logging back in to {{ env['EMAIL_PLATFORM_NAME'] }}. + +If you had multi-factor authentication enabled, it has been temporarily disabled as a precaution, in case it was modified by someone attempting to lock you out of your account. + +If you have any questions, please reach out to {{ env['EMAIL_PLATFORM_SUPPORT'] }}. +{% endblock %} \ No newline at end of file diff --git a/email_templates/recover.html b/email_templates/recover.html new file mode 100644 index 0000000..6a33174 --- /dev/null +++ b/email_templates/recover.html @@ -0,0 +1,30 @@ +{% extends "_base.html" %} +{% block body %} + + + To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please click the button below. + + + + + + If you didn't request this, please ignore this email, no further action is required. + + + + + This link will expire in 30 minutes. + + + + + + Reset Password + + + +{% endblock %} \ No newline at end of file diff --git a/email_templates/recover.txt b/email_templates/recover.txt new file mode 100644 index 0000000..5035e13 --- /dev/null +++ b/email_templates/recover.txt @@ -0,0 +1,6 @@ +{% extends "_base.txt" %} +{% block body %} +To reset your {{ env['EMAIL_PLATFORM_NAME'] }} account password, please follow this link (this link will expire in 30 minutes): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/recover#{{ token }} + +If you didn't request this, please ignore this email, no further action is required. +{% endblock %} \ No newline at end of file diff --git a/email_templates/security_alert.html b/email_templates/security_alert.html new file mode 100644 index 0000000..38c755b --- /dev/null +++ b/email_templates/security_alert.html @@ -0,0 +1,32 @@ +{% extends "_base.html" %} +{% block body %} + {% if token %} + + {{ msg }} + + + + If this wasn't you, please click the button below to secure your account. + + + + This link will expire in 24 hours. + + + + + + This wasn't me! + + + + {% else %} + + {{ msg }} + + {% endif %} +{% endblock %} \ No newline at end of file diff --git a/email_templates/security_alert.txt b/email_templates/security_alert.txt new file mode 100644 index 0000000..edea1c4 --- /dev/null +++ b/email_templates/security_alert.txt @@ -0,0 +1,7 @@ +{% extends "_base.txt" %} +{% block body %} +{{ msg }} +{% if token %} +If this wasn't you, please follow this link to secure your account (this link will expire in 24 hours): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/lockdown#{{ token }} +{% endif %} +{% endblock %} \ No newline at end of file diff --git a/email_templates/verify.html b/email_templates/verify.html new file mode 100644 index 0000000..12b168c --- /dev/null +++ b/email_templates/verify.html @@ -0,0 +1,26 @@ +{% extends "_base.html" %} +{% block body %} + + To confirm adding your email address ({{ address }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please click the button below. + + + + If this wasn't you, please ignore this email, no further action is required. + + + + This link will expire in 30 minutes. + + + + + + Verify Email Address + + + +{% endblock %} \ No newline at end of file diff --git a/email_templates/verify.txt b/email_templates/verify.txt new file mode 100644 index 0000000..786ecd3 --- /dev/null +++ b/email_templates/verify.txt @@ -0,0 +1,6 @@ +{% extends "_base.txt" %} +{% block body %} +To confirm adding your email address ({{ address }}) to your {{ env['EMAIL_PLATFORM_NAME'] }} account, please follow this link (this link will expire in 30 minutes): {{ env['EMAIL_PLATFORM_FRONTEND'] }}/emails/verify#{{ token }} + +If this wasn't you, please ignore this email, no further action is required. +{% endblock %} \ No newline at end of file diff --git a/errors.py b/errors.py new file mode 100644 index 0000000..92f4d9b --- /dev/null +++ b/errors.py @@ -0,0 +1,7 @@ +class InvalidTokenSignature(Exception): pass + +class AccSessionTokenExpired(Exception): pass + +class AccSessionNotFound(Exception): pass + +class EmailTicketExpired(Exception): pass \ No newline at end of file diff --git a/grpc_auth/service.py b/grpc_auth/service.py index e4c9181..9a433bd 100644 --- a/grpc_auth/service.py +++ b/grpc_auth/service.py @@ -6,7 +6,10 @@ auth_service_pb2 as pb2 ) +from sentry_sdk import capture_exception + from database import db +from sessions import AccSession class AuthService(pb2_grpc.AuthServicer): @@ -22,15 +25,19 @@ def CheckToken(self, request, context): if not authed: context.abort(grpc.StatusCode.UNAUTHENTICATED, "Invalid or missing token") - account = db.usersv0.find_one({"tokens": request.token}, projection={ - "_id": 1, - "ban.state": 1, - "ban.expires": 1 - }) - if account: + try: + username = AccSession.get_username_by_token(request.token) + except Exception as e: + capture_exception(e) + else: + account = db.usersv0.find_one({"_id": username}, projection={ + "_id": 1, + "ban.state": 1, + "ban.expires": 1 + }) if account and \ - (account["ban"]["state"] == "perm_ban" or \ - (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time())): + (account["ban"]["state"] == "perm_ban" or \ + (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time())): account = None return pb2.CheckTokenResp( diff --git a/main.py b/main.py index b453bb1..a2a1955 100644 --- a/main.py +++ b/main.py @@ -5,6 +5,7 @@ import asyncio import os import uvicorn +import sentry_sdk from threading import Thread @@ -16,6 +17,9 @@ if __name__ == "__main__": + # Initialise Sentry (uses SENTRY_DSN env var) + sentry_sdk.init() + # Create Cloudlink server cl = CloudlinkServer() diff --git a/requirements.txt b/requirements.txt index 5c8dca2..11e0c6c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,4 +16,5 @@ protobuf pyotp emoji websockets -qrcode \ No newline at end of file +qrcode +sentry-sdk \ No newline at end of file diff --git a/rest_api/__init__.py b/rest_api/__init__.py index bd76c2d..6153be7 100755 --- a/rest_api/__init__.py +++ b/rest_api/__init__.py @@ -2,6 +2,7 @@ from quart_cors import cors from quart_schema import QuartSchema, RequestSchemaValidationError, validate_headers, hide from pydantic import BaseModel +from sentry_sdk import capture_exception import time, os from .v0 import v0 @@ -10,6 +11,7 @@ from .admin import admin_bp from database import db, blocked_ips, registration_blocked_ips +from sessions import AccSession import security @@ -41,18 +43,17 @@ async def internal_auth(): abort(401) request.internal_ip = request.headers.get("X-Internal-Ip") + request.headers["User-Agent"] = request.headers.get("X-Internal-UA") request.internal_username = request.headers.get("X-Internal-Username") request.bypass_captcha = True @app.before_request -async def check_ip(): +async def get_ip(): if hasattr(request, "internal_ip") and request.internal_ip: # internal IP forwarding request.ip = request.internal_ip else: request.ip = (request.headers.get("Cf-Connecting-Ip", request.remote_addr)) - if request.path != "/status" and blocked_ips.search_best(request.ip): - return {"error": True, "type": "ipBlocked"}, 403 @app.before_request @@ -74,13 +75,18 @@ async def check_auth(headers: TokenHeader): "ban.expires": 1 }) elif headers.token: # external auth - account = db.usersv0.find_one({"tokens": headers.token}, projection={ - "_id": 1, - "flags": 1, - "permissions": 1, - "ban.state": 1, - "ban.expires": 1 - }) + try: + username = AccSession.get_username_by_token(headers.token) + except Exception as e: + capture_exception(e) + else: + account = db.usersv0.find_one({"_id": username}, projection={ + "_id": 1, + "flags": 1, + "permissions": 1, + "ban.state": 1, + "ban.expires": 1 + }) if account: if account["ban"]["state"] == "perm_ban" or (account["ban"]["state"] == "temp_ban" and account["ban"]["expires"] > time.time()): diff --git a/rest_api/admin.py b/rest_api/admin.py index 5240fdc..d4c8c0d 100644 --- a/rest_api/admin.py +++ b/rest_api/admin.py @@ -8,6 +8,7 @@ import security from database import db, get_total_pages, blocked_ips, registration_blocked_ips +from sessions import AccSession admin_bp = Blueprint("admin_bp", __name__, url_prefix="/admin") @@ -531,21 +532,19 @@ async def get_user(username): # Get netlogs netlogs = [ { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], + "ip": session._db["ip"], + "user": session.username, + "last_used": session._db["refreshed_at"], } - for netlog in db.netlog.find( - {"_id.user": username}, sort=[("last_used", pymongo.DESCENDING)] - ) + for session in AccSession.get_all(username) ] # Get alts alts = [ - netlog["_id"]["user"] - for netlog in db.netlog.find( - {"_id.ip": {"$in": [netlog["ip"] for netlog in netlogs]}} - ) + session["user"] + for session in db.acc_sessions.find({ + "ip": {"$in": [netlog["ip"] for netlog in netlogs]} + }) ] if username in alts: alts.remove(username) @@ -556,7 +555,7 @@ async def get_user(username): payload["recent_ips"] = [ { "ip": netlog["ip"], - "netinfo": security.get_netinfo(netlog["ip"]), + "netinfo": security.get_ip_info(netlog["ip"]), "last_used": netlog["last_used"], "blocked": ( blocked_ips.search_best(netlog["ip"]) @@ -651,17 +650,17 @@ async def delete_user(username, query_args: DeleteUserQueryArgs): {"_id": username}, {"$set": {"delete_after": None}} ) elif deletion_mode in ["schedule", "immediate", "purge"]: - db.usersv0.update_one( - {"_id": username}, - { - "$set": { - "tokens": [], - "delete_after": int(time.time()) + (604800 if deletion_mode == "schedule" else 0), - } - }, - ) - for client in app.cl.usernames.get(username, []): - client.kick() + if deletion_mode == "schedule": + db.usersv0.update_one( + {"_id": username}, + { + "$set": { + "delete_after": int(time.time()) + (604800 if deletion_mode == "schedule" else 0), + } + }, + ) + for session in AccSession.get_all(username): + session.revoke() if deletion_mode in ["immediate", "purge"]: security.delete_account(username, purge=(deletion_mode == "purge")) else: @@ -709,7 +708,7 @@ async def ban_user(username, data: UpdateUserBanBody): data.state == "temp_ban" and data.expires > time.time() ): for client in app.cl.usernames.get(username, []): - client.kick() + await client.websocket.close() else: app.cl.send_event("update_config", {"ban": data.model_dump()}, usernames=[username]) @@ -828,12 +827,9 @@ async def kick_user(username): if not security.has_permission(request.permissions, security.AdminPermissions.KICK_USERS): abort(401) - # Revoke tokens - db.usersv0.update_one({"_id": username}, {"$set": {"tokens": []}}) - - # Kick clients - for client in app.cl.usernames.get(username, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(username): + session.revoke() # Add log security.add_audit_log( @@ -1098,9 +1094,6 @@ async def get_netinfo(ip): if not security.has_permission(request.permissions, security.AdminPermissions.VIEW_IPS): abort(403) - # Get netinfo - netinfo = security.get_netinfo(ip) - # Get netblocks netblocks = [] for radix_node in blocked_ips.search_covering(ip): @@ -1109,16 +1102,17 @@ async def get_netinfo(ip): netblocks.append(db.netblock.find_one({"_id": radix_node.prefix})) # Get netlogs - netlogs = [ - { - "ip": netlog["_id"]["ip"], - "user": netlog["_id"]["user"], - "last_used": netlog["last_used"], - } - for netlog in db.netlog.find( - {"_id.ip": ip}, sort=[("last_used", pymongo.DESCENDING)] - ) - ] + hit_users = set() + netlogs = [] + for session in db.acc_sessions.find({"ip": ip}, sort=[("refreshed_at", pymongo.DESCENDING)]): + if session["user"] in hit_users: + continue + netlogs.append({ + "ip": session["ip"], + "user": session["user"], + "last_used": session["refreshed_at"] + }) + hit_users.add(session["user"]) # Add log security.add_audit_log("got_netinfo", request.user, request.ip, {"ip": ip}) @@ -1126,7 +1120,7 @@ async def get_netinfo(ip): # Return netinfo, netblocks, and netlogs return { "error": False, - "netinfo": netinfo, + "netinfo": security.get_ip_info(ip), "netblocks": netblocks, "netlogs": netlogs, }, 200 @@ -1219,7 +1213,7 @@ async def create_netblock(cidr, data: NetblockBody): if data.type == 0: for client in copy(app.cl.clients): if blocked_ips.search_best(client.ip): - client.kick() + await client.websocket.close() # Add log security.add_audit_log( @@ -1321,7 +1315,7 @@ async def kick_all_clients(): # Kick all clients for client in copy(app.cl.clients): - client.kick() + await client.websocket.close() # Add log security.add_audit_log("kicked_all", request.user, request.ip, {}) @@ -1343,7 +1337,7 @@ async def enable_repair_mode(): # Kick all clients for client in copy(app.cl.clients): - client.kick() + await client.websocket.close() # Add log security.add_audit_log("enabled_repair_mode", request.user, request.ip, {}) diff --git a/rest_api/v0/__init__.py b/rest_api/v0/__init__.py index 3d9fc16..7fb6162 100644 --- a/rest_api/v0/__init__.py +++ b/rest_api/v0/__init__.py @@ -9,10 +9,12 @@ from .home import home_bp from .me import me_bp from .emojis import emojis_bp +from .emails import emails_bp v0 = Blueprint("v0", __name__) v0.register_blueprint(auth_bp) +v0.register_blueprint(emails_bp) v0.register_blueprint(me_bp) v0.register_blueprint(home_bp) v0.register_blueprint(inbox_bp) diff --git a/rest_api/v0/auth.py b/rest_api/v0/auth.py index f7c9857..1b35b5b 100644 --- a/rest_api/v0/auth.py +++ b/rest_api/v0/auth.py @@ -1,10 +1,15 @@ -import re, os, requests, pyotp, secrets +import re, os, requests, pyotp, secrets, time from pydantic import BaseModel from quart import Blueprint, request, abort, current_app as app from quart_schema import validate_request from pydantic import Field from typing import Optional -from database import db, registration_blocked_ips +from base64 import urlsafe_b64encode +from hashlib import sha256 +from threading import Thread + +from database import db, rdb, blocked_ips, registration_blocked_ips +from sessions import AccSession, EmailTicket import security auth_bp = Blueprint("auth_bp", __name__, url_prefix="/auth") @@ -16,6 +21,16 @@ class AuthRequest(BaseModel): mfa_recovery_code: Optional[str] = Field(default="", min_length=10, max_length=10) captcha: Optional[str] = Field(default="", max_length=2000) +class RecoverAccountBody(BaseModel): + email: str = Field(min_length=1, max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) + + +@auth_bp.before_request +async def ip_block_check(): + if blocked_ips.search_best(request.ip): + return {"error": True, "type": "ipBlocked"}, 403 + @auth_bp.post("/login") @validate_request(AuthRequest) @@ -26,10 +41,13 @@ async def login(data: AuthRequest): security.ratelimit(f"login:i:{request.ip}", 50, 900) # Get basic account details - account = db.usersv0.find_one({"lower_username": data.username.lower()}, projection={ + account = db.usersv0.find_one({ + "email": data.username + } if "@" in data.username else { + "lower_username": data.username.lower() + }, projection={ "_id": 1, "flags": 1, - "tokens": 1, "pswd": 1, "mfa_recovery_code": 1 }) @@ -40,12 +58,31 @@ async def login(data: AuthRequest): if account["flags"] & security.UserFlags.DELETED: return {"error": True, "type": "accountDeleted"}, 401 + # Make sure account isn't locked + if account["flags"] & security.UserFlags.LOCKED: + return {"error": True, "type": "accountLocked"}, 401 + # Make sure account isn't ratelimited if security.ratelimited(f"login:u:{account['_id']}"): abort(429) - # Check credentials - if data.password not in account["tokens"]: + # Legacy tokens (remove in the future at some point) + if len(data.password) == 86: + encoded_token = urlsafe_b64encode(sha256(data.password.encode()).digest()) + username = rdb.get(encoded_token) + if username and username.decode() == account["_id"]: + data.password = AccSession.create( + username.decode(), + request.ip, + request.headers.get("User-Agent") + ).token + rdb.delete(encoded_token) + + # Check credentials & get session + try: # token for already existing session + session = AccSession.get_by_token(data.password) + session.refresh(request.ip, request.headers.get("User-Agent"), check_token=data.password) + except: # no error capturing here, as it's probably just a password rather than a token, and we don't want to capture passwords # Check password password_valid = security.check_password_hash(data.password, account["pswd"]) @@ -70,6 +107,11 @@ async def login(data: AuthRequest): # Abort if password is invalid if not password_valid: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_password", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) # Check MFA @@ -85,34 +127,61 @@ async def login(data: AuthRequest): break if not passed: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_totp_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) elif data.mfa_recovery_code: if data.mfa_recovery_code == account["mfa_recovery_code"]: db.authenticators.delete_many({"user": account["_id"]}) + + new_recovery_code = secrets.token_hex(5) db.usersv0.update_one({"_id": account["_id"]}, {"$set": { - "mfa_recovery_code": secrets.token_hex(5) + "mfa_recovery_code": new_recovery_code }}) - app.supporter.create_post("inbox", account["_id"], "All multi-factor authenticators have been removed from your account by someone who used your multi-factor authentication recovery code. If this wasn't you, please secure your account immediately.") + security.log_security_action("mfa_recovery_used", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(data.mfa_recovery_code.encode()).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) else: security.ratelimit(f"login:u:{account['_id']}", 5, 60) + security.log_security_action("auth_fail", account["_id"], { + "status": "invalid_recovery_code", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) abort(401) else: mfa_methods = set() for authenticator in authenticators: mfa_methods.add(authenticator["type"]) + security.log_security_action("auth_fail", account["_id"], { + "status": "mfa_required", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return { "error": True, "type": "mfaRequired", "mfa_methods": list(mfa_methods) }, 401 - # Return account and token + # Create session + session = AccSession.create(account["_id"], request.ip, request.headers.get("User-Agent")) + + # Return session and account details return { "error": False, - "account": security.get_account(account['_id'], True), - "token": security.create_user_token(account['_id'], request.ip) + "session": session.v0, + "token": session.token, + "account": security.get_account(account['_id'], True) }, 200 + @auth_bp.post("/register") @validate_request(AuthRequest) async def register(data: AuthRequest): @@ -156,9 +225,47 @@ async def register(data: AuthRequest): # Ratelimit security.ratelimit(f"register:{request.ip}:s", 5, 900) - # Return account and token + # Create session + session = AccSession.create(data.username, request.ip, request.headers.get("User-Agent")) + + # Return session and account details return { "error": False, - "account": security.get_account(data.username, True), - "token": security.create_user_token(data.username, request.ip) + "session": session.v0, + "token": session.token, + "account": security.get_account(data.username, True) }, 200 + + +@auth_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Check ratelimits + if security.ratelimited(f"recover:{request.ip}"): + abort(429) + security.ratelimit(f"recover:{request.ip}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Get account + account = db.usersv0.find_one({"email": data.email}, projection={"_id": 1, "email": 1, "flags": 1}) + if not account: + return {"error": False}, 200 + + # Create recovery email ticket + ticket = EmailTicket(data.email, account["_id"], "recover", expires_at=int(time.time())+1800) + + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("recover", account["_id"], account["email"], {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["recover"], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 diff --git a/rest_api/v0/emails.py b/rest_api/v0/emails.py new file mode 100644 index 0000000..7d8365c --- /dev/null +++ b/rest_api/v0/emails.py @@ -0,0 +1,187 @@ +from quart import Blueprint, current_app as app, request, abort +from quart_schema import validate_request +from pydantic import BaseModel, Field +from hashlib import sha256 +from base64 import urlsafe_b64encode +import time, secrets + +from sessions import EmailTicket, AccSession +from database import db, rdb +import security, errors + + +emails_bp = Blueprint("emails_bp", __name__, url_prefix="/emails") + + +class VerifyEmailBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + +class RecoverAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + password: str = Field(min_length=8, max_length=72) + +class LockAccountBody(BaseModel): + token: str = Field(min_length=1, max_length=1024) + + +@emails_bp.post("/verify") +@validate_request(VerifyEmailBody) +async def verify_email(data: VerifyEmailBody): + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "verify": + abort(401) + + # Make sure the email address matches the user's pending email address + pending_email = rdb.get(f"pe{ticket.username}") + if (not pending_email) or (pending_email.decode() != ticket.email_address): + abort(401) + rdb.delete(f"pe{ticket.username}") + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1 + }) + + # Log action + security.log_security_action("email_changed", account["_id"], { + "old_email_hash": security.get_normalized_email_hash(account["email"]) if account.get("email") else None, + "new_email_hash": security.get_normalized_email_hash(ticket.email_address), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Update user's email address + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address) + }}) + app.cl.send_event("update_config", {"email": ticket.email_address}, usernames=[account["_id"]]) + + return {"error": False}, 200 + + +@emails_bp.post("/recover") +@validate_request(RecoverAccountBody) +async def recover_account(data: RecoverAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "recover": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "email": 1, + "pswd": 1, + "flags": 1 + }) + + # Make sure the ticket email matches the user's current email + if ticket.email_address != account["email"]: + abort(401) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Update password (and remove locked flag) + new_hash = security.hash_password(data.password) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "pswd": new_hash, + "flags": account["flags"] ^ security.UserFlags.LOCKED + }}) + + # Log action + security.log_security_action("password_changed", account["_id"], { + "method": "email", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 + + +@emails_bp.post("/lockdown") +@validate_request(LockAccountBody) +async def lock_account(data: LockAccountBody): + # Make sure ticket hasn't already been used + if rdb.exists(urlsafe_b64encode(sha256(data.token.encode()).digest()).decode()): + abort(401) + + # Get ticket + try: + ticket = EmailTicket.get_by_token(data.token) + except errors.InvalidTokenSignature|errors.EmailTicketExpired: + abort(401) + + # Validate action + if ticket.action != "lockdown": + abort(401) + + # Get account + account = db.usersv0.find_one({"_id": ticket.username}, projection={ + "_id": 1, + "flags": 1 + }) + + # Revoke ticket + rdb.set( + urlsafe_b64encode(sha256(data.token.encode()).digest()).decode(), + "", + ex=(ticket.expires_at-int(time.time()))+1 + ) + + # Make sure the account hasn't already been locked in the last 24 hours (lockdown tickets last for 24 hours) + # This is to stop multiple identity/credential rotations by an attacker to keep access via lockdown tickets. + if security.ratelimited(f"lock:{account['_id']}"): + abort(429) + security.ratelimit(f"lock:{account['_id']}", 1, 86400) + + # Update account + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": ticket.email_address, + "normalized_email_hash": security.get_normalized_email_hash(ticket.email_address), + "flags": account["flags"] | security.UserFlags.LOCKED, + "mfa_recovery_code": secrets.token_hex(5) + }}) + + # Remove authenticators + db.authenticators.delete_many({"user": account["_id"]}) + + # Log event + security.log_security_action("locked", account["_id"], { + "method": "email", + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Revoke sessions + for session in AccSession.get_all(account["_id"]): + session.revoke() + + return {"error": False}, 200 diff --git a/rest_api/v0/me.py b/rest_api/v0/me.py index b98ac62..8e7b0e9 100644 --- a/rest_api/v0/me.py +++ b/rest_api/v0/me.py @@ -3,8 +3,9 @@ from pydantic import BaseModel, Field from typing import Optional, List, Literal from copy import copy -from base64 import b64encode -from io import BytesIO +from base64 import urlsafe_b64encode +from hashlib import sha256 +from threading import Thread import pymongo import uuid import time @@ -12,10 +13,13 @@ import qrcode, qrcode.image.svg import uuid import secrets +import os +import requests import security from database import db, rdb, get_total_pages from uploads import claim_file, delete_file +from sessions import AccSession, EmailTicket from utils import log @@ -45,6 +49,14 @@ class Config: validate_assignment = True str_strip_whitespace = True +class UpdateEmailBody(BaseModel): + password: str = Field(min_length=1, max_length=255) # change in API v1 + email: str = Field(max_length=255, pattern=security.EMAIL_REGEX) + captcha: Optional[str] = Field(default="", max_length=2000) + +class RemoveEmailBody(BaseModel): + password: str = Field(min_length=1, max_length=255) # change in API v1 + class ChangePasswordBody(BaseModel): old: str = Field(min_length=1, max_length=255) # change in API v1 new: str = Field(min_length=8, max_length=72) @@ -101,13 +113,12 @@ async def delete_account(data: DeleteAccountBody): # Schedule account for deletion db.usersv0.update_one({"_id": request.user}, {"$set": { - "tokens": [], "delete_after": int(time.time())+604800 # 7 days }}) - # Disconnect clients - for client in app.cl.usernames.get(request.user, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(request.user): + session.revoke() return {"error": False}, 200 @@ -200,6 +211,93 @@ async def get_relationships(): }, 200 +@me_bp.patch("/email") +@validate_request(UpdateEmailBody) +async def update_email(data: UpdateEmailBody): + # Make sure email is enabled + if not os.getenv("EMAIL_SMTP_HOST"): + return {"error": True, "type": "featureDisabled"}, 503 + + # Check authorization + if not request.user: + abort(401) + + # Check ratelimits + if security.ratelimited(f"login:u:{request.user}") or security.ratelimited(f"emailch:{request.user}"): + abort(429) + + # Check password + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + if not security.check_password_hash(data.password, account["pswd"]): + security.ratelimit(f"login:u:{request.user}", 5, 60) + return {"error": True, "type": "invalidCredentials"}, 401 + + # Make sure the email address hasn't been used before + if db.usersv0.count_documents({"normalized_email_hash": security.get_normalized_email_hash(data.email)}, limit=1): + return {"error": True, "type": "emailExists"}, 409 + + # Ratelimit + security.ratelimit(f"emailch:{request.user}", 3, 2700) + + # Check captcha + if os.getenv("CAPTCHA_SECRET") and not (hasattr(request, "bypass_captcha") and request.bypass_captcha): + if not requests.post("https://api.hcaptcha.com/siteverify", data={ + "secret": os.getenv("CAPTCHA_SECRET"), + "response": data.captcha, + }).json()["success"]: + return {"error": True, "type": "invalidCaptcha"}, 403 + + # Create email verification ticket + ticket = EmailTicket(data.email, request.user, "verify", expires_at=int(time.time())+1800) + + # Set pending email address + rdb.set(f"pe{request.user}", data.email, ex=1800) + + # Send email + txt_tmpl, html_tmpl = security.render_email_tmpl("verify", request.user, data.email, {"token": ticket.token}) + Thread( + target=security.send_email, + args=[security.EMAIL_SUBJECTS["verify"], request.user, data.email, txt_tmpl, html_tmpl] + ).start() + + return {"error": False}, 200 + + +@me_bp.delete("/email") +@validate_request(RemoveEmailBody) +async def remove_email(data: RemoveEmailBody): + # Check authorization + if not request.user: + abort(401) + + # Check ratelimits + if security.ratelimited(f"login:u:{request.user}"): + abort(429) + + # Check password + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) + if not security.check_password_hash(data.password, account["pswd"]): + security.ratelimit(f"login:u:{request.user}", 5, 60) + return {"error": True, "type": "invalidCredentials"}, 401 + + # Log action + security.log_security_action("email_changed", account["_id"], { + "old_email_hash": security.get_normalized_email_hash(account["email"]) if account.get("email") else None, + "new_email_hash": None, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) + + # Update user's email address + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "email": "", + "normalized_email_hash": "" + }}) + app.cl.send_event("update_config", {"email": ""}, usernames=[account["_id"]]) + + return {"error": False}, 200 + + @me_bp.patch("/password") @validate_request(ChangePasswordBody) async def change_password(data: ChangePasswordBody): @@ -212,16 +310,23 @@ async def change_password(data: ChangePasswordBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.old, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Update password - db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": security.hash_password(data.new)}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your account password has been changed. If this wasn't requested by you, please secure your account immediately.") + new_hash = security.hash_password(data.new) + db.usersv0.update_one({"_id": request.user}, {"$set": {"pswd": new_hash}}) + + # Log event + security.log_security_action("password_changed", account["_id"], { + "method": "self", + "old_pswd_hash": account["pswd"], + "new_pswd_hash": new_hash, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False}, 200 @@ -260,7 +365,7 @@ async def add_authenticator(data: AddAuthenticatorBody): abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -276,8 +381,12 @@ async def add_authenticator(data: AddAuthenticatorBody): } db.authenticators.insert_one(authenticator) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been added to your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_added", account["_id"], { + "authenticator_id": authenticator["_id"], + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) # Return authenticator and MFA recovery code del authenticator["user"] @@ -334,7 +443,7 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB abort(429) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"email": 1, "pswd": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 @@ -347,8 +456,12 @@ async def remove_authenticator(authenticator_id: str, data: RemoveAuthenticatorB if result.deleted_count < 1: abort(404) - # Send alert - app.supporter.create_post("inbox", account["_id"], "A multi-factor authenticator has been removed from your account. If this wasn't requested by you, please secure your account immediately.") + # Log action + security.log_security_action("mfa_removed", account["_id"], { + "authenticator_id": authenticator_id, + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) return {"error": False} @@ -382,19 +495,24 @@ async def reset_mfa_recovery_code(data: ResetMFARecoveryCodeBody): abort(401) # Check password - account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1}) + account = db.usersv0.find_one({"_id": request.user}, projection={"pswd": 1, "mfa_recovery_code": 1}) if not security.check_password_hash(data.password, account["pswd"]): security.ratelimit(f"login:u:{request.user}", 5, 60) return {"error": True, "type": "invalidCredentials"}, 401 # Reset MFA recovery code - mfa_recovery_code = secrets.token_hex(5) - db.usersv0.update_one({"_id": request.user}, {"$set": {"mfa_recovery_code": mfa_recovery_code}}) - - # Send alert - app.supporter.create_post("inbox", account["_id"], "Your multi-factor authentication recovery code has been reset. If this wasn't requested by you, please secure your account immediately.") + new_recovery_code = secrets.token_hex(5) + db.usersv0.update_one({"_id": account["_id"]}, {"$set": { + "mfa_recovery_code": new_recovery_code + }}) + security.log_security_action("mfa_recovery_reset", account["_id"], { + "old_recovery_code_hash": urlsafe_b64encode(sha256(account["mfa_recovery_code"]).digest()).decode(), + "new_recovery_code_hash": urlsafe_b64encode(sha256(new_recovery_code.encode()).digest()).decode(), + "ip": request.ip, + "user_agent": request.headers.get("User-Agent") + }) - return {"error": False, "mfa_recovery_code": mfa_recovery_code} + return {"error": False, "mfa_recovery_code": new_recovery_code} @me_bp.delete("/tokens") @@ -403,12 +521,9 @@ async def delete_tokens(): if not request.user: abort(401) - # Revoke tokens - db.usersv0.update_one({"_id": request.user}, {"$set": {"tokens": []}}) - - # Disconnect clients - for client in app.cl.usernames.get(request.user, []): - client.kick() + # Revoke sessions + for session in AccSession.get_all(request.user): + session.revoke() return {"error": False}, 200 diff --git a/security.py b/security.py index ccea8a8..67cdac5 100644 --- a/security.py +++ b/security.py @@ -1,10 +1,16 @@ +from typing import Optional, Any, Literal from hashlib import sha256 -from typing import Optional -import time, requests, os, uuid, secrets, bcrypt, msgpack - -from database import db, rdb +from base64 import urlsafe_b64encode, urlsafe_b64decode +from threading import Thread +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formataddr +import time, requests, os, uuid, secrets, bcrypt, hmac, msgpack, jinja2, smtplib, re + +from database import db, rdb, signing_keys from utils import log from uploads import clear_files +import errors """ Meower Security Module @@ -12,9 +18,9 @@ """ SENSITIVE_ACCOUNT_FIELDS = { + "normalized_email_hash", "pswd", "mfa_recovery_code", - "tokens", "delete_after" } @@ -22,6 +28,9 @@ for key in SENSITIVE_ACCOUNT_FIELDS: SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION[key] = 0 +SYSTEM_USER_USERNAMES = {"server", "deleted", "meower", "admin", "username"} +SYSTEM_USER = {} + DEFAULT_USER_SETTINGS = { "unread_inbox": True, "theme": "orange", @@ -38,16 +47,38 @@ USERNAME_REGEX = "[a-zA-Z0-9-_]{1,20}" +# I hate this. But, thanks https://stackoverflow.com/a/201378 +EMAIL_REGEX = r"""(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])""" TOTP_REGEX = "[0-9]{6}" BCRYPT_SALT_ROUNDS = 14 TOKEN_BYTES = 64 +TOKEN_TYPES = Literal[ + "acc", # account authorization + "email", # email actions (such as email verification or account recovery) +] + + +EMAIL_SUBJECTS = { + "verify": "Verify your email address", + "recover": "Reset your password", + "security_alert": "Security alert", + "locked": "Your account has been locked" +} + + +email_file_loader = jinja2.FileSystemLoader("email_templates") +email_env = jinja2.Environment(loader=email_file_loader) + + class UserFlags: SYSTEM = 1 DELETED = 2 PROTECTED = 4 POST_RATELIMIT_BYPASS = 8 + REQUIRE_EMAIL = 16 # not used yet + LOCKED = 32 class AdminPermissions: @@ -119,6 +150,9 @@ def account_exists(username, ignore_case=False): log(f"Error on account_exists: Expected str for username, got {type(username)}") return False + if (ignore_case or username == username.lower()) and username.lower() in SYSTEM_USER_USERNAMES: + return True + query = ({"lower_username": username.lower()} if ignore_case else {"_id": username}) return (db.usersv0.count_documents(query, limit=1) > 0) @@ -134,9 +168,10 @@ def create_account(username: str, password: str, ip: str): "avatar": "", "avatar_color": "000000", "quote": "", + "email": "", + "normalized_email_hash": "", "pswd": hash_password(password), "mfa_recovery_code": secrets.token_hex(5), - "tokens": [], "flags": 0, "permissions": 0, "ban": { @@ -158,7 +193,7 @@ def create_account(username: str, password: str, ip: str): })) # Automatically report if VPN is detected - if get_netinfo(ip)["vpn"]: + if get_ip_info(ip)["vpn"]: db.reports.insert_one({ "_id": str(uuid.uuid4()), "type": "user", @@ -168,7 +203,7 @@ def create_account(username: str, password: str, ip: str): "reports": [{ "user": "Server", "ip": ip, - "reason": "User registered while using a VPN.", + "reason": f"User registered while using a VPN ({ip}).", "comment": "", "time": int(time()) }] @@ -181,6 +216,24 @@ def get_account(username, include_config=False): log(f"Error on get_account: Expected str for username, got {type(username)}") return None + # System users + if username.lower() in SYSTEM_USER_USERNAMES: + return { + "_id": username.title(), + "lower_username": username.lower(), + "uuid": None, + "created": None, + "pfp_data": None, + "avatar": None, + "avatar_color": None, + "quote": None, + "email": None, + "flags": 1, + "permissions": None, + "ban": None, + "last_seen": None + } + # Get account account = db.usersv0.find_one({"lower_username": username.lower()}, projection=SENSITIVE_ACCOUNT_FIELDS_DB_PROJECTION) if not account: @@ -211,52 +264,41 @@ def get_account(username, include_config=False): del user_settings["_id"] account.update(user_settings) else: - # Remove ban if not including config + # Remove email and ban if not including config + del account["email"] del account["ban"] return account -def create_user_token(username: str, ip: str, used_token: Optional[str] = None) -> str: - # Get required account details - account = db.usersv0.find_one({"_id": username}, projection={ - "_id": 1, - "tokens": 1, - "delete_after": 1 - }) +def create_token(ttype: TOKEN_TYPES, claims: Any) -> str: + # Encode claims + encoded_claims = msgpack.packb(claims) - # Update netlog - db.netlog.update_one({"_id": { - "ip": ip, - "user": username, - }}, {"$set": {"last_used": int(time.time())}}, upsert=True) + # Sign encoded claims + signature = hmac.digest(signing_keys[ttype], encoded_claims, digest=sha256) - # Restore account - if account["delete_after"]: - db.usersv0.update_one({"_id": account["_id"]}, {"$set": {"delete_after": None}}) - rdb.publish("admin", msgpack.packb({ - "op": "alert_user", - "user": account["_id"], - "content": "Your account was scheduled for deletion but you logged back in. Your account is no longer scheduled for deletion! If you didn't request for your account to be deleted, please change your password immediately." - })) - - # Return original token if one is specified - # This is a temporary fix for people being randomly logged out - if used_token: - return used_token - - # Generate new token, revoke used token, and update last seen timestamp - new_token = secrets.token_urlsafe(TOKEN_BYTES) - account["tokens"].append(new_token) - if used_token in account["tokens"]: - account["tokens"].remove(used_token) - db.usersv0.update_one({"_id": account["_id"]}, {"$set": { - "tokens": account["tokens"], - "last_seen": int(time.time()) - }}) + # Construct token + token = b".".join([urlsafe_b64encode(encoded_claims), urlsafe_b64encode(signature)]) + + return token.decode() + + +def extract_token(token: str, expected_type: TOKEN_TYPES) -> Optional[Any]: + # Extract data and signature + encoded_claims, signature = token.split(".") + encoded_claims = urlsafe_b64decode(encoded_claims) + signature = urlsafe_b64decode(signature) + + # Check signature + expected_signature = hmac.digest(signing_keys[expected_type], encoded_claims, digest=sha256) + if not hmac.compare_digest(signature, expected_signature): + raise errors.InvalidTokenSignature - # Return new token - return new_token + # Decode claims + claims = msgpack.unpackb(encoded_claims) + + return claims def update_settings(username, newdata): @@ -367,9 +409,10 @@ def delete_account(username, purge=False): "avatar": None, "avatar_color": None, "quote": None, + "email": None, + "normalized_email_hash": None, "pswd": None, "mfa_recovery_code": None, - "tokens": None, "flags": account["flags"], "permissions": None, "ban": None, @@ -377,18 +420,24 @@ def delete_account(username, purge=False): "delete_after": None }}) + # Delete pending email + rdb.delete(f"pe{username}") + # Delete authenticators db.authenticators.delete_many({"user": username}) + # Delete sessions + db.acc_sessions.delete_many({"user": username}) + + # Delete security logs + db.security_log.delete_many({"user": username}) + # Delete uploaded files clear_files(username) # Delete user settings db.user_settings.delete_one({"_id": username}) - # Delete netlogs - db.netlog.delete_many({"_id.user": username}) - # Remove from reports db.reports.update_many({"reports.user": username}, {"$pull": { "reports": {"user": username} @@ -426,62 +475,100 @@ def delete_account(username, purge=False): db.usersv0.delete_one({"_id": username}) -def get_netinfo(ip_address): - """ - Get IP info from IP-API. - - Returns: - ```json - { - "_id": str, - "country_code": str, - "country_name": str, - "asn": int, - "isp": str, - "vpn": bool +def get_ip_info(ip_address): + # Get IP hash + ip_hash = urlsafe_b64encode(sha256(ip_address.encode()).digest()).decode() + + # Get from cache + ip_info = rdb.get(f"ip{ip_hash}") + if ip_info: + return msgpack.unpackb(ip_info) + + # Get from IP-API + resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") + if resp.ok and resp.json()["status"] == "success": + resp_json = resp.json() + ip_info = { + "country_code": resp_json["countryCode"], + "country_name": resp_json["country"], + "region": resp_json["regionName"], + "city": resp_json["city"], + "timezone": resp_json["timezone"], + "currency": resp_json["currency"], + "as": resp_json["as"], + "isp": resp_json["isp"], + "vpn": (resp_json.get("hosting") or resp_json.get("proxy")) + } + rdb.set(f"ip{ip_hash}", msgpack.packb(ip_info), ex=int(time.time())+(86400*21)) # cache for 3 weeks + return ip_info + + # Fallback + return { + "country_code": "Unknown", + "country_name": "Unknown", + "region": "Unknown", + "city": "Unknown", + "timezone": "Unknown", + "currency": "Unknown", + "as": "Unknown", + "isp": "Unknown", + "vpn": False } - ``` - """ - # Get IP hash - ip_hash = sha256(ip_address.encode()).hexdigest() - - # Get from database or IP-API if not cached - netinfo = db.netinfo.find_one({"_id": ip_hash}) - if not netinfo: - resp = requests.get(f"http://ip-api.com/json/{ip_address}?fields=25349915") - if resp.ok: - resp_json = resp.json() - netinfo = { - "_id": ip_hash, - "country_code": resp_json["countryCode"], - "country_name": resp_json["country"], - "region": resp_json["regionName"], - "city": resp_json["city"], - "timezone": resp_json["timezone"], - "currency": resp_json["currency"], - "as": resp_json["as"], - "isp": resp_json["isp"], - "vpn": (resp_json.get("hosting") or resp_json.get("proxy")), - "last_refreshed": int(time.time()) - } - db.netinfo.update_one({"_id": ip_hash}, {"$set": netinfo}, upsert=True) - else: - netinfo = { - "_id": ip_hash, - "country_code": "Unknown", - "country_name": "Unknown", - "region": "Unknown", - "city": "Unknown", - "timezone": "Unknown", - "currency": "Unknown", - "as": "Unknown", - "isp": "Unknown", - "vpn": False, - "last_refreshed": int(time.time()) - } - - return netinfo + +def log_security_action(action_type: str, user: str, data: dict): + db.security_log.insert_one({ + "_id": str(uuid.uuid4()), + "type": action_type, + "user": user, + "time": int(time.time()), + "data": data + }) + + if action_type in { + "email_changed", + "password_changed", + "mfa_added", + "mfa_removed", + "mfa_recovery_reset", + "mfa_recovery_used", + "locked" + }: + tmpl_name = "locked" if action_type == "locked" else "security_alert" + platform_name = os.environ["EMAIL_PLATFORM_NAME"] + + account = db.usersv0.find_one({"_id": user}, projection={"_id": 1, "email": 1}) + + txt_tmpl, html_tmpl = render_email_tmpl(tmpl_name, account["_id"], account.get("email", ""), { + "msg": { + "email_changed": f"The email address on your {platform_name} account has been changed.", + "password_changed": f"The password on your {platform_name} account has been changed.", + "mfa_added": f"A multi-factor authenticator has been added to your {platform_name} account.", + "mfa_removed": f"A multi-factor authenticator has been removed from your {platform_name} account.", + "mfa_recovery_reset": f"The multi-factor authentication recovery code on your {platform_name} account has been reset.", + "mfa_recovery_used": f"Your multi-factor authentication recovery code has been used to reset multi-factor authentication on your {platform_name} account." + }[action_type] if action_type != "locked" else None, + "token": create_token("email", [ # this doesn't use EmailTicket in sessions.py because it'd be a recursive import + account["email"], + account["_id"], + "lockdown", + int(time.time())+86400 + ]) if account.get("email") and action_type != "locked" else None + }) + + # Email + if account.get('email'): + Thread( + target=send_email, + args=[EMAIL_SUBJECTS[tmpl_name], account["_id"], account["email"], txt_tmpl, html_tmpl] + ).start() + + # Inbox + rdb.publish("admin", msgpack.packb({ + "op": "alert_user", + "user": account["_id"], + "content": txt_tmpl + })) def add_audit_log(action_type, mod_username, mod_ip, data): @@ -508,11 +595,8 @@ def background_tasks_loop(): except Exception as e: log(f"Failed to delete account {user['_id']}: {e}") - # Purge old netinfo - db.netinfo.delete_many({"last_refreshed": {"$lt": int(time.time())-2419200}}) - - # Purge old netlogs - db.netlog.delete_many({"last_used": {"$lt": int(time.time())-2419200}}) + # Revoke inactive sessions (3 weeks of inactivity) + db.acc_sessions.delete_many({"refreshed_at": {"$lt": int(time.time())-(86400*21)}}) # Purge old deleted posts db.posts.delete_many({"deleted_at": {"$lt": int(time.time())-2419200}}) @@ -520,7 +604,8 @@ def background_tasks_loop(): # Purge old post revisions db.post_revisions.delete_many({"time": {"$lt": int(time.time())-2419200}}) - # Purge old admin audit logs + """ we should probably not be getting rid of audit logs... + # Purge old "get" admin audit logs db.audit_log.delete_many({ "time": {"$lt": int(time.time())-2419200}, "type": {"$in": [ @@ -537,6 +622,7 @@ def background_tasks_loop(): "got_announcements" ]} }) + """ log("Finished background tasks!") @@ -547,3 +633,46 @@ def hash_password(password: str) -> str: def check_password_hash(password: str, hashed_password: str) -> bool: return bcrypt.checkpw(password.encode(), hashed_password.encode()) + + +def get_normalized_email_hash(address: str) -> str: + """ + Get a hash of an email address with aliases and dots stripped. + This is to allow using address aliases, but to still detect ban evasion. + Also, Gmail ignores dots in addresses. Thanks Google. + """ + + identifier, domain = address.split("@") + identifier = re.split(r'\+|\%', identifier)[0] + identifier = identifier.replace(".", "") + + return urlsafe_b64encode(sha256(f"{identifier}@{domain}".encode()).digest()).decode() + + +def render_email_tmpl(template: str, to_name: str, to_address: str, data: Optional[dict[str, str]] = {}) -> tuple[str, str]: + data.update({ + "subject": EMAIL_SUBJECTS[template], + "name": to_name, + "address": to_address, + "env": os.environ + }) + + txt_tmpl = email_env.get_template(f"{template}.txt") + html_tmpl = email_env.get_template(f"{template}.html") + + return txt_tmpl.render(data), html_tmpl.render(data) + + +def send_email(subject: str, to_name: str, to_address: str, txt_tmpl: str, html_tmpl: str): + message = MIMEMultipart("alternative") + message["From"] = formataddr((os.environ["EMAIL_FROM_NAME"], os.environ["EMAIL_FROM_ADDRESS"])) + message["To"] = formataddr((to_name, to_address)) + message["Subject"] = subject + message.attach(MIMEText(txt_tmpl, "plain")) + message.attach(MIMEText(html_tmpl, "html")) + + with smtplib.SMTP(os.environ["EMAIL_SMTP_HOST"], int(os.environ["EMAIL_SMTP_PORT"])) as server: + if os.getenv("EMAIL_SMTP_TLS"): + server.starttls() + server.login(os.environ["EMAIL_SMTP_USERNAME"], os.environ["EMAIL_SMTP_PASSWORD"]) + server.sendmail(os.environ["EMAIL_FROM_ADDRESS"], to_address, message.as_string()) diff --git a/sessions.py b/sessions.py new file mode 100644 index 0000000..a5e9b5d --- /dev/null +++ b/sessions.py @@ -0,0 +1,186 @@ +from typing import Optional, TypedDict, Literal +import uuid, time, msgpack, pymongo + +from database import db, rdb +import security, errors + + +class AccSessionDB(TypedDict): + _id: str + user: str + ip: str + user_agent: str + created_at: int + refreshed_at: int + + +class AccSessionV0(TypedDict): + _id: str + ip: str + location: str + user_agent: str + created_at: int + refreshed_at: int + + +class AccSession: + def __init__(self, data: AccSessionDB): + self._db = data + + @classmethod + def create(cls: "AccSession", user: str, ip: str, user_agent: str) -> "AccSession": + # restore account if it is pending deletion + result = db.usersv0.update_one({"_id": user}, {"$set": {"delete_after": None}}) + if result.modified_count: + rdb.publish("admin", msgpack.packb({ + "op": "alert_user", + "user": user, + "content": "Your account was scheduled for deletion but you logged back in. Your account is no longer scheduled for deletion! If you didn't request for your account to be deleted, please change your password immediately." + })) + + data: AccSessionDB = { + "_id": str(uuid.uuid4()), + "user": user, + "ip": ip, + "user_agent": user_agent, + "created_at": int(time.time()), + "refreshed_at": int(time.time()) + } + db.acc_sessions.insert_one(data) + + security.log_security_action("session_create", user, { + "session_id": data["_id"], + "ip": ip, + "user_agent": user_agent + }) + + return cls(data) + + @classmethod + def get_by_id(cls: "AccSession", session_id: str) -> "AccSession": + data: Optional[AccSessionDB] = db.acc_sessions.find_one({"_id": session_id}) + if not data: + raise errors.AccSessionNotFound + + return cls(data) + + @classmethod + def get_by_token(cls: "AccSession", token: str) -> "AccSession": + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired + return cls.get_by_id(session_id) + + @classmethod + def get_username_by_token(cls: "AccSession", token: str) -> str: + session_id, _, expires_at = security.extract_token(token, "acc") + if expires_at < int(time.time()): + raise errors.AccSessionTokenExpired + username = rdb.get(session_id) + if username: + return username.decode() + else: + session = cls.get_by_id(session_id) + username = session.username + rdb.set(session_id, username, ex=300) + return username + + @classmethod + def get_all(cls: "AccSession", user: str) -> list["AccSession"]: + return [ + cls(data) + for data in db.acc_sessions.find( + {"user": user}, + sort=[("refreshed_at", pymongo.DESCENDING)] + ) + ] + + @property + def id(self) -> str: + return self._db["_id"] + + @property + def token(self) -> str: + return security.create_token("acc", [ + self._db["_id"], + self._db["refreshed_at"], + self._db["refreshed_at"]+(86400*21) # expire token after 3 weeks + ]) + + @property + def username(self): + return self._db["user"] + + @property + def v0(self) -> AccSessionV0: + return { + "_id": self._db["_id"], + "ip": self._db["ip"], + "location": "", + "user_agent": self._db["user_agent"], + "created_at": self._db["created_at"], + "refreshed_at": self._db["refreshed_at"] + } + + def refresh(self, ip: str, user_agent: str, check_token: Optional[str] = None): + if check_token: + # token re-use prevention + _, refreshed_at, _ = security.extract_token(check_token, "acc") + if refreshed_at != self._db["refreshed_at"]: + return self.revoke() + + self._db.update({ + "ip": ip, + "user_agent": user_agent, + "refreshed_at": int(time.time()) + }) + db.acc_sessions.update_one({"_id": self._db["_id"]}, {"$set": self._db}) + + security.log_security_action("session_refresh", self._db["user"], { + "session_id": self._db["_id"], + "ip": ip, + "user_agent": user_agent + }) + + def revoke(self): + db.acc_sessions.delete_one({"_id": self._db["_id"]}) + rdb.delete(f"u{self._db['_id']}") + rdb.publish("admin", msgpack.packb({ + "op": "revoke_acc_session", + "user": self._db["user"], + "sid": self._db["_id"] + })) + + security.log_security_action("session_revoke", self._db["user"], { + "session_id": self._db["_id"] + }) + + +class EmailTicket: + def __init__( + self, + email_address: str, + username: str, + action: Literal["verify", "recover", "lockdown"], + expires_at: int + ): + self.email_address = email_address + self.username = username + self.action = action + self.expires_at = expires_at + + if self.expires_at < int(time.time()): + raise errors.EmailTicketExpired + + @classmethod + def get_by_token(cls: "EmailTicket", token: str) -> "EmailTicket": + return cls(*security.extract_token(token, "email")) + + @property + def token(self) -> str: + return security.create_token("email", [ + self.email_address, + self.username, + self.action, + self.expires_at + ]) diff --git a/supporter.py b/supporter.py index bd220fa..d79d713 100644 --- a/supporter.py +++ b/supporter.py @@ -1,6 +1,6 @@ from threading import Thread from typing import Optional, Iterable, Any -import uuid, time, msgpack, pymongo, re, copy +import uuid, time, msgpack, pymongo, re, copy, asyncio from cloudlink import CloudlinkServer from database import db, rdb @@ -128,8 +128,13 @@ def listen_for_admin_pubsub(self): pubsub.subscribe("admin") for msg in pubsub.listen(): try: - msg = msgpack.loads(msg["data"]) + msg = msgpack.unpackb(msg["data"]) match msg.pop("op"): + case "revoke_acc_session": + for c in self.cl.usernames.get(msg["user"], []): + if "sid" in msg and msg["sid"] != c.acc_session_id: + continue + asyncio.run(c.kick()) case "alert_user": self.create_post("inbox", msg["user"], msg["content"]) case "ban_user": @@ -163,7 +168,7 @@ def listen_for_admin_pubsub(self): # Logout user (can't kick because of async stuff) for c in self.cl.usernames.get(username, []): - c.logout() + asyncio.run(c.kick()) except: continue diff --git a/utils.py b/utils.py index 1dacd93..b359b6b 100644 --- a/utils.py +++ b/utils.py @@ -1,6 +1,4 @@ -from typing import Literal from datetime import datetime -import time import sys import traceback