intset

inbox/crispin.py

@@ -26,6 +26,7 @@
 import imapclient.imap_utf7
 import imapclient.imapclient
 import imapclient.response_parser
+import intset
 
 from inbox import interruptible_threading
 from inbox.constants import MAX_MESSAGE_BODY_LENGTH
@@ -1064,7 +1065,7 @@ def fetch_headers(self, uids: Iterable[int]) -> Dict[int, Dict[bytes, Any]]:
 
     def find_by_header(self, header_name, header_value):
         """Find all uids in the selected folder with the given header value."""
-        all_uids = self.all_uids()
+        all_uids = intset.IntSet(self.all_uids())
         # It would be nice to just search by header too, but some backends
         # don't support that, at least not if you want to search by X-INBOX-ID
         # header. So fetch the header for each draft and see if we

inbox/mailsync/backends/gmail.py

@@ -25,6 +25,7 @@
 from threading import Semaphore
 from typing import TYPE_CHECKING, Dict, List
 
+import intset
 from sqlalchemy.orm import joinedload, load_only
 
 from inbox import interruptible_threading
@@ -250,7 +251,7 @@ def initial_sync_impl(self, crispin_client: "CrispinClient") -> None:
 
         try:
             with self.global_lock:
-                remote_uids = set(crispin_client.all_uids())
+                remote_uids = intset.IntSet(crispin_client.all_uids())
                 with self.syncmanager_lock:
                     with session_scope(self.namespace_id) as db_session:
                         local_uids = common.local_uids(
@@ -280,15 +281,15 @@ def initial_sync_impl(self, crispin_client: "CrispinClient") -> None:
                 if self.is_all_mail(crispin_client):
                     # Prioritize UIDs for messages in the inbox folder.
                     if len_remote_uids < 1e6:
-                        inbox_uids = set(
+                        inbox_uids = intset.IntSet(
                             crispin_client.search_uids(["X-GM-LABELS", "inbox"])
                         )
                     else:
                         # The search above is really slow (times out) on really
                         # large mailboxes, so bound the search to messages within
                         # the past month in order to get anywhere.
                         since = datetime.utcnow() - timedelta(days=30)
-                        inbox_uids = set(
+                        inbox_uids = intset.IntSet(
                             crispin_client.search_uids(
                                 ["X-GM-LABELS", "inbox", "SINCE", since]
                             )

inbox/mailsync/backends/imap/common.py

@@ -15,6 +15,7 @@
 from datetime import datetime
 from typing import List, Set
 
+import intset
 from sqlalchemy import bindparam, desc
 from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound
@@ -58,7 +59,7 @@ def local_uids(
     # We're using a raw DB-API cursor here to avoid the overhead of the ORM.
     db_api_cursor = get_db_api_cursor_with_query(session, q)
 
-    return {uid for uid, in db_api_cursor.fetchall()}
+    return intset.IntSet(uid for uid, in db_api_cursor.fetchall())
 
 
 def lastseenuid(account_id, session, folder_id):

inbox/mailsync/backends/imap/generic.py

@@ -68,6 +68,7 @@
 from datetime import datetime, timedelta
 from typing import Any, Dict, NoReturn, Optional
 
+import intset
 from sqlalchemy import func
 from sqlalchemy.exc import IntegrityError
 from sqlalchemy.orm import Session
@@ -458,19 +459,17 @@ def initial_sync_impl(self, crispin_client: CrispinClient):
         assert crispin_client.selected_folder_name == self.folder_name
         try:
             with self.global_lock:
-                remote_uids = set(crispin_client.all_uids())
+                remote_uids = intset.IntSet(crispin_client.all_uids())
                 with self.syncmanager_lock:
                     with session_scope(self.namespace_id) as db_session:
                         local_uids = common.local_uids(
                             self.account_id, db_session, self.folder_id
                         )
-                    common.remove_deleted_uids(
-                        self.account_id,
-                        self.folder_id,
-                        local_uids.difference(remote_uids),
-                    )
+                common.remove_deleted_uids(
+                    self.account_id, self.folder_id, local_uids - remote_uids
+                )
 
-                new_uids = sorted(remote_uids.difference(local_uids), reverse=True)
+                new_uids = sorted(remote_uids - local_uids, reverse=True)
 
                 len_remote_uids = len(remote_uids)
                 del remote_uids  # free up memory as soon as possible
@@ -870,14 +869,14 @@ def condstore_refresh_flags(self, crispin_client: CrispinClient) -> None:
         del changed_flags  # free memory as soon as possible
 
         with self.global_lock:
-            remote_uids = set(crispin_client.all_uids())
+            remote_uids = intset.IntSet(crispin_client.all_uids())
 
             with session_scope(self.namespace_id) as db_session:
                 local_uids = common.local_uids(
                     self.account_id, db_session, self.folder_id
                 )
 
-            expunged_uids = local_uids.difference(remote_uids)
+            expunged_uids = local_uids - remote_uids
             del local_uids  # free memory as soon as possible
             max_remote_uid = max(remote_uids) if remote_uids else 0
             del remote_uids  # free memory as soon as possible
@@ -922,14 +921,14 @@ def refresh_flags_impl(self, crispin_client: CrispinClient, max_uids: int) -> No
 
         with self.global_lock:
             # Check for any deleted messages.
-            remote_uids = crispin_client.all_uids()
+            remote_uids = intset.IntSet(crispin_client.all_uids())
 
             with session_scope(self.namespace_id) as db_session:
                 local_uids = common.local_uids(
                     self.account_id, db_session, self.folder_id
                 )
 
-            expunged_uids = local_uids.difference(remote_uids)
+            expunged_uids = local_uids - remote_uids
             del local_uids  # free memory as soon as possible
             del remote_uids  # free memory as soon as possible
 
@@ -964,7 +963,7 @@ def refresh_flags_impl(self, crispin_client: CrispinClient, max_uids: int) -> No
         log.debug(
             "Changed flags refresh response, persisting changes", max_uids=max_uids
         )
-        expunged_uids = local_uids.difference(flags)
+        expunged_uids = local_uids - intset.IntSet(flags)
         with self.syncmanager_lock:
             common.remove_deleted_uids(self.account_id, self.folder_id, expunged_uids)
 

requirements/prod.txt

@@ -33,6 +33,7 @@ hiredis==2.3.2
 html2text==2020.1.16
 icalendar==4.0.9
 idna==3.7
+intset==1.0.0
 imapclient==2.2.0
 importlib-metadata==4.8.1
 importlib-resources==5.4.0