feat: Optimize Contacts Sync (#271)

* feat: Optimize Contacts Sync * Adding sleep for sync_contacts too * Fixing test * Fixing test
fylein · Nov 16, 2023 · bcaa7ef · bcaa7ef
1 parent d52f453
commit bcaa7ef
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 55 deletions.
diff --git a/apps/xero/utils.py b/apps/xero/utils.py
@@ -68,12 +68,20 @@ def get_suppliers(self):
         tenant_mapping = TenantMapping.objects.get(workspace_id=self.workspace_id)
         self.connection.set_tenant_id(tenant_mapping.tenant_id)
 
-        suppliers_generator = self.connection.contacts.list_all_generator()
+        params = {
+            'where': 'IsSupplier=true',
+            'includeArchived': 'false',
+        }
+
+        suppliers_generator = self.connection.contacts.list_all_generator(**params)
         merchant_names: List[str] = []
+
         for suppliers in suppliers_generator:
             for supplier in suppliers["Contacts"]:
-                if supplier["IsSupplier"] and supplier["ContactStatus"] == "ACTIVE":
-                    merchant_names.append(supplier["Name"])
+                merchant_names.append(supplier["Name"])
+
+            sleep(1)
+
         return merchant_names
 
     def get_or_create_contact(
@@ -268,8 +276,13 @@ def sync_contacts(self):
 
         updated_at = get_last_synced_at(self.workspace_id, "CONTACT")
 
+        params = {
+            'where': 'IsCustomer=false',
+            'includeArchived': 'true'
+        }
+
         contacts_generator = self.connection.contacts.list_all_generator(
-            modified_after=updated_at
+            modified_after=updated_at, **params
         )
 
         for contacts in contacts_generator:
@@ -288,12 +301,14 @@ def sync_contacts(self):
                         "value": contact["Name"],
                         "destination_id": contact["ContactID"],
                         "detail": detail,
+                        "active": True if contact["ContactStatus"] == "ACTIVE" else False,
                     }
                 )
 
             DestinationAttribute.bulk_create_or_update_destination_attributes(
                 contact_attributes, "CONTACT", self.workspace_id, True
             )
+            sleep(1)
 
         return []
 
@@ -305,61 +320,36 @@ def sync_customers(self):
 
         self.connection.set_tenant_id(tenant_mapping.tenant_id)
 
-        customers_generator = self.connection.contacts.list_all_generator()
-
-        customer_attributes = []
-
-        destination_attributes = DestinationAttribute.objects.filter(
-            workspace_id=self.workspace_id,
-            attribute_type="CUSTOMER",
-            display_name="Customer",
-        ).values("destination_id", "value", "detail")
-
-        disabled_fields_map = {}
+        params = {
+            'where': 'IsCustomer=true',
+            'includeArchived': 'true',
+        }
 
-        for destination_attribute in destination_attributes:
-            disabled_fields_map[destination_attribute["destination_id"]] = {
-                "value": destination_attribute["value"],
-                "detail": destination_attribute["detail"],
-            }
+        customers_generator = self.connection.contacts.list_all_generator(**params)
 
         for customers in customers_generator:
-            for customer in customers["Contacts"]:
-                if customer["IsCustomer"]:
-                    customer_attributes.append(
-                        {
-                            "attribute_type": "CUSTOMER",
-                            "display_name": "Customer",
-                            "value": customer["Name"],
-                            "destination_id": customer["ContactID"],
-                            "detail": {
-                                "email": customer["EmailAddress"]
-                                if "EmailAddress" in customer
-                                else None
-                            },
-                            "active": True if customer["ContactStatus"] == "ACTIVE" else False,
-                        }
-                    )
+            customer_attributes = []
 
-                    if (customer["ContactStatus"] == "ACTIVE" and customer["ContactID"] in disabled_fields_map):
-                        disabled_fields_map.pop(customer["ContactID"])
-            sleep(0.5)
+            for customer in customers["Contacts"]:
+                customer_attributes.append(
+                    {
+                        "attribute_type": "CUSTOMER",
+                        "display_name": "Customer",
+                        "value": customer["Name"],
+                        "destination_id": customer["ContactID"],
+                        "detail": {
+                            "email": customer["EmailAddress"]
+                            if "EmailAddress" in customer
+                            else None
+                        },
+                        "active": True if customer["ContactStatus"] == "ACTIVE" else False,
+                    }
+                )
 
-        for destination_id in disabled_fields_map:
-            customer_attributes.append(
-                {
-                    "attribute_type": "CUSTOMER",
-                    "display_name": "customer",
-                    "value": disabled_fields_map[destination_id]["value"],
-                    "destination_id": destination_id,
-                    "active": False,
-                    "detail": disabled_fields_map[destination_id]["detail"],
-                }
+            DestinationAttribute.bulk_create_or_update_destination_attributes(
+                customer_attributes, "CUSTOMER", self.workspace_id, True
             )
-
-        DestinationAttribute.bulk_create_or_update_destination_attributes(
-            customer_attributes, "CUSTOMER", self.workspace_id, True
-        )
+            sleep(1)
 
         return []
 

diff --git a/requirements.txt b/requirements.txt
@@ -41,7 +41,7 @@ Unidecode==1.1.2
 urllib3==1.26.11
 wcwidth==0.1.8
 wrapt==1.12.1
-xerosdk==0.13.0
+xerosdk==0.14.0
 pytest==7.1.2
 pytest-cov==3.0.0
 pytest-django==4.5.2

diff --git a/tests/test_xero/test_utils.py b/tests/test_xero/test_utils.py
@@ -191,7 +191,7 @@ def test_sync_customers(mocker, db):
     new_customers_count = DestinationAttribute.objects.filter(
         workspace_id=workspace_id, attribute_type="CUSTOMER"
     ).count()
-    assert new_customers_count == 14
+    assert new_customers_count == 62
 
 
 def test_sync_tracking_categories(mocker, db):