Max retry for exports limit (#161)

* Max retry for exports limit

* test case resolved

apps/business_central/exports/helpers.py

@@ -1,6 +1,7 @@
 import itertools
 import logging
 import traceback
+from datetime import datetime, timezone, timedelta
 
 from fyle_accounting_mappings.models import CategoryMapping, EmployeeMapping, ExpenseAttribute, Mapping
 from fyle_integrations_platform_connector import PlatformConnector
@@ -205,3 +206,14 @@ def load_attachments(
             accounting_export.workspace_id,
             {"error": error},
         )
+
+
+def validate_failing_export(is_auto_export: bool, interval_hours: int, error: Error):
+    """
+    Validate failing export
+    :param is_auto_export: Is auto export
+    :param interval_hours: Interval hours
+    :param error: Error
+    """
+    # If auto export is enabled and interval hours is set and error repetition count is greater than 100, export only once a day
+    return is_auto_export and interval_hours and error and error.repetition_count > 100 and datetime.now().replace(tzinfo=timezone.utc) - error.updated_at <= timedelta(hours=24)

apps/business_central/exports/journal_entry/queues.py

@@ -1,13 +1,20 @@
 from typing import List
+import logging
+
 
 from django.db.models import Q
 from django_q.tasks import Chain
 
-from apps.accounting_exports.models import AccountingExport
+from apps.accounting_exports.models import AccountingExport, Error
+from apps.business_central.exports.helpers import validate_failing_export
 from apps.workspaces.models import FyleCredential
 
 
-def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str]):
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
+
+
+def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str], is_auto_export: bool, interval_hours: int):
     """
     Check accounting export group and start export
     """
@@ -18,10 +25,17 @@ def check_accounting_export_and_start_import(workspace_id: int, accounting_expor
         workspace_id=workspace_id, id__in=accounting_export_ids, journal_entry__id__isnull=True,
         exported_at__isnull=True).all()
 
+    errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, accounting_export_id__in=accounting_export_ids).all()
+
     chain = Chain()
     chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials)
 
     for index, accounting_export_group in enumerate(accounting_exports):
+        error = errors.filter(workspace_id=workspace_id, accounting_export=accounting_export_group, is_resolved=False).first()
+        skip_export = validate_failing_export(is_auto_export, interval_hours, error)
+        if skip_export:
+            logger.info('Skipping expense group %s as it has %s errors', accounting_export_group.id, error.repetition_count)
+            continue
         accounting_export, _ = AccountingExport.objects.update_or_create(
             workspace_id=accounting_export_group.workspace_id,
             id=accounting_export_group.id,

apps/business_central/exports/journal_entry/tasks.py

@@ -26,11 +26,11 @@ def __init__(self):
         self.body_model = JournalEntry
         self.lineitem_model = JournalEntryLineItems
 
-    def trigger_export(self, workspace_id, accounting_export_ids):
+    def trigger_export(self, workspace_id, accounting_export_ids, is_auto_export, interval_hours):
         '''
         Trigger the import process for the Project module.
         '''
-        check_accounting_export_and_start_import(workspace_id, accounting_export_ids)
+        check_accounting_export_and_start_import(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     def __construct_journal_entry(self, body: JournalEntry, lineitems: List[JournalEntryLineItems]) -> Dict:
         '''

apps/business_central/exports/purchase_invoice/queues.py

@@ -1,27 +1,38 @@
 from typing import List
+import logging
 
 from django.db.models import Q
 from django_q.tasks import Chain
 
-from apps.accounting_exports.models import AccountingExport
+from apps.accounting_exports.models import AccountingExport, Error
+from apps.business_central.exports.helpers import validate_failing_export
 from apps.workspaces.models import FyleCredential
 
+logger = logging.getLogger(__name__)
+logger.level = logging.INFO
 
-def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str]):
+
+def check_accounting_export_and_start_import(workspace_id: int, accounting_export_ids: List[str], is_auto_export: bool, interval_hours: int):
     """
     Check accounting export group and start export
     """
 
     fyle_credentials = FyleCredential.objects.filter(workspace_id=workspace_id).first()
 
     accounting_exports = AccountingExport.objects.filter(~Q(status__in=['IN_PROGRESS', 'COMPLETE', 'EXPORT_QUEUED']),
-        workspace_id=workspace_id, id__in=accounting_export_ids, purchase_invoice__id__isnull=True,
-        exported_at__isnull=True).all()
+        workspace_id=workspace_id, id__in=accounting_export_ids, purchase_invoice__id__isnull=True, exported_at__isnull=True).all()
+
+    errors = Error.objects.filter(workspace_id=workspace_id, is_resolved=False, accounting_export_id__in=accounting_export_ids).all()
 
     chain = Chain()
     chain.append('apps.fyle.helpers.sync_dimensions', fyle_credentials)
 
     for index, accounting_export_group in enumerate(accounting_exports):
+        error = errors.filter(workspace_id=workspace_id, accounting_export=accounting_export_group, is_resolved=False).first()
+        skip_export = validate_failing_export(is_auto_export, interval_hours, error)
+        if skip_export:
+            logger.info('Skipping expense group %s as it has %s errors', accounting_export_group.id, error.repetition_count)
+            continue
         accounting_export, _ = AccountingExport.objects.update_or_create(
             workspace_id=accounting_export_group.workspace_id,
             id=accounting_export_group.id,

apps/business_central/exports/purchase_invoice/tasks.py

@@ -27,11 +27,11 @@ def __init__(self):
         self.body_model = PurchaseInvoice
         self.lineitem_model = PurchaseInvoiceLineitems
 
-    def trigger_export(self, workspace_id, accounting_export_ids):
+    def trigger_export(self, workspace_id, accounting_export_ids, is_auto_export, interval_hours):
         '''
         Trigger the import process for the Project module.
         '''
-        check_accounting_export_and_start_import(workspace_id, accounting_export_ids)
+        check_accounting_export_and_start_import(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     def __construct_purchase_invoice(self, body: PurchaseInvoice, lineitems: List[PurchaseInvoiceLineitems]) -> Dict:
         '''

apps/workspaces/tasks.py

@@ -37,6 +37,9 @@ def run_import_export(workspace_id: int, export_mode = None):
         workspace_id=workspace_id
     )
 
+    interval_hours = advance_settings.interval_hours
+    is_auto_export = advance_settings.schedule_is_enabled
+
     last_exported_at = datetime.now()
     is_expenses_exported = False
 
@@ -61,7 +64,7 @@ def run_import_export(workspace_id: int, export_mode = None):
                 is_expenses_exported = True
                 export = export_map.get(export_settings.reimbursable_expenses_export_type, None)
                 if export:
-                    export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids)
+                    export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     # For Credit Card Expenses
     if export_settings.credit_card_expense_export_type:
@@ -78,7 +81,7 @@ def run_import_export(workspace_id: int, export_mode = None):
                 is_expenses_exported = True
                 export = export_map.get(export_settings.credit_card_expense_export_type, None)
                 if export:
-                    export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids)
+                    export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     if is_expenses_exported:
         accounting_summary.last_exported_at = last_exported_at
@@ -139,6 +142,9 @@ def export_to_business_central(workspace_id: int):
     export_settings = ExportSetting.objects.get(workspace_id=workspace_id)
     advance_settings = AdvancedSetting.objects.filter(workspace_id=workspace_id).first()
 
+    is_auto_export = False
+    interval_hours = 0
+
     # Update or create an AccountingExportSummary for the workspace
     accounting_summary, _ = AccountingExportSummary.objects.update_or_create(workspace_id=workspace_id)
 
@@ -166,7 +172,7 @@ def export_to_business_central(workspace_id: int):
             # Get the appropriate export class and trigger the export
             export = export_map.get(export_settings.reimbursable_expenses_export_type, None)
             if export:
-                export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids)
+                export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     # Check and export credit card expenses if configured
     if export_settings.credit_card_expense_export_type:
@@ -180,7 +186,7 @@ def export_to_business_central(workspace_id: int):
             # Get the appropriate export class and trigger the export
             export = export_map.get(export_settings.credit_card_expense_export_type, None)
             if export:
-                export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids)
+                export.trigger_export(workspace_id=workspace_id, accounting_export_ids=accounting_export_ids, is_auto_export=is_auto_export, interval_hours=interval_hours)
 
     # Update the accounting summary if expenses are exported
     if is_expenses_exported:

tests/test_business_central/test_queues.py

@@ -29,7 +29,9 @@ def test_check_accounting_export_and_start_import_journal_entry(
 
     check_accounting_export_and_start_import_journal_entry(
         accounting_export.workspace_id,
-        [accounting_export.id]
+        [accounting_export.id],
+        False,
+        0
     )
 
     accounting_export.refresh_from_db()
@@ -42,7 +44,9 @@ def test_check_accounting_export_and_start_import_journal_entry(
 
     check_accounting_export_and_start_import_journal_entry(
         accounting_export.workspace_id,
-        [accounting_export.id]
+        [accounting_export.id],
+        False,
+        0
     )
 
     accounting_export.refresh_from_db()
@@ -70,7 +74,9 @@ def test_check_accounting_export_and_start_import_purchase_invoice(
 
     check_accounting_export_and_start_import_purchase_invoice(
         accounting_export.workspace_id,
-        [accounting_export.id]
+        [accounting_export.id],
+        False,
+        0
     )
 
     accounting_export.refresh_from_db()
@@ -83,7 +89,9 @@ def test_check_accounting_export_and_start_import_purchase_invoice(
 
     check_accounting_export_and_start_import_purchase_invoice(
         accounting_export.workspace_id,
-        [accounting_export.id]
+        [accounting_export.id],
+        False,
+        0
     )
 
     accounting_export.refresh_from_db()

tests/test_business_central/test_tasks.py

@@ -18,7 +18,7 @@ def test_trigger_export_journal_entry(db, mocker):
     )
 
     export_journal_entry = ExportJournalEntry()
-    export_journal_entry.trigger_export(1, [1])
+    export_journal_entry.trigger_export(1, [1], False, 0)
 
     assert True
 
@@ -140,7 +140,7 @@ def test_trigger_export_purchase_invoice(db, mocker):
     )
 
     export_purchase_invoice = ExportPurchaseInvoice()
-    export_purchase_invoice.trigger_export(1, [1])
+    export_purchase_invoice.trigger_export(1, [1], False, 0)
 
     assert True