Initial add reports

kolzchut · Oct 1, 2024 · 6b8bc6d · 6b8bc6d
1 parent 8febd79
commit 6b8bc6d
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 4 deletions.
diff --git a/operators/entities/__init__.py b/operators/entities/__init__.py
@@ -9,7 +9,7 @@
 
 from openlocationcode import openlocationcode as olc
 
-from srm_tools.stats import Stats
+from srm_tools.stats import Report, Stats
 from srm_tools.update_table import airtable_updater
 from srm_tools.guidestar_api import GuidestarAPI
 from srm_tools.budgetkey import fetch_from_budgetkey
@@ -44,6 +44,14 @@ def fetchEntityFromBudgetKey(regNum):
 
 
 def updateOrgFromSourceData(ga: GuidestarAPI, stats: Stats):
+
+    unknown_entity_ids = Report(
+        'Entities: Unknown ID Report',
+        'entities-unknown-id-report',
+        ['id', 'kind'],
+        ['id']
+    )
+
     def func(rows):
         for row in rows:
             regNums = [row['id']]
@@ -83,9 +91,11 @@ def func(rows):
                     row.update(data['data'])
                 else:
                     stats.increase('Entities: Unknown ID')
+                    unknown_entity_ids.add(row)
             if 'name' in row:
                 row['name'] = row['name'].replace(' (חל"צ)', '').replace(' (ע"ר)', '')
             yield row
+        unknown_entity_ids.save()
     return func
 
 def recent_org(row):

diff --git a/operators/manual_data_entry/__init__.py b/operators/manual_data_entry/__init__.py
@@ -1,21 +1,29 @@
 import dataflows as DF
 
-from dataflows_airtable import load_from_airtable
+from dataflows_airtable import load_from_airtable, AIRTABLE_ID_FIELD
 
 from conf import settings
 from srm_tools.logger import logger
-from srm_tools.stats import Stats
+from srm_tools.stats import Report, Stats
 from .mde_utils import load_manual_data
 from .external import main as load_external_data
 
 stats = Stats()
 
 def mde_prepare():
+
+    unknown_entity_ids = Report(
+        'Manual Data Entry: No Org ID or Org Name Report',
+        'manual-data-entry-no-org-id-or-org-name',
+        ['Org Id', 'Org Name', 'Service Name'],
+        [AIRTABLE_ID_FIELD]
+    )
+
     source_flow = DF.Flow(
         load_from_airtable(settings.AIRTABLE_DATAENTRY_BASE, settings.AIRTABLE_SERVICE_TABLE, settings.AIRTABLE_VIEW, settings.AIRTABLE_API_KEY),
         DF.filter_rows(lambda r: r.get('Org Id') != 'dummy'),
-        stats.filter_with_stat('Manual Data Entry: No Org ID or Org Name', lambda r: (r.get('Org Id') or r.get('Org Name'))),
         stats.filter_with_stat('Manual Data Entry: Entry not ready to publish', lambda r: r.get('Status') == 'בייצור'),
+        stats.filter_with_stat('Manual Data Entry: No Org ID or Org Name', lambda r: (r.get('Org Id') or r.get('Org Name')), report=unknown_entity_ids),
     )
 
     data_sources = DF.Flow(